From ea5cd2e3437f9d9572bb84892204552c5b8413bf Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 8 Nov 2024 17:39:04 +0100 Subject: [PATCH 01/33] feat: working on new buffer structure --- .../xcshareddata/IDEWorkspaceChecks.plist | 8 +++ .../src/main/cpp/AudioPlayer/AudioPlayer.cpp | 7 ++ .../src/main/cpp/AudioPlayer/AudioPlayer.h | 1 + .../common/cpp/core/AudioArray.cpp | 50 +++++++++++++ .../common/cpp/core/AudioArray.h | 27 +++++++ .../common/cpp/core/AudioBuffer.h | 2 +- .../common/cpp/core/AudioBufferSourceNode.cpp | 70 +++++++++--------- .../common/cpp/core/AudioBufferSourceNode.h | 2 +- .../common/cpp/core/AudioChannel.cpp | 0 .../common/cpp/core/AudioChannel.h | 31 ++++++++ .../common/cpp/core/AudioDestinationNode.cpp | 30 ++++---- .../common/cpp/core/AudioDestinationNode.h | 2 +- .../common/cpp/core/AudioNode.cpp | 22 +++--- .../common/cpp/core/AudioNode.h | 6 +- .../common/cpp/core/AudioNodeBuffer.h | 31 ++++++++ .../common/cpp/core/BaseAudioContext.cpp | 4 ++ .../common/cpp/core/BaseAudioContext.h | 2 + .../common/cpp/core/BiquadFilterNode.cpp | 72 +++++++++---------- .../common/cpp/core/BiquadFilterNode.h | 2 +- .../common/cpp/core/GainNode.cpp | 24 +++---- .../common/cpp/core/GainNode.h | 2 +- .../common/cpp/core/OscillatorNode.cpp | 58 +++++++-------- .../common/cpp/core/OscillatorNode.h | 2 +- .../common/cpp/core/StereoPannerNode.cpp | 64 ++++++++--------- .../common/cpp/core/StereoPannerNode.h | 2 +- .../common/cpp/utils/VectorMath.cpp | 4 +- .../common/cpp/utils/VectorMath.h | 2 +- .../ios/AudioPlayer/AudioPlayer.h | 2 + .../ios/AudioPlayer/AudioPlayer.m | 22 ++++-- .../ios/AudioPlayer/IOSAudioPlayer.h | 3 + .../ios/AudioPlayer/IOSAudioPlayer.mm | 15 ++-- .../src/core/AudioScheduledSourceNode.ts | 2 + 32 files changed, 379 insertions(+), 192 deletions(-) create mode 100644 apps/fabric-example/ios/FabricExample.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioArray.cpp create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioArray.h create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioChannel.h create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h diff --git a/apps/fabric-example/ios/FabricExample.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/apps/fabric-example/ios/FabricExample.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 00000000..18d98100 --- /dev/null +++ b/apps/fabric-example/ios/FabricExample.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp index 029ebb30..9616737e 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp @@ -2,9 +2,11 @@ #include "AudioContext.h" namespace audioapi { + AudioPlayer::AudioPlayer(const std::function &renderAudio) : renderAudio_(renderAudio) { AudioStreamBuilder builder; + builder.setSharingMode(SharingMode::Exclusive) ->setFormat(AudioFormat::Float) ->setFormatConversionAllowed(true) @@ -19,6 +21,10 @@ int AudioPlayer::getSampleRate() const { return mStream_->getSampleRate(); } +int AudioPlayer::getBufferSizeInFrames() const { + return mStream_->getBufferSizeInFrames(); +} + void AudioPlayer::start() { if (mStream_) { mStream_->requestStart(); @@ -42,4 +48,5 @@ DataCallbackResult AudioPlayer::onAudioReady( return DataCallbackResult::Continue; } + } // namespace audioapi diff --git a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h index 02994793..41f4f860 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h +++ b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h @@ -14,6 +14,7 @@ class AudioPlayer : public AudioStreamDataCallback { explicit AudioPlayer(const std::function &renderAudio); int getSampleRate() const; + int getBufferSizeInFrames() const; void start(); void stop(); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp new file mode 100644 index 00000000..2c06d562 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -0,0 +1,50 @@ +#include "AudioArray.h" + +namespace audioapi { + +AudioArray::AudioArray(int size) : size_(size) { + resize(size); +}; + +AudioArray::~AudioArray() { + delete[] data_; +}; + +int AudioArray::getSize() const { + return size_; +} + +float& AudioArray::operator[](int index) { + return data_[index]; +} + +const float& AudioArray::operator[](int index) const { + return data_[index]; +} + +void AudioArray::zero() { + memset(data_, 0, size_ * sizeof(float)); +} + +void AudioArray::resize(int size) { + if (size == size_) { + zero(); + return; + } + + delete[] data_; + size_ = size; + data_ = new float[size_]; + + zero(); +} + +void AudioArray::copy(const AudioArray &source) { + if (size_ != source.size_) { + resize(source.size_); + } + + memcpy(data_, source.data_, size_ * sizeof(float)); +} + +} diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.h b/packages/react-native-audio-api/common/cpp/core/AudioArray.h new file mode 100644 index 00000000..564721c3 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace audioapi { + +class AudioArray { + public: + explicit AudioArray(int size); + ~AudioArray(); + + [[nodiscard]] int getSize() const; + + + float& operator[](int index); + const float& operator[](int index) const; + + void zero(); + void resize(int size); + void copy(const AudioArray &source); + + private: + float *data_; + int size_; +}; + +} diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h index 073b6e87..3e04f1ac 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h @@ -16,6 +16,7 @@ class AudioBuffer : public std::enable_shared_from_this { [[nodiscard]] int getSampleRate() const; [[nodiscard]] double getDuration() const; [[nodiscard]] float *getChannelData(int channel) const; + void copyFromChannel( float *destination, int destinationLength, @@ -28,7 +29,6 @@ class AudioBuffer : public std::enable_shared_from_this { int startInChannel); private: - friend class AudioBufferSourceNode; int numberOfChannels_; int length_; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index 05ffa043..97cbe8f7 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -5,55 +5,55 @@ namespace audioapi { AudioBufferSourceNode::AudioBufferSourceNode(BaseAudioContext *context) : AudioScheduledSourceNode(context), loop_(false), bufferIndex_(0) { - numberOfInputs_ = 0; - buffer_ = std::shared_ptr(nullptr); + // numberOfInputs_ = 0; + // buffer_ = std::shared_ptr(nullptr); } bool AudioBufferSourceNode::getLoop() const { - return loop_; + // return loop_; } std::shared_ptr AudioBufferSourceNode::getBuffer() const { - return buffer_; + // return buffer_; } void AudioBufferSourceNode::setLoop(bool loop) { - loop_ = loop; + // loop_ = loop; } void AudioBufferSourceNode::setBuffer( const std::shared_ptr &buffer) { - if (!buffer) { - buffer_ = std::shared_ptr(nullptr); - return; - } + // if (!buffer) { + // buffer_ = std::shared_ptr(nullptr); + // return; + // } - buffer_ = buffer->mix(channelCount_); + // buffer_ = buffer->mix(channelCount_); } -bool AudioBufferSourceNode::processAudio(float *audioData, int32_t numFrames) { - if (!isPlaying_ || !buffer_) { - return false; - } else { - for (int i = 0; i < numFrames; ++i) { - for (int j = 0; j < channelCount_; j++) { - audioData[i * channelCount_ + j] = - buffer_->getChannelData(j)[bufferIndex_]; - } - - bufferIndex_++; - - if (bufferIndex_ >= buffer_->getLength()) { - if (loop_) { - bufferIndex_ = 0; - } else { - isPlaying_ = false; - break; - } - } - } - - return true; - } -} +// bool AudioBufferSourceNode::processAudio(float *audioData, int32_t numFrames) { +// if (!isPlaying_ || !buffer_) { +// return false; +// } else { +// for (int i = 0; i < numFrames; ++i) { +// for (int j = 0; j < channelCount_; j++) { +// audioData[i * channelCount_ + j] = +// buffer_->getChannelData(j)[bufferIndex_]; +// } + +// bufferIndex_++; + +// if (bufferIndex_ >= buffer_->getLength()) { +// if (loop_) { +// bufferIndex_ = 0; +// } else { +// isPlaying_ = false; +// break; +// } +// } +// } + +// return true; +// } +// } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h index 4abce23e..ed993357 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h @@ -15,7 +15,7 @@ class AudioBufferSourceNode : public AudioScheduledSourceNode { [[nodiscard]] std::shared_ptr getBuffer() const; void setLoop(bool loop); void setBuffer(const std::shared_ptr &buffer); - [[nodiscard]] bool processAudio(float *audioData, int32_t numFrames) override; + // [[nodiscard]] bool processAudio(float *audioData, int32_t numFrames) override; private: bool loop_; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp b/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp new file mode 100644 index 00000000..e69de29b diff --git a/packages/react-native-audio-api/common/cpp/core/AudioChannel.h b/packages/react-native-audio-api/common/cpp/core/AudioChannel.h new file mode 100644 index 00000000..add540d4 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioChannel.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include "AudioArray.h" + +namespace audioapi { + +class AudioChannel { + public: + explicit AudioChannel(int length, int sampleRate); + + [[nodiscard]] int getLength() const; + [[nodiscard]] int getSampleRate() const; + + void zero(); + void scale(float scale); + + void sumFrom(const AudioChannel &source); + void copyFrom(const AudioChannel &source); + + float maxAbsValue() const; + + private: + + int length_; + int sampleRate_; + + std::unique_ptr data_; +}; + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 3febab08..d3bb6327 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -11,27 +11,27 @@ AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) } void AudioDestinationNode::renderAudio(float *audioData, int32_t numFrames) { - processAudio(audioData, numFrames); + // processAudio(audioData, numFrames); } -bool AudioDestinationNode::processAudio(float *audioData, int32_t numFrames) { - int numSamples = numFrames * CHANNEL_COUNT; +// bool AudioDestinationNode::processAudio(float *audioData, int32_t numFrames) { +// int numSamples = numFrames * CHANNEL_COUNT; - if (mixingBuffer == nullptr) { - mixingBuffer = std::make_unique(numSamples); - } +// if (mixingBuffer == nullptr) { +// mixingBuffer = std::make_unique(numSamples); +// } - memset(audioData, 0.0f, sizeof(float) * numSamples); +// memset(audioData, 0.0f, sizeof(float) * numSamples); - for (auto &node : inputNodes_) { - if (node && node->processAudio(mixingBuffer.get(), numFrames)) { - normalize(mixingBuffer.get(), numFrames); - VectorMath::add(audioData, mixingBuffer.get(), audioData, numSamples); - } - } +// for (auto &node : inputNodes_) { +// if (node && node->processAudio(mixingBuffer.get(), numFrames)) { +// normalize(mixingBuffer.get(), numFrames); +// VectorMath::add(audioData, mixingBuffer.get(), audioData, numSamples); +// } +// } - return true; -} +// return true; +// } void AudioDestinationNode::normalize(float *audioData, int32_t numFrames) { auto maxValue = std::max( diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h index 424cb7f3..8ee32082 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h @@ -16,7 +16,7 @@ class AudioDestinationNode : public AudioNode { void renderAudio(float *audioData, int32_t numFrames); protected: - bool processAudio(float *audioData, int32_t numFrames) override; + // bool processAudio(float *audioData, int32_t numFrames) override; private: std::unique_ptr mixingBuffer; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index c7fff825..6ee29a77 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -3,7 +3,9 @@ namespace audioapi { -AudioNode::AudioNode(BaseAudioContext *context) : context_(context) {} +AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { + buffer_ = std::make_unique(context_, channelCount_); +} AudioNode::~AudioNode() { cleanup(); @@ -54,15 +56,15 @@ void AudioNode::cleanup() { inputNodes_.clear(); } -bool AudioNode::processAudio(float *audioData, int32_t numFrames) { - bool isPlaying = false; - for (auto &node : inputNodes_) { - if (node->processAudio(audioData, numFrames)) { - isPlaying = true; - } - } +// bool AudioNode::processAudio(float *audioData, int32_t numFrames) { +// bool isPlaying = false; +// for (auto &node : inputNodes_) { +// if (node->processAudio(audioData, numFrames)) { +// isPlaying = true; +// } +// } - return isPlaying; -} +// return isPlaying; +// } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index 071d4f67..619080ef 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -4,6 +4,7 @@ #include #include #include "Constants.h" +#include "AudioNodeBuffer.h" // channelCount always equal to 2 @@ -24,7 +25,7 @@ class AudioNode : public std::enable_shared_from_this { void disconnect(const std::shared_ptr &node); // Change public to protected - virtual bool processAudio(float *audioData, int32_t numFrames); + // virtual bool processAudio(float *audioData, int32_t numFrames); protected: enum class ChannelCountMode { MAX, CLAMPED_MAX, EXPLICIT }; @@ -57,9 +58,12 @@ class AudioNode : public std::enable_shared_from_this { protected: BaseAudioContext *context_; + std::unique_ptr buffer_; + int numberOfInputs_ = 1; int numberOfOutputs_ = 1; int channelCount_ = CHANNEL_COUNT; + ChannelCountMode channelCountMode_ = ChannelCountMode::MAX; ChannelInterpretation channelInterpretation_ = ChannelInterpretation::SPEAKERS; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h new file mode 100644 index 00000000..831ccd81 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +// namespace audioapi { + +// class AudioNodeBuffer { +// public: +// explicit AudioNodeBuffer(BaseAudioContext * context, int channelCount); + +// void zero(); +// void normalize(); +// void scale(float scale); + +// void sumFrom(const AudioNodeBuffer &source); +// void copyFrom(const AudioNodeBuffer &source); + +// float maxAbsValue() const; + +// private: +// BaseAudioContext *context_; +// std::unique_ptr buffer_; + +// int bufferSizeInFrames_; +// int channelCount_; +// }; + +// } diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp index d2cb745b..596b967e 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp @@ -29,6 +29,10 @@ int BaseAudioContext::getSampleRate() const { return sampleRate_; } +int BaseAudioContext::getBufferSizeInFrames() const { + return bufferSizeInFrames_; +} + double BaseAudioContext::getCurrentTime() const { auto now = std::chrono::high_resolution_clock ::now(); auto currentTime = diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h index d19b1933..c7d71af6 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h @@ -30,6 +30,7 @@ class BaseAudioContext { std::string getState(); [[nodiscard]] int getSampleRate() const; [[nodiscard]] double getCurrentTime() const; + [[nodiscard]] int getBufferSizeInFrames() const; std::shared_ptr getDestination(); std::shared_ptr createOscillator(); @@ -66,6 +67,7 @@ class BaseAudioContext { #endif State state_ = State::RUNNING; int sampleRate_; + int bufferSizeInFrames_; double contextStartTime_; }; diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index bc27886a..fabb61e9 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -351,40 +351,40 @@ void BiquadFilterNode::applyFilter() { } } -bool BiquadFilterNode::processAudio(float *audioData, int32_t numFrames) { - if (!AudioNode::processAudio(audioData, numFrames)) { - return false; - } - - resetCoefficients(); - applyFilter(); - - float x1 = x1_; - float x2 = x2_; - float y1 = y1_; - float y2 = y2_; - - float b0 = b0_; - float b1 = b1_; - float b2 = b2_; - float a1 = a1_; - float a2 = a2_; - - for (int i = 0; i < numFrames; i++) { - auto input = audioData[i * channelCount_]; - auto output = - static_cast(b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2); - - for (int j = 0; j < channelCount_; j++) { - audioData[i * channelCount_ + j] = output; - } - - x2 = x1; - x1 = input; - y2 = y1; - y1 = output; - } - - return true; -} +// bool BiquadFilterNode::processAudio(float *audioData, int32_t numFrames) { +// if (!AudioNode::processAudio(audioData, numFrames)) { +// return false; +// } + +// resetCoefficients(); +// applyFilter(); + +// float x1 = x1_; +// float x2 = x2_; +// float y1 = y1_; +// float y2 = y2_; + +// float b0 = b0_; +// float b1 = b1_; +// float b2 = b2_; +// float a1 = a1_; +// float a2 = a2_; + +// for (int i = 0; i < numFrames; i++) { +// auto input = audioData[i * channelCount_]; +// auto output = +// static_cast(b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2); + +// for (int j = 0; j < channelCount_; j++) { +// audioData[i * channelCount_ + j] = output; +// } + +// x2 = x1; +// x1 = input; +// y2 = y1; +// y1 = output; +// } + +// return true; +// } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h index d1f504d6..7265d089 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h @@ -29,7 +29,7 @@ class BiquadFilterNode : public AudioNode { std::vector &phaseResponseOutput); protected: - bool processAudio(float *audioData, int32_t numFrames) override; + // bool processAudio(float *audioData, int32_t numFrames) override; private: enum class FilterType { diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index 2435b385..d9438fc8 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -11,20 +11,20 @@ std::shared_ptr GainNode::getGainParam() const { return gainParam_; } -bool GainNode::processAudio(float *audioData, int32_t numFrames) { - if (!AudioNode::processAudio(audioData, numFrames)) { - return false; - } +// bool GainNode::processAudio(float *audioData, int32_t numFrames) { +// if (!AudioNode::processAudio(audioData, numFrames)) { +// return false; +// } - auto time = context_->getCurrentTime(); - auto deltaTime = 1.0 / context_->getSampleRate(); +// auto time = context_->getCurrentTime(); +// auto deltaTime = 1.0 / context_->getSampleRate(); - for (int i = 0; i < numFrames * channelCount_; i++) { - audioData[i] *= gainParam_->getValueAtTime(time); - time += deltaTime; - } +// for (int i = 0; i < numFrames * channelCount_; i++) { +// audioData[i] *= gainParam_->getValueAtTime(time); +// time += deltaTime; +// } - return true; -} +// return true; +// } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.h b/packages/react-native-audio-api/common/cpp/core/GainNode.h index 7a74c203..65f12b8e 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.h +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.h @@ -14,7 +14,7 @@ class GainNode : public AudioNode { [[nodiscard]] std::shared_ptr getGainParam() const; protected: - bool processAudio(float *audioData, int32_t numFrames) override; + // bool processAudio(float *audioData, int32_t numFrames) override; private: std::shared_ptr gainParam_; diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index b8e6806c..dc790bf2 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -27,40 +27,40 @@ void OscillatorNode::setType(const std::string &type) { type_ = OscillatorNode::fromString(type); } -bool OscillatorNode::processAudio(float *audioData, int32_t numFrames) { - if (!isPlaying_) { - return false; - } else { - auto time = context_->getCurrentTime(); - auto deltaTime = 1.0 / context_->getSampleRate(); +// bool OscillatorNode::processAudio(float *audioData, int32_t numFrames) { +// if (!isPlaying_) { +// return false; +// } else { +// auto time = context_->getCurrentTime(); +// auto deltaTime = 1.0 / context_->getSampleRate(); - for (int i = 0; i < numFrames; ++i) { - auto detuneRatio = - std::pow(2.0f, detuneParam_->getValueAtTime(time) / 1200.0f); - auto detunedFrequency = - round(frequencyParam_->getValueAtTime(time) * detuneRatio); - auto phaseIncrement = static_cast( - 2 * M_PI * detunedFrequency / context_->getSampleRate()); +// for (int i = 0; i < numFrames; ++i) { +// auto detuneRatio = +// std::pow(2.0f, detuneParam_->getValueAtTime(time) / 1200.0f); +// auto detunedFrequency = +// round(frequencyParam_->getValueAtTime(time) * detuneRatio); +// auto phaseIncrement = static_cast( +// 2 * M_PI * detunedFrequency / context_->getSampleRate()); - float value = OscillatorNode::getWaveBufferElement(phase_, type_); +// float value = OscillatorNode::getWaveBufferElement(phase_, type_); - for (int j = 0; j < channelCount_; j++) { - audioData[i * channelCount_ + j] = value; - } +// for (int j = 0; j < channelCount_; j++) { +// audioData[i * channelCount_ + j] = value; +// } - phase_ += phaseIncrement; - time += deltaTime; +// phase_ += phaseIncrement; +// time += deltaTime; - if (phase_ >= 2 * M_PI) { - phase_ -= 2 * M_PI; - } +// if (phase_ >= 2 * M_PI) { +// phase_ -= 2 * M_PI; +// } - if (phase_ < 0) { - phase_ += 2 * M_PI; - } - } +// if (phase_ < 0) { +// phase_ += 2 * M_PI; +// } +// } - return true; - } -} +// return true; +// } +// } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h index b9688ff3..03a1b181 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h @@ -19,7 +19,7 @@ class OscillatorNode : public AudioScheduledSourceNode { void setType(const std::string &type); protected: - bool processAudio(float *audioData, int32_t numFrames) override; + // bool processAudio(float *audioData, int32_t numFrames) override; private: enum class WaveType { SINE, SQUARE, SAWTOOTH, TRIANGLE, CUSTOM }; diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index 8958adf4..d1752514 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -15,36 +15,36 @@ std::shared_ptr StereoPannerNode::getPanParam() const { return panParam_; } -bool StereoPannerNode::processAudio(float *audioData, int32_t numFrames) { - // assumed channelCount = 2 - if (!AudioNode::processAudio(audioData, numFrames)) { - return false; - } - - auto time = context_->getCurrentTime(); - auto deltaTime = 1.0 / context_->getSampleRate(); - - for (int i = 0; i < numFrames; i++) { - auto pan = panParam_->getValueAtTime(time); - auto x = (pan <= 0 ? pan + 1 : pan) * M_PI / 2; - - auto gainL = static_cast(cos(x)); - auto gainR = static_cast(sin(x)); - - auto inputL = audioData[i * 2]; - auto inputR = audioData[i * 2 + 1]; - - if (pan <= 0) { - audioData[i * 2] = inputL + inputR * gainL; - audioData[i * 2 + 1] = inputR * gainR; - } else { - audioData[i * 2] = inputL * gainL; - audioData[i * 2 + 1] = inputR + inputL * gainR; - } - - time += deltaTime; - } - - return true; -} +// bool StereoPannerNode::processAudio(float *audioData, int32_t numFrames) { +// // assumed channelCount = 2 +// if (!AudioNode::processAudio(audioData, numFrames)) { +// return false; +// } + +// auto time = context_->getCurrentTime(); +// auto deltaTime = 1.0 / context_->getSampleRate(); + +// for (int i = 0; i < numFrames; i++) { +// auto pan = panParam_->getValueAtTime(time); +// auto x = (pan <= 0 ? pan + 1 : pan) * M_PI / 2; + +// auto gainL = static_cast(cos(x)); +// auto gainR = static_cast(sin(x)); + +// auto inputL = audioData[i * 2]; +// auto inputR = audioData[i * 2 + 1]; + +// if (pan <= 0) { +// audioData[i * 2] = inputL + inputR * gainL; +// audioData[i * 2 + 1] = inputR * gainR; +// } else { +// audioData[i * 2] = inputL * gainL; +// audioData[i * 2 + 1] = inputR + inputL * gainR; +// } + +// time += deltaTime; +// } + +// return true; +// } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h index 10c436f8..36fb7286 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h @@ -16,7 +16,7 @@ class StereoPannerNode : public AudioNode { [[nodiscard]] std::shared_ptr getPanParam() const; protected: - bool processAudio(float *audioData, int32_t numFrames) override; + // bool processAudio(float *audioData, int32_t numFrames) override; private: std::shared_ptr panParam_; diff --git a/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp b/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp index 859b4289..8eeb2ea1 100644 --- a/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp +++ b/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp @@ -77,7 +77,7 @@ void add( numberOfElementsToProcess); } -void substract( +void subtract( const float *inputVector1, const float *inputVector2, float *outputVector, @@ -363,7 +363,7 @@ void add( } } -void substract( +void subtract( const float *inputVector1, const float *inputVector2, float *outputVector, diff --git a/packages/react-native-audio-api/common/cpp/utils/VectorMath.h b/packages/react-native-audio-api/common/cpp/utils/VectorMath.h index a7507e0c..5a6464a5 100644 --- a/packages/react-native-audio-api/common/cpp/utils/VectorMath.h +++ b/packages/react-native-audio-api/common/cpp/utils/VectorMath.h @@ -47,7 +47,7 @@ void add( const float *inputVector2, float *outputVector, size_t numberOfElementsToProcess); -void substract( +void subtract( const float *inputVector1, const float *inputVector2, float *outputVector, diff --git a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h index d8b9a9d0..2d0ffff9 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h +++ b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h @@ -18,6 +18,8 @@ typedef void (^RenderAudioBlock)(float *audioData, int numFrames); - (int)getSampleRate; +- (int)getBufferSizeInFrames; + - (void)start; - (void)stop; diff --git a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m index 5b3f4932..1c7f61e2 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m +++ b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m @@ -12,6 +12,8 @@ - (instancetype)initWithRenderAudioBlock:(RenderAudioBlock)renderAudio self.audioSession = AVAudioSession.sharedInstance; NSError *error = nil; + NSLog(@"buffer time: %f %f", self.audioSession.IOBufferDuration, self.audioSession.IOBufferDuration * self.audioSession.sampleRate); + // TODO: // We will probably want to change it to AVAudioSessionCategoryPlayAndRecord in the future. // Eventually we to make this a dynamic setting, if user of the lib wants to use recording features. @@ -43,7 +45,7 @@ - (instancetype)initWithRenderAudioBlock:(RenderAudioBlock)renderAudio outputData:outputData]; }]; - self.buffer = nil; + self.buffer = malloc([self getBufferSizeInFrames] * 2 * sizeof(float)); } return self; @@ -54,11 +56,17 @@ - (int)getSampleRate return [self.audioSession sampleRate]; } +- (int)getBufferSizeInFrames +{ + return (int)(self.audioSession.IOBufferDuration * self.audioSession.sampleRate); +} + - (void)start { [self.audioEngine attachNode:self.sourceNode]; [self.audioEngine connect:self.sourceNode to:self.audioEngine.mainMixerNode format:self.format]; + if (!self.audioEngine.isRunning) { NSError *error = nil; if (![self.audioEngine startAndReturnError:&error]) { @@ -101,18 +109,18 @@ - (OSStatus)renderCallbackWithIsSilence:(BOOL *)isSilence return noErr; // Ensure we have stereo output } - if (!self.buffer) { - self.buffer = malloc(frameCount * 2 * sizeof(float)); - } + memset(self.buffer, 0, frameCount * 2 * sizeof(float)); float *leftBuffer = (float *)outputData->mBuffers[0].mData; float *rightBuffer = (float *)outputData->mBuffers[1].mData; - self.renderAudio(self.buffer, frameCount); + // self.renderAudio(self.buffer, frameCount); for (int frame = 0; frame < frameCount; frame += 1) { - leftBuffer[frame] = self.buffer[frame * 2]; - rightBuffer[frame] = self.buffer[frame * 2 + 1]; + leftBuffer[frame] = 0; + rightBuffer[frame] = 0; + // leftBuffer[frame] = self.buffer[frame * 2]; + // rightBuffer[frame] = self.buffer[frame * 2 + 1]; } return noErr; diff --git a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h index d523bba1..19832eaa 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h +++ b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h @@ -22,8 +22,11 @@ class IOSAudioPlayer { ~IOSAudioPlayer(); int getSampleRate() const; + int getBufferSizeInFrames() const; + void start(); void stop(); void renderAudio(float *audioData, int32_t numFrames); + }; } // namespace audioapi diff --git a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm index ecc2852a..c6a9a0a3 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm @@ -17,11 +17,6 @@ [audioPlayer_ cleanup]; } -int IOSAudioPlayer::getSampleRate() const -{ - return [audioPlayer_ getSampleRate]; -} - void IOSAudioPlayer::start() { return [audioPlayer_ start]; @@ -31,4 +26,14 @@ { return [audioPlayer_ stop]; } + +int IOSAudioPlayer::getSampleRate() const +{ + return [audioPlayer_ getSampleRate]; +} + +int IOSAudioPlayer::getBufferSizeInFrames() const +{ + return [audioPlayer_ getBufferSizeInFrames]; +} } // namespace audioapi diff --git a/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts b/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts index 09817403..7c1051c0 100644 --- a/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts +++ b/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts @@ -21,6 +21,7 @@ export default class AudioScheduledSourceNode extends AudioNode { throw new InvalidStateError('Cannot call start more than once'); } + this.hasBeenStarted = true; (this.node as IAudioScheduledSourceNode).start(when); } @@ -37,6 +38,7 @@ export default class AudioScheduledSourceNode extends AudioNode { ); } + // TODO: should we reset hasBeenStarted here? :thunk: (this.node as IAudioScheduledSourceNode).stop(when); } } From 7ca0ce7592cf2b86da85c7a0b4c9ab5eacd86138 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Wed, 13 Nov 2024 10:20:45 +0100 Subject: [PATCH 02/33] feat: more work --- .../common/cpp/core/AudioArray.cpp | 26 ++++++++++++++++++- .../common/cpp/core/AudioArray.h | 6 +++++ .../common/cpp/core/AudioChannel.cpp | 18 +++++++++++++ .../common/cpp/core/AudioChannel.h | 14 +++++++--- .../common/cpp/core/AudioDestinationNode.cpp | 1 + .../common/cpp/core/AudioDestinationNode.h | 1 - .../common/cpp/core/StereoPannerNode.h | 1 - 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index 2c06d562..b88028f0 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -1,4 +1,5 @@ #include "AudioArray.h" +#include "VectorMath.h" namespace audioapi { @@ -47,4 +48,27 @@ void AudioArray::copy(const AudioArray &source) { memcpy(data_, source.data_, size_ * sizeof(float)); } -} +float AudioArray::getMaxAbsValue() const { + return VectorMath::maximumMagnitude(data_, size_); +}; + +void AudioArray::normalize() { + float maxAbsValue = getMaxAbsValue(); + + if (maxAbsValue == 0.0f || maxAbsValue == 1.0f) { + return; + } + + VectorMath::multiplyByScalar(data_, 1.0f / maxAbsValue, data_, size_); +}; + +void AudioArray::scale(float value) { + VectorMath::multiplyByScalar(data_, value, data_, size_); +}; + +void AudioArray::sum(const AudioArray &source) { + VectorMath::add(data_, source.data_, data_, size_); +}; + +} // namespace audioapi + diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.h b/packages/react-native-audio-api/common/cpp/core/AudioArray.h index 564721c3..a1cd1e30 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.h @@ -19,6 +19,12 @@ class AudioArray { void resize(int size); void copy(const AudioArray &source); + float getMaxAbsValue() const; + + void normalize(); + void scale(float value); + void sum(const AudioArray &source); + private: float *data_; int size_; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp b/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp index e69de29b..a0880f53 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp @@ -0,0 +1,18 @@ +#include "AudioChannel.h" +#include "AudioArray.h" + +namespace audioapi { + +explicit AudioChannel::AudioChannel(int length, int sampleRate) : length_(length), sampleRate_(sampleRate) { + data_ = std::make_unique(length_); +}; + +int AudioChannel::getLength() const { + return length_; +}; + +int AudioChannel::getSampleRate() const { + return sampleRate_; +}; + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioChannel.h b/packages/react-native-audio-api/common/cpp/core/AudioChannel.h index add540d4..78054fc5 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioChannel.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioChannel.h @@ -13,12 +13,18 @@ class AudioChannel { [[nodiscard]] int getSampleRate() const; void zero(); - void scale(float scale); + void copy(const AudioChannel &source); + + float& operator[](int index); + const float& operator[](int index) const; + - void sumFrom(const AudioChannel &source); - void copyFrom(const AudioChannel &source); + float getMaxAbsValue() const; + + void normalize(); + void scale(float scale); + void sum(const AudioChannel &source); - float maxAbsValue() const; private: diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index d3bb6327..8ab3d4e6 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -1,5 +1,6 @@ #include "AudioDestinationNode.h" #include "BaseAudioContext.h" +#include "VectorMath.h" namespace audioapi { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h index 8ee32082..048dc3bd 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h @@ -5,7 +5,6 @@ #include #include "AudioNode.h" -#include "VectorMath.h" namespace audioapi { diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h index 36fb7286..d49ad1a4 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h @@ -5,7 +5,6 @@ #include "AudioNode.h" #include "AudioParam.h" -#include "VectorMath.h" namespace audioapi { From 3b4c9c78b5074bf0d699d42a84d7052b39b95612 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 15 Nov 2024 10:59:45 +0100 Subject: [PATCH 03/33] feat: channel mixer --- .../common/cpp/core/AudioArray.cpp | 16 +- .../common/cpp/core/AudioArray.h | 6 +- .../common/cpp/core/AudioBuffer.cpp | 146 ++++---- .../common/cpp/core/AudioBuffer.h | 9 +- .../common/cpp/core/AudioBus.cpp | 316 ++++++++++++++++++ .../common/cpp/core/AudioBus.h | 57 ++++ .../common/cpp/core/AudioChannel.cpp | 18 - .../common/cpp/core/AudioChannel.h | 37 -- .../common/cpp/core/AudioNode.h | 4 +- .../common/cpp/core/AudioNodeBuffer.h | 31 -- .../common/cpp/utils/VectorMath.cpp | 73 +++- .../common/cpp/utils/VectorMath.h | 6 + 12 files changed, 550 insertions(+), 169 deletions(-) create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioBus.cpp create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioBus.h delete mode 100644 packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp delete mode 100644 packages/react-native-audio-api/common/cpp/core/AudioChannel.h delete mode 100644 packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index b88028f0..c137db83 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -15,6 +15,10 @@ int AudioArray::getSize() const { return size_; } +float* AudioArray::getData() const { + return data_; +} + float& AudioArray::operator[](int index) { return data_[index]; } @@ -40,12 +44,12 @@ void AudioArray::resize(int size) { zero(); } -void AudioArray::copy(const AudioArray &source) { - if (size_ != source.size_) { - resize(source.size_); +void AudioArray::copy(const AudioArray* source) { + if (size_ != source->getSize()) { + resize(source->getSize()); } - memcpy(data_, source.data_, size_ * sizeof(float)); + memcpy(data_, source->getData(), size_ * sizeof(float)); } float AudioArray::getMaxAbsValue() const { @@ -66,8 +70,8 @@ void AudioArray::scale(float value) { VectorMath::multiplyByScalar(data_, value, data_, size_); }; -void AudioArray::sum(const AudioArray &source) { - VectorMath::add(data_, source.data_, data_, size_); +void AudioArray::sum(const AudioArray* source) { + VectorMath::add(data_, source->getData(), data_, size_); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.h b/packages/react-native-audio-api/common/cpp/core/AudioArray.h index a1cd1e30..0b79060a 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.h @@ -10,6 +10,7 @@ class AudioArray { ~AudioArray(); [[nodiscard]] int getSize() const; + float* getData() const; float& operator[](int index); @@ -17,13 +18,14 @@ class AudioArray { void zero(); void resize(int size); - void copy(const AudioArray &source); + void copy(const AudioArray* source); float getMaxAbsValue() const; void normalize(); void scale(float value); - void sum(const AudioArray &source); + void sum(const AudioArray* source); + private: float *data_; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp index 28a28baa..0a7a3e7f 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp @@ -7,81 +7,91 @@ AudioBuffer::AudioBuffer(int numberOfChannels, int length, int sampleRate) length_(length), sampleRate_(sampleRate), duration_(static_cast(length) / sampleRate) { - channels_ = new float *[numberOfChannels]; - for (int i = 0; i < numberOfChannels; i++) { - channels_[i] = new float[length]; - - for (int j = 0; j < length; j++) { - channels_[i][j] = 0.0f; - } - } } -int AudioBuffer::getNumberOfChannels() const { - return numberOfChannels_; -} +} // namespace audioapi -int AudioBuffer::getLength() const { - return length_; -} -int AudioBuffer::getSampleRate() const { - return sampleRate_; -} +// AudioBuffer::AudioBuffer(int numberOfChannels, int length, int sampleRate) +// : numberOfChannels_(numberOfChannels), +// length_(length), +// sampleRate_(sampleRate), +// duration_(static_cast(length) / sampleRate) { +// channels_ = new float *[numberOfChannels]; -double AudioBuffer::getDuration() const { - return duration_; -} +// for (int i = 0; i < numberOfChannels; i++) { +// channels_[i] = new float[length]; -float *AudioBuffer::getChannelData(int channel) const { - return channels_[channel]; -} +// for (int j = 0; j < length; j++) { +// channels_[i][j] = 0.0f; +// } +// } +// } -std::shared_ptr AudioBuffer::mix(int outputNumberOfChannels) { - if (outputNumberOfChannels == numberOfChannels_) { - return shared_from_this(); - } - - auto mixedBuffer = std::make_shared( - outputNumberOfChannels, length_, sampleRate_); - - switch (this->numberOfChannels_) { - case 1: - mixedBuffer->copyToChannel(this->channels_[0], length_, 0, 0); - mixedBuffer->copyToChannel(this->channels_[0], length_, 1, 0); - break; - case 2: - for (int i = 0; i < length_; i++) { - mixedBuffer->channels_[0][i] = - (this->channels_[0][i] + this->channels_[1][i]) / 2; - } - break; - } - - return mixedBuffer; -} +// int AudioBuffer::getNumberOfChannels() const { +// return numberOfChannels_; +// } -void AudioBuffer::copyFromChannel( - float *destination, - int destinationLength, - int channelNumber, - int startInChannel) const { - std::copy( - channels_[channelNumber] + startInChannel, - channels_[channelNumber] + startInChannel + - std::min(destinationLength, length_ - startInChannel), - destination); -} +// int AudioBuffer::getLength() const { +// return length_; +// } -void AudioBuffer::copyToChannel( - const float *source, - int sourceLength, - int channelNumber, - int startInChannel) { - std::copy( - source, - source + std::min(sourceLength, length_ - startInChannel), - channels_[channelNumber] + startInChannel); -} -} // namespace audioapi +// int AudioBuffer::getSampleRate() const { +// return sampleRate_; +// } + +// double AudioBuffer::getDuration() const { +// return duration_; +// } + +// float *AudioBuffer::getChannelData(int channel) const { +// return channels_[channel]; +// } + +// std::shared_ptr AudioBuffer::mix(int outputNumberOfChannels) { +// if (outputNumberOfChannels == numberOfChannels_) { +// return shared_from_this(); +// } + +// auto mixedBuffer = std::make_shared( +// outputNumberOfChannels, length_, sampleRate_); + +// switch (this->numberOfChannels_) { +// case 1: +// mixedBuffer->copyToChannel(this->channels_[0], length_, 0, 0); +// mixedBuffer->copyToChannel(this->channels_[0], length_, 1, 0); +// break; +// case 2: +// for (int i = 0; i < length_; i++) { +// mixedBuffer->channels_[0][i] = +// (this->channels_[0][i] + this->channels_[1][i]) / 2; +// } +// break; +// } + +// return mixedBuffer; +// } + +// void AudioBuffer::copyFromChannel( +// float *destination, +// int destinationLength, +// int channelNumber, +// int startInChannel) const { +// std::copy( +// channels_[channelNumber] + startInChannel, +// channels_[channelNumber] + startInChannel + +// std::min(destinationLength, length_ - startInChannel), +// destination); +// } + +// void AudioBuffer::copyToChannel( +// const float *source, +// int sourceLength, +// int channelNumber, +// int startInChannel) { +// std::copy( +// source, +// source + std::min(sourceLength, length_ - startInChannel), +// channels_[channelNumber] + startInChannel); +// } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h index 3e04f1ac..642519ec 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h @@ -7,14 +7,17 @@ namespace audioapi { +class AudioBus; + class AudioBuffer : public std::enable_shared_from_this { public: explicit AudioBuffer(int numberOfChannels, int length, int sampleRate); - [[nodiscard]] int getNumberOfChannels() const; [[nodiscard]] int getLength() const; [[nodiscard]] int getSampleRate() const; [[nodiscard]] double getDuration() const; + + [[nodiscard]] int getNumberOfChannels() const; [[nodiscard]] float *getChannelData(int channel) const; void copyFromChannel( @@ -34,9 +37,7 @@ class AudioBuffer : public std::enable_shared_from_this { int length_; int sampleRate_; double duration_; - float **channels_; - - std::shared_ptr mix(int outputNumberOfChannels); + std::unique_ptr bus_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp new file mode 100644 index 00000000..795e1a1c --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -0,0 +1,316 @@ + +#include "AudioBus.h" +#include "Constants.h" +#include "AudioArray.h" +#include "VectorMath.h" +#include "BaseAudioContext.h" + +// Implementation of channel summing/mixing is based on the WebKit approach, source: +// https://github.com/WebKit/WebKit/blob/main/Source/WebCore/platform/audio/AudioBus.cpp +const float SQRT_HALF = sqrtf(0.5f); + +namespace audioapi { + +AudioBus::AudioBus(BaseAudioContext *context) + : context_(context), numberOfChannels_(CHANNEL_COUNT) { + sampleRate_ = context_->getSampleRate(); + size_ = context_->getBufferSizeInFrames(); + + + createChannels(); +}; + +AudioBus::AudioBus(BaseAudioContext *context, int numberOfChannels) + : context_(context), numberOfChannels_(numberOfChannels) { + sampleRate_ = context_->getSampleRate(); + size_ = context_->getBufferSizeInFrames(); + + createChannels(); +}; + +AudioBus::AudioBus(BaseAudioContext *context, int numberOfChannels, int size) + : context_(context), numberOfChannels_(numberOfChannels), size_(size) { + sampleRate_ = context_->getSampleRate(); + + createChannels(); +}; + + +AudioBus::~AudioBus() { + channels_.clear(); +}; + +void AudioBus::createChannels() { + channels_ = std::vector>(numberOfChannels_, std::make_unique(size_)); +}; + +int AudioBus::getNumberOfChannels() const { + return numberOfChannels_; +}; + +int AudioBus::getSampleRate() const { + return sampleRate_; +}; + +int AudioBus::getSize() const { + return size_; +}; + +AudioArray* AudioBus::getChannel(int index) const { + return channels_[index].get(); +}; + +AudioArray* AudioBus::getChannelByType(int channelType) const { + switch (getNumberOfChannels()) { + case 1: // mono + if (channelType == ChannelMono || channelType == ChannelLeft) { + return getChannel(0); + } + return 0; + + case 2: // stereo + switch (channelType) { + case ChannelLeft: return getChannel(0); + case ChannelRight: return getChannel(1); + default: return 0; + } + + case 4: // quad + switch (channelType) { + case ChannelLeft: return getChannel(0); + case ChannelRight: return getChannel(1); + case ChannelSurroundLeft: return getChannel(2); + case ChannelSurroundRight: return getChannel(3); + default: return 0; + } + + case 5: // 5.0 + switch (channelType) { + case ChannelLeft: return getChannel(0); + case ChannelRight: return getChannel(1); + case ChannelCenter: return getChannel(2); + case ChannelSurroundLeft: return getChannel(3); + case ChannelSurroundRight: return getChannel(4); + default: return 0; + } + + case 6: // 5.1 + switch (channelType) { + case ChannelLeft: return getChannel(0); + case ChannelRight: return getChannel(1); + case ChannelCenter: return getChannel(2); + case ChannelLFE: return getChannel(3); + case ChannelSurroundLeft: return getChannel(4); + case ChannelSurroundRight: return getChannel(5); + default: return 0; + } + default: + return 0; + } +}; + +void AudioBus::zero() { + for (auto it = channels_.begin(); it != channels_.end(); it += 1) { + it->get()->zero(); + } +}; + +void AudioBus::normalize() { + float maxAbsValue = this->maxAbsValue(); + + if (maxAbsValue == 0.0f || maxAbsValue == 1.0f) { + return; + } + + float scale = 1.0f / maxAbsValue; + this->scale(scale); +}; + +void AudioBus::scale(float value) { + for (auto it = channels_.begin(); it != channels_.end(); it += 1) { + it->get()->scale(value); + } +}; + +float AudioBus::maxAbsValue() const { + float maxAbsValue = 0.0f; + + for (auto it = channels_.begin(); it != channels_.end(); it += 1) { + float channelMaxAbsValue = it->get()->getMaxAbsValue(); + maxAbsValue = std::max(maxAbsValue, channelMaxAbsValue); + } + + return maxAbsValue; +}; + +void AudioBus::copy(const AudioBus &source) { + if (&source == this) { + return; + } + + zero(); + sum(source); +}; + +void AudioBus::sum(const AudioBus &source) { + if (&source == this) { + return; + } + + int numberOfSourceChannels = source.getNumberOfChannels(); + int numberOfChannels = getNumberOfChannels(); + + // TODO: consider adding ability to enforce discrete summing (if/when it will be useful). + if (numberOfSourceChannels < numberOfChannels) { + sumByDownMixing(source); + return; + } + + if (numberOfSourceChannels > numberOfChannels) { + sumByUpMixing(source); + return; + } + + for (int i = 0; i < numberOfChannels_; i++) { + channels_[i]->sum(source.getChannel(i)); + } +}; + +void AudioBus::discreteSum(const AudioBus &source) { + int numberOfChannels = std::min(getNumberOfChannels(), source.getNumberOfChannels()); + + // In case of source > destination, we "down-mix" and drop the extra channels. + // In case of source < destination, we "up-mix" as many channels as we have, leaving the remaining channels untouched. + for (int i = 0; i < numberOfChannels; i++) { + getChannel(i)->sum(source.getChannel(i)); + } +}; + +void AudioBus::sumByUpMixing(const AudioBus &source) { + int numberOfSourceChannels = source.getNumberOfChannels(); + int numberOfChannels = getNumberOfChannels(); + + // Mono to stereo (1 -> 2, 4) + if (numberOfSourceChannels == 1 && (numberOfChannels == 2 || numberOfChannels == 4)) { + AudioArray* sourceChannel = source.getChannelByType(ChannelMono); + + getChannelByType(ChannelLeft)->sum(sourceChannel); + getChannelByType(ChannelRight)->sum(sourceChannel); + return; + } + + // Mono to 5.1 (1 -> 6) + if (numberOfSourceChannels == 1 && numberOfChannels == 6) { + AudioArray* sourceChannel = source.getChannel(0); + + getChannelByType(ChannelCenter)->sum(sourceChannel); + return; + } + + // Stereo 2 to stereo 4 or 5.1 (2 -> 4, 6) + if (numberOfSourceChannels == 2 && (numberOfChannels == 4 || numberOfChannels == 6)) { + + getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft)); + getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight)); + return; + } + + // Stereo 4 to 5.1 (4 -> 6) + if (numberOfSourceChannels == 4 && numberOfChannels == 6) { + getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft)); + getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight)); + getChannelByType(ChannelSurroundLeft)->sum(source.getChannelByType(ChannelSurroundLeft)); + getChannelByType(ChannelSurroundRight)->sum(source.getChannelByType(ChannelSurroundRight)); + return; + } + + discreteSum(source); +}; + +void AudioBus::sumByDownMixing(const AudioBus &source) { + int numberOfSourceChannels = source.getNumberOfChannels(); + int numberOfChannels = getNumberOfChannels(); + + // Stereo to mono (2 -> 1): output += 0.5 * (input.left + input.right). + if (numberOfSourceChannels == 2 && numberOfChannels == 1) { + float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source.getChannelByType(ChannelRight)->getData(); + + float* destinationData = getChannelByType(ChannelMono)->getData(); + + VectorMath::multiplyByScalarThenAddToOutput(sourceLeft, 0.5f, destinationData, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceRight, 0.5f, destinationData, getSize()); + return; + } + + // Stereo 4 to mono: output += 0.25 * (input.left + input.right + input.surroundLeft + input.surroundRight) + if (numberOfSourceChannels == 4 && numberOfChannels == 1) { + float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source.getChannelByType(ChannelRight)->getData(); + float* sourceSurroundLeft = source.getChannelByType(ChannelSurroundLeft)->getData(); + float* sourceSurroundRight = source.getChannelByType(ChannelSurroundRight)->getData(); + + float* destinationData = getChannelByType(ChannelMono)->getData(); + + VectorMath::multiplyByScalarThenAddToOutput(sourceLeft, 0.25f, destinationData, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceRight, 0.25f, destinationData, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundLeft, 0.25f, destinationData, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundRight, 0.25f, destinationData, getSize()); + return; + } + + // 5.1 to stereo: + // output.left += input.left + sqrt(1/2) * (input.center + input.surroundLeft) + // output.right += input.right + sqrt(1/2) * (input.center + input.surroundRight) + if (numberOfSourceChannels == 6 && numberOfChannels == 2) { + float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source.getChannelByType(ChannelRight)->getData(); + float* sourceCenter = source.getChannelByType(ChannelCenter)->getData(); + float* sourceSurroundLeft = source.getChannelByType(ChannelSurroundLeft)->getData(); + float* sourceSurroundRight = source.getChannelByType(ChannelSurroundRight)->getData(); + + float* destinationLeft = getChannelByType(ChannelLeft)->getData(); + float* destinationRight = getChannelByType(ChannelRight)->getData(); + + VectorMath::add(sourceLeft, destinationLeft, destinationLeft, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationLeft, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundLeft, SQRT_HALF, destinationLeft, getSize()); + + VectorMath::add(sourceRight, destinationRight, destinationRight, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationRight, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundRight, SQRT_HALF, destinationRight, getSize()); + return; + } + + // 5.1 to stereo 4: + // output.left += input.left + sqrt(1/2) * input.center + // output.right += input.right + sqrt(1/2) * input.center + // output.surroundLeft += input.surroundLeft + // output.surroundRight += input.surroundRight + if (numberOfSourceChannels == 6 && numberOfChannels == 4) { + float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source.getChannelByType(ChannelRight)->getData(); + float* sourceCenter = source.getChannelByType(ChannelCenter)->getData(); + float* sourceSurroundLeft = source.getChannelByType(ChannelSurroundLeft)->getData(); + float* sourceSurroundRight = source.getChannelByType(ChannelSurroundRight)->getData(); + + float* destinationLeft = getChannelByType(ChannelLeft)->getData(); + float* destinationRight = getChannelByType(ChannelRight)->getData(); + float* destinationSurroundLeft = getChannelByType(ChannelSurroundLeft)->getData(); + float* destinationSurroundRight = getChannelByType(ChannelSurroundRight)->getData(); + + VectorMath::add(sourceLeft, destinationLeft, destinationLeft, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationLeft, getSize()); + + VectorMath::add(sourceRight, destinationRight, destinationRight, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationRight, getSize()); + + VectorMath::add(sourceSurroundLeft, destinationSurroundLeft, destinationSurroundLeft, getSize()); + VectorMath::add(sourceSurroundRight, destinationSurroundRight, destinationSurroundRight, getSize()); + return; + } + + discreteSum(source); +}; + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.h b/packages/react-native-audio-api/common/cpp/core/AudioBus.h new file mode 100644 index 00000000..1c74b179 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include + +namespace audioapi { + +class BaseAudioContext; +class AudioArray; + +class AudioBus { + public: + enum { + ChannelMono = 0, + ChannelLeft = 0, + ChannelRight = 1, + ChannelCenter = 2, + ChannelLFE = 3, + ChannelSurroundLeft = 4, + ChannelSurroundRight = 5, + }; + + explicit AudioBus(BaseAudioContext *context); + explicit AudioBus(BaseAudioContext *context, int numberOfChannels); + explicit AudioBus(BaseAudioContext *context, int numberOfChannels, int size); + ~AudioBus(); + + [[nodiscard]] int getNumberOfChannels() const; + [[nodiscard]] int getSampleRate() const; + [[nodiscard]] int getSize() const; + AudioArray* getChannel(int index) const; + AudioArray* getChannelByType(int channelType) const; + + void zero(); + void normalize(); + void scale(float value); + float maxAbsValue() const; + + void copy(const AudioBus &source); + void sum(const AudioBus &source); + + private: + BaseAudioContext *context_; + std::vector> channels_; + + + int numberOfChannels_; + int sampleRate_; + int size_; + + void createChannels(); + void discreteSum(const AudioBus &source); + void sumByUpMixing(const AudioBus &source); + void sumByDownMixing(const AudioBus &source); +}; + +} diff --git a/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp b/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp deleted file mode 100644 index a0880f53..00000000 --- a/packages/react-native-audio-api/common/cpp/core/AudioChannel.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "AudioChannel.h" -#include "AudioArray.h" - -namespace audioapi { - -explicit AudioChannel::AudioChannel(int length, int sampleRate) : length_(length), sampleRate_(sampleRate) { - data_ = std::make_unique(length_); -}; - -int AudioChannel::getLength() const { - return length_; -}; - -int AudioChannel::getSampleRate() const { - return sampleRate_; -}; - -} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioChannel.h b/packages/react-native-audio-api/common/cpp/core/AudioChannel.h deleted file mode 100644 index 78054fc5..00000000 --- a/packages/react-native-audio-api/common/cpp/core/AudioChannel.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include -#include "AudioArray.h" - -namespace audioapi { - -class AudioChannel { - public: - explicit AudioChannel(int length, int sampleRate); - - [[nodiscard]] int getLength() const; - [[nodiscard]] int getSampleRate() const; - - void zero(); - void copy(const AudioChannel &source); - - float& operator[](int index); - const float& operator[](int index) const; - - - float getMaxAbsValue() const; - - void normalize(); - void scale(float scale); - void sum(const AudioChannel &source); - - - private: - - int length_; - int sampleRate_; - - std::unique_ptr data_; -}; - -} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index 619080ef..d96210cb 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -4,13 +4,13 @@ #include #include #include "Constants.h" -#include "AudioNodeBuffer.h" // channelCount always equal to 2 namespace audioapi { class BaseAudioContext; +class AudioBus; class AudioNode : public std::enable_shared_from_this { public: @@ -58,7 +58,7 @@ class AudioNode : public std::enable_shared_from_this { protected: BaseAudioContext *context_; - std::unique_ptr buffer_; + std::unique_ptr buffer_; int numberOfInputs_ = 1; int numberOfOutputs_ = 1; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h deleted file mode 100644 index 831ccd81..00000000 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeBuffer.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -// namespace audioapi { - -// class AudioNodeBuffer { -// public: -// explicit AudioNodeBuffer(BaseAudioContext * context, int channelCount); - -// void zero(); -// void normalize(); -// void scale(float scale); - -// void sumFrom(const AudioNodeBuffer &source); -// void copyFrom(const AudioNodeBuffer &source); - -// float maxAbsValue() const; - -// private: -// BaseAudioContext *context_; -// std::unique_ptr buffer_; - -// int bufferSizeInFrames_; -// int channelCount_; -// }; - -// } diff --git a/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp b/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp index 8eeb2ea1..9e2692e4 100644 --- a/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp +++ b/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp @@ -115,6 +115,11 @@ float maximumMagnitude( return maximumValue; } +void multiplyByScalarThenAddToOutput(const float* inputVector, float scalar, float* outputVector, size_t numberOfElementsToProcess) +{ + vDSP_vsma(inputVector, 1, &scalar, outputVector, 1, outputVector, 1, numberOfElementsToProcess); +} + #else #if defined(HAVE_X86_SSE2) @@ -578,7 +583,7 @@ float maximumMagnitude( max = std::max(max, groupMaxP[3]); n = tailFrames; -#elif defined(HAVE_ARM_NEON_INTRINSICS) +#elif defined(c) size_t tailFrames = n % 4; const float *endP = inputVector + n - tailFrames; @@ -605,5 +610,71 @@ float maximumMagnitude( return max; } +void multiplyByScalarThenAddToOutput(const float* inputVector, float scalar, float* outputVector, size_t numberOfElementsToProcess) +{ + size_t n = numberOfElementsToProcess; + +#if HAVE_X86_SSE2 + // If the inputVector address is not 16-byte aligned, the first several frames (at most three) should be processed separately. + while (!is16ByteAligned(inputVector) && n) { + *outputVector += scalar * *inputVector; + inputVector++; + outputVector++; + n--; + } + + // Now the inputVector is aligned, use SSE. + size_t tailFrames = n % 4; + const float* endP = outputVector + n - tailFrames; + + __m128 pSource; + __m128 dest; + __m128 temp; + __m128 mScale = _mm_set_ps1(scalar); + + bool destAligned = is16ByteAligned(outputVector); + +#define SSE2_MULT_ADD(loadInstr, storeInstr) \ + while (outputVector < endP) \ + { \ + pSource = _mm_load_ps(inputVector); \ + temp = _mm_mul_ps(pSource, mScale); \ + dest = _mm_##loadInstr##_ps(outputVector); \ + dest = _mm_add_ps(dest, temp); \ + _mm_##storeInstr##_ps(outputVector, dest); \ + inputVector += 4; \ + outputVector += 4; \ + } + + if (destAligned) + SSE2_MULT_ADD(load, store) + else + SSE2_MULT_ADD(loadu, storeu) + + n = tailFrames; +#elif HAVE_ARM_NEON_INTRINSICS + size_t tailFrames = n % 4; + const float* endP = outputVector + n - tailFrames; + + float32x4_t k = vdupq_n_f32(scalar); + while (outputVector < endP) { + float32x4_t source = vld1q_f32(inputVector); + float32x4_t dest = vld1q_f32(outputVector); + + dest = vmlaq_f32(dest, source, k); + vst1q_f32(outputVector, dest); + + inputVector += 4; + outputVector += 4; + } + n = tailFrames; +#endif + while (n--) { + *outputVector += *inputVector * scalar; + ++inputVector; + ++outputVector; + } +} + #endif } // namespace audioapi::VectorMath diff --git a/packages/react-native-audio-api/common/cpp/utils/VectorMath.h b/packages/react-native-audio-api/common/cpp/utils/VectorMath.h index 5a6464a5..c29f58f8 100644 --- a/packages/react-native-audio-api/common/cpp/utils/VectorMath.h +++ b/packages/react-native-audio-api/common/cpp/utils/VectorMath.h @@ -32,6 +32,12 @@ namespace audioapi::VectorMath { +void multiplyByScalarThenAddToOutput( + const float *inputVector, + float scalar, + float *outputVector, + size_t numberOfElementsToProcess); + void multiplyByScalar( const float *inputVector, float scalar, From 1e029f52f12da21422a41fd1d41d99367a332309 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 15 Nov 2024 11:35:21 +0100 Subject: [PATCH 04/33] feat: add some comments --- .../common/cpp/core/AudioBus.cpp | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index 795e1a1c..49631852 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -7,10 +7,14 @@ // Implementation of channel summing/mixing is based on the WebKit approach, source: // https://github.com/WebKit/WebKit/blob/main/Source/WebCore/platform/audio/AudioBus.cpp + const float SQRT_HALF = sqrtf(0.5f); namespace audioapi { +/** + * Public interfaces - memory management + */ AudioBus::AudioBus(BaseAudioContext *context) : context_(context), numberOfChannels_(CHANNEL_COUNT) { sampleRate_ = context_->getSampleRate(); @@ -40,9 +44,9 @@ AudioBus::~AudioBus() { channels_.clear(); }; -void AudioBus::createChannels() { - channels_ = std::vector>(numberOfChannels_, std::make_unique(size_)); -}; +/** + * Public interfaces - getters + */ int AudioBus::getNumberOfChannels() const { return numberOfChannels_; @@ -109,6 +113,10 @@ AudioArray* AudioBus::getChannelByType(int channelType) const { } }; +/** + * Public interfaces - audio processing and setters + */ + void AudioBus::zero() { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { it->get()->zero(); @@ -148,6 +156,7 @@ void AudioBus::copy(const AudioBus &source) { return; } + // zero + sum is equivalent to copy, but takes care of up/down-mixing. zero(); sum(source); }; @@ -161,21 +170,37 @@ void AudioBus::sum(const AudioBus &source) { int numberOfChannels = getNumberOfChannels(); // TODO: consider adding ability to enforce discrete summing (if/when it will be useful). + // Source channel count is smaller than current bus, we need to up-mix. if (numberOfSourceChannels < numberOfChannels) { - sumByDownMixing(source); + sumByUpMixing(source); return; } + // Source channel count is larger than current bus, we need to down-mix. if (numberOfSourceChannels > numberOfChannels) { - sumByUpMixing(source); + sumByDownMixing(source); return; } + // Source and destination channel counts are the same. Just sum the channels. for (int i = 0; i < numberOfChannels_; i++) { channels_[i]->sum(source.getChannel(i)); } }; +/** + * Internal tooling - channel initialization + */ + +void AudioBus::createChannels() { + // TODO: verify if its correct way of memory allocation + channels_ = std::vector>(numberOfChannels_, std::make_unique(size_)); +}; + +/** + * Internal tooling - channel summing + */ + void AudioBus::discreteSum(const AudioBus &source) { int numberOfChannels = std::min(getNumberOfChannels(), source.getNumberOfChannels()); From d938b7738a636b6661d4a91f7d2b874cabe69b51 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 15 Nov 2024 11:38:29 +0100 Subject: [PATCH 05/33] fix: spellchecker howling --- .github/actions/spelling/allow.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 8b812f17..9fb9ad37 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -27,4 +27,5 @@ swmansion tada vec +rnaa From 9002e52fb505dfb708f33da9b4e0e914f2021490 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 15 Nov 2024 12:01:56 +0100 Subject: [PATCH 06/33] fix: linter howling --- .../common/cpp/core/AudioArray.cpp | 12 +-- .../common/cpp/core/AudioArray.h | 36 +++---- .../common/cpp/core/AudioBuffer.cpp | 1 - .../common/cpp/core/AudioBuffer.h | 1 - .../common/cpp/core/AudioBus.cpp | 41 ++++--- .../common/cpp/core/AudioBus.h | 88 +++++++-------- .../common/cpp/core/AudioNode.cpp | 68 ++++++++---- .../common/cpp/core/AudioNode.h | 100 +++++++----------- .../common/cpp/utils/VectorMath.cpp | 6 +- 9 files changed, 175 insertions(+), 178 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index c137db83..d149e7d9 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -5,11 +5,11 @@ namespace audioapi { AudioArray::AudioArray(int size) : size_(size) { resize(size); -}; +} AudioArray::~AudioArray() { delete[] data_; -}; +} int AudioArray::getSize() const { return size_; @@ -54,7 +54,7 @@ void AudioArray::copy(const AudioArray* source) { float AudioArray::getMaxAbsValue() const { return VectorMath::maximumMagnitude(data_, size_); -}; +} void AudioArray::normalize() { float maxAbsValue = getMaxAbsValue(); @@ -64,15 +64,15 @@ void AudioArray::normalize() { } VectorMath::multiplyByScalar(data_, 1.0f / maxAbsValue, data_, size_); -}; +} void AudioArray::scale(float value) { VectorMath::multiplyByScalar(data_, value, data_, size_); -}; +} void AudioArray::sum(const AudioArray* source) { VectorMath::add(data_, source->getData(), data_, size_); -}; +} } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.h b/packages/react-native-audio-api/common/cpp/core/AudioArray.h index 0b79060a..f61636db 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.h @@ -5,31 +5,31 @@ namespace audioapi { class AudioArray { - public: - explicit AudioArray(int size); - ~AudioArray(); + public: + explicit AudioArray(int size); + ~AudioArray(); - [[nodiscard]] int getSize() const; - float* getData() const; + [[nodiscard]] int getSize() const; + float* getData() const; - float& operator[](int index); - const float& operator[](int index) const; + float& operator[](int index); + const float& operator[](int index) const; - void zero(); - void resize(int size); - void copy(const AudioArray* source); + void zero(); + void resize(int size); + void copy(const AudioArray* source); - float getMaxAbsValue() const; + float getMaxAbsValue() const; - void normalize(); - void scale(float value); - void sum(const AudioArray* source); + void normalize(); + void scale(float value); + void sum(const AudioArray* source); - private: - float *data_; - int size_; + private: + float *data_; + int size_; }; -} +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp index 0a7a3e7f..bef4f1c9 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp @@ -7,7 +7,6 @@ AudioBuffer::AudioBuffer(int numberOfChannels, int length, int sampleRate) length_(length), sampleRate_(sampleRate), duration_(static_cast(length) / sampleRate) { - } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h index 642519ec..98df0771 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h @@ -32,7 +32,6 @@ class AudioBuffer : public std::enable_shared_from_this { int startInChannel); private: - int numberOfChannels_; int length_; int sampleRate_; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index 49631852..fba622ef 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -1,3 +1,4 @@ +#include #include "AudioBus.h" #include "Constants.h" @@ -22,7 +23,7 @@ AudioBus::AudioBus(BaseAudioContext *context) createChannels(); -}; +} AudioBus::AudioBus(BaseAudioContext *context, int numberOfChannels) : context_(context), numberOfChannels_(numberOfChannels) { @@ -30,19 +31,18 @@ AudioBus::AudioBus(BaseAudioContext *context, int numberOfChannels) size_ = context_->getBufferSizeInFrames(); createChannels(); -}; +} AudioBus::AudioBus(BaseAudioContext *context, int numberOfChannels, int size) : context_(context), numberOfChannels_(numberOfChannels), size_(size) { sampleRate_ = context_->getSampleRate(); createChannels(); -}; - +} AudioBus::~AudioBus() { channels_.clear(); -}; +} /** * Public interfaces - getters @@ -50,19 +50,19 @@ AudioBus::~AudioBus() { int AudioBus::getNumberOfChannels() const { return numberOfChannels_; -}; +} int AudioBus::getSampleRate() const { return sampleRate_; -}; +} int AudioBus::getSize() const { return size_; -}; +} AudioArray* AudioBus::getChannel(int index) const { return channels_[index].get(); -}; +} AudioArray* AudioBus::getChannelByType(int channelType) const { switch (getNumberOfChannels()) { @@ -111,7 +111,7 @@ AudioArray* AudioBus::getChannelByType(int channelType) const { default: return 0; } -}; +} /** * Public interfaces - audio processing and setters @@ -121,7 +121,7 @@ void AudioBus::zero() { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { it->get()->zero(); } -}; +} void AudioBus::normalize() { float maxAbsValue = this->maxAbsValue(); @@ -132,13 +132,13 @@ void AudioBus::normalize() { float scale = 1.0f / maxAbsValue; this->scale(scale); -}; +} void AudioBus::scale(float value) { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { it->get()->scale(value); } -}; +} float AudioBus::maxAbsValue() const { float maxAbsValue = 0.0f; @@ -149,7 +149,7 @@ float AudioBus::maxAbsValue() const { } return maxAbsValue; -}; +} void AudioBus::copy(const AudioBus &source) { if (&source == this) { @@ -159,7 +159,7 @@ void AudioBus::copy(const AudioBus &source) { // zero + sum is equivalent to copy, but takes care of up/down-mixing. zero(); sum(source); -}; +} void AudioBus::sum(const AudioBus &source) { if (&source == this) { @@ -186,7 +186,7 @@ void AudioBus::sum(const AudioBus &source) { for (int i = 0; i < numberOfChannels_; i++) { channels_[i]->sum(source.getChannel(i)); } -}; +} /** * Internal tooling - channel initialization @@ -195,7 +195,7 @@ void AudioBus::sum(const AudioBus &source) { void AudioBus::createChannels() { // TODO: verify if its correct way of memory allocation channels_ = std::vector>(numberOfChannels_, std::make_unique(size_)); -}; +} /** * Internal tooling - channel summing @@ -209,7 +209,7 @@ void AudioBus::discreteSum(const AudioBus &source) { for (int i = 0; i < numberOfChannels; i++) { getChannel(i)->sum(source.getChannel(i)); } -}; +} void AudioBus::sumByUpMixing(const AudioBus &source) { int numberOfSourceChannels = source.getNumberOfChannels(); @@ -234,7 +234,6 @@ void AudioBus::sumByUpMixing(const AudioBus &source) { // Stereo 2 to stereo 4 or 5.1 (2 -> 4, 6) if (numberOfSourceChannels == 2 && (numberOfChannels == 4 || numberOfChannels == 6)) { - getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft)); getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight)); return; @@ -250,7 +249,7 @@ void AudioBus::sumByUpMixing(const AudioBus &source) { } discreteSum(source); -}; +} void AudioBus::sumByDownMixing(const AudioBus &source) { int numberOfSourceChannels = source.getNumberOfChannels(); @@ -336,6 +335,6 @@ void AudioBus::sumByDownMixing(const AudioBus &source) { } discreteSum(source); -}; +} } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.h b/packages/react-native-audio-api/common/cpp/core/AudioBus.h index 1c74b179..62981a5c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.h @@ -9,49 +9,49 @@ class BaseAudioContext; class AudioArray; class AudioBus { - public: - enum { - ChannelMono = 0, - ChannelLeft = 0, - ChannelRight = 1, - ChannelCenter = 2, - ChannelLFE = 3, - ChannelSurroundLeft = 4, - ChannelSurroundRight = 5, - }; - - explicit AudioBus(BaseAudioContext *context); - explicit AudioBus(BaseAudioContext *context, int numberOfChannels); - explicit AudioBus(BaseAudioContext *context, int numberOfChannels, int size); - ~AudioBus(); - - [[nodiscard]] int getNumberOfChannels() const; - [[nodiscard]] int getSampleRate() const; - [[nodiscard]] int getSize() const; - AudioArray* getChannel(int index) const; - AudioArray* getChannelByType(int channelType) const; - - void zero(); - void normalize(); - void scale(float value); - float maxAbsValue() const; - - void copy(const AudioBus &source); - void sum(const AudioBus &source); - - private: - BaseAudioContext *context_; - std::vector> channels_; - - - int numberOfChannels_; - int sampleRate_; - int size_; - - void createChannels(); - void discreteSum(const AudioBus &source); - void sumByUpMixing(const AudioBus &source); - void sumByDownMixing(const AudioBus &source); + public: + enum { + ChannelMono = 0, + ChannelLeft = 0, + ChannelRight = 1, + ChannelCenter = 2, + ChannelLFE = 3, + ChannelSurroundLeft = 4, + ChannelSurroundRight = 5, + }; + + explicit AudioBus(BaseAudioContext *context); + explicit AudioBus(BaseAudioContext *context, int numberOfChannels); + explicit AudioBus(BaseAudioContext *context, int numberOfChannels, int size); + ~AudioBus(); + + [[nodiscard]] int getNumberOfChannels() const; + [[nodiscard]] int getSampleRate() const; + [[nodiscard]] int getSize() const; + AudioArray* getChannel(int index) const; + AudioArray* getChannelByType(int channelType) const; + + void zero(); + void normalize(); + void scale(float value); + float maxAbsValue() const; + + void copy(const AudioBus &source); + void sum(const AudioBus &source); + + private: + BaseAudioContext *context_; + std::vector> channels_; + + + int numberOfChannels_; + int sampleRate_; + int size_; + + void createChannels(); + void discreteSum(const AudioBus &source); + void sumByUpMixing(const AudioBus &source); + void sumByDownMixing(const AudioBus &source); }; -} +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 6ee29a77..27d2b15c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -1,10 +1,11 @@ +#include "AudioBus.h" #include "AudioNode.h" #include "BaseAudioContext.h" namespace audioapi { AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { - buffer_ = std::make_unique(context_, channelCount_); + audioBus_ = std::make_unique(context_, channelCount_); } AudioNode::~AudioNode() { @@ -25,46 +26,73 @@ int AudioNode::getChannelCount() const { std::string AudioNode::getChannelCountMode() const { return AudioNode::toString(channelCountMode_); -} +}; std::string AudioNode::getChannelInterpretation() const { return AudioNode::toString(channelInterpretation_); -} +}; void AudioNode::connect(const std::shared_ptr &node) { - if (numberOfOutputs_ > outputNodes_.size() && - node->getNumberOfInputs() > node->inputNodes_.size()) { - outputNodes_.push_back(node); - node->inputNodes_.push_back(shared_from_this()); - } -} + outputNodes_.push_back(node); + node->inputNodes_.push_back(shared_from_this()); +}; void AudioNode::disconnect(const std::shared_ptr &node) { outputNodes_.erase( std::remove(outputNodes_.begin(), outputNodes_.end(), node), outputNodes_.end()); + if (auto sharedThis = shared_from_this()) { node->inputNodes_.erase( std::remove( node->inputNodes_.begin(), node->inputNodes_.end(), sharedThis), node->inputNodes_.end()); } +}; + + +std::string AudioNode::toString(ChannelCountMode mode) { + switch (mode) { + case ChannelCountMode::MAX: + return "max"; + case ChannelCountMode::CLAMPED_MAX: + return "clamped-max"; + case ChannelCountMode::EXPLICIT: + return "explicit"; + default: + throw std::invalid_argument("Unknown channel count mode"); + } } +std::string AudioNode::toString(ChannelInterpretation interpretation) { + switch (interpretation) { + case ChannelInterpretation::SPEAKERS: + return "speakers"; + case ChannelInterpretation::DISCRETE: + return "discrete"; + default: + throw std::invalid_argument("Unknown channel interpretation"); + } +} + +bool AudioNode::processAudio(AudioBus *outputBus, int framesToProcess) { + // bool isPlaying = false; + // for (auto &node : inputNodes_) { + // if (node->processAudio(buffer_.get(), framesToProcess)) { + // isPlaying = true; + // } + // } + + // if (isPlaying) { + // outputBus->copyFrom(*buffer_.get(), 0, 0, framesToProcess); + // } + + // return isPlaying; +}; + void AudioNode::cleanup() { outputNodes_.clear(); inputNodes_.clear(); } -// bool AudioNode::processAudio(float *audioData, int32_t numFrames) { -// bool isPlaying = false; -// for (auto &node : inputNodes_) { -// if (node->processAudio(audioData, numFrames)) { -// isPlaying = true; -// } -// } - -// return isPlaying; -// } - } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index d96210cb..f75b309d 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -3,9 +3,6 @@ #include #include #include -#include "Constants.h" - -// channelCount always equal to 2 namespace audioapi { @@ -13,66 +10,43 @@ class BaseAudioContext; class AudioBus; class AudioNode : public std::enable_shared_from_this { - public: - explicit AudioNode(BaseAudioContext *context); - virtual ~AudioNode(); - int getNumberOfInputs() const; - int getNumberOfOutputs() const; - int getChannelCount() const; - std::string getChannelCountMode() const; - std::string getChannelInterpretation() const; - void connect(const std::shared_ptr &node); - void disconnect(const std::shared_ptr &node); - - // Change public to protected - // virtual bool processAudio(float *audioData, int32_t numFrames); - - protected: - enum class ChannelCountMode { MAX, CLAMPED_MAX, EXPLICIT }; - - static std::string toString(ChannelCountMode mode) { - switch (mode) { - case ChannelCountMode::MAX: - return "max"; - case ChannelCountMode::CLAMPED_MAX: - return "clamped-max"; - case ChannelCountMode::EXPLICIT: - return "explicit"; - default: - throw std::invalid_argument("Unknown channel count mode"); - } - } - - enum class ChannelInterpretation { SPEAKERS, DISCRETE }; - - static std::string toString(ChannelInterpretation interpretation) { - switch (interpretation) { - case ChannelInterpretation::SPEAKERS: - return "speakers"; - case ChannelInterpretation::DISCRETE: - return "discrete"; - default: - throw std::invalid_argument("Unknown channel interpretation"); - } - } - - protected: - BaseAudioContext *context_; - std::unique_ptr buffer_; - - int numberOfInputs_ = 1; - int numberOfOutputs_ = 1; - int channelCount_ = CHANNEL_COUNT; - - ChannelCountMode channelCountMode_ = ChannelCountMode::MAX; - ChannelInterpretation channelInterpretation_ = - ChannelInterpretation::SPEAKERS; - - std::vector> inputNodes_ = {}; - std::vector> outputNodes_ = {}; - - private: - void cleanup(); + public: + explicit AudioNode(BaseAudioContext *context); + virtual ~AudioNode(); + int getNumberOfInputs() const; + int getNumberOfOutputs() const; + int getChannelCount() const; + std::string getChannelCountMode() const; + std::string getChannelInterpretation() const; + void connect(const std::shared_ptr &node); + void disconnect(const std::shared_ptr &node); + + protected: + enum class ChannelCountMode { MAX, CLAMPED_MAX, EXPLICIT }; + enum class ChannelInterpretation { SPEAKERS, DISCRETE }; + + static std::string toString(ChannelCountMode mode); + static std::string toString(ChannelInterpretation interpretation); + + protected: + BaseAudioContext *context_; + std::unique_ptr audioBus_; + + int numberOfInputs_ = 1; + int numberOfOutputs_ = 1; + int channelCount_ = CHANNEL_COUNT; + + ChannelCountMode channelCountMode_ = ChannelCountMode::MAX; + ChannelInterpretation channelInterpretation_ = + ChannelInterpretation::SPEAKERS; + + std::vector> inputNodes_ = {}; + std::vector> outputNodes_ = {}; + + bool processAudio(AudioBus *outputBus, int framesToProcess); + + private: + void cleanup(); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp b/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp index 9e2692e4..4195b10a 100644 --- a/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp +++ b/packages/react-native-audio-api/common/cpp/utils/VectorMath.cpp @@ -115,8 +115,7 @@ float maximumMagnitude( return maximumValue; } -void multiplyByScalarThenAddToOutput(const float* inputVector, float scalar, float* outputVector, size_t numberOfElementsToProcess) -{ +void multiplyByScalarThenAddToOutput(const float* inputVector, float scalar, float* outputVector, size_t numberOfElementsToProcess) { vDSP_vsma(inputVector, 1, &scalar, outputVector, 1, outputVector, 1, numberOfElementsToProcess); } @@ -610,8 +609,7 @@ float maximumMagnitude( return max; } -void multiplyByScalarThenAddToOutput(const float* inputVector, float scalar, float* outputVector, size_t numberOfElementsToProcess) -{ +void multiplyByScalarThenAddToOutput(const float* inputVector, float scalar, float* outputVector, size_t numberOfElementsToProcess) { size_t n = numberOfElementsToProcess; #if HAVE_X86_SSE2 From e50355fe5f2d058a838c11cf16ce1fa0f31d89e5 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 15 Nov 2024 12:06:21 +0100 Subject: [PATCH 07/33] fix: linter howling part 1.27 --- .../common/cpp/core/AudioNode.cpp | 10 +-- .../common/cpp/core/AudioNode.h | 74 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 27d2b15c..8db8b005 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -26,16 +26,16 @@ int AudioNode::getChannelCount() const { std::string AudioNode::getChannelCountMode() const { return AudioNode::toString(channelCountMode_); -}; +} std::string AudioNode::getChannelInterpretation() const { return AudioNode::toString(channelInterpretation_); -}; +} void AudioNode::connect(const std::shared_ptr &node) { outputNodes_.push_back(node); node->inputNodes_.push_back(shared_from_this()); -}; +} void AudioNode::disconnect(const std::shared_ptr &node) { outputNodes_.erase( @@ -48,7 +48,7 @@ void AudioNode::disconnect(const std::shared_ptr &node) { node->inputNodes_.begin(), node->inputNodes_.end(), sharedThis), node->inputNodes_.end()); } -}; +} std::string AudioNode::toString(ChannelCountMode mode) { @@ -88,7 +88,7 @@ bool AudioNode::processAudio(AudioBus *outputBus, int framesToProcess) { // } // return isPlaying; -}; +} void AudioNode::cleanup() { outputNodes_.clear(); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index f75b309d..0d3de9fa 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -10,43 +10,43 @@ class BaseAudioContext; class AudioBus; class AudioNode : public std::enable_shared_from_this { - public: - explicit AudioNode(BaseAudioContext *context); - virtual ~AudioNode(); - int getNumberOfInputs() const; - int getNumberOfOutputs() const; - int getChannelCount() const; - std::string getChannelCountMode() const; - std::string getChannelInterpretation() const; - void connect(const std::shared_ptr &node); - void disconnect(const std::shared_ptr &node); - - protected: - enum class ChannelCountMode { MAX, CLAMPED_MAX, EXPLICIT }; - enum class ChannelInterpretation { SPEAKERS, DISCRETE }; - - static std::string toString(ChannelCountMode mode); - static std::string toString(ChannelInterpretation interpretation); - - protected: - BaseAudioContext *context_; - std::unique_ptr audioBus_; - - int numberOfInputs_ = 1; - int numberOfOutputs_ = 1; - int channelCount_ = CHANNEL_COUNT; - - ChannelCountMode channelCountMode_ = ChannelCountMode::MAX; - ChannelInterpretation channelInterpretation_ = - ChannelInterpretation::SPEAKERS; - - std::vector> inputNodes_ = {}; - std::vector> outputNodes_ = {}; - - bool processAudio(AudioBus *outputBus, int framesToProcess); - - private: - void cleanup(); + public: + explicit AudioNode(BaseAudioContext *context); + virtual ~AudioNode(); + int getNumberOfInputs() const; + int getNumberOfOutputs() const; + int getChannelCount() const; + std::string getChannelCountMode() const; + std::string getChannelInterpretation() const; + void connect(const std::shared_ptr &node); + void disconnect(const std::shared_ptr &node); + + protected: + enum class ChannelCountMode { MAX, CLAMPED_MAX, EXPLICIT }; + enum class ChannelInterpretation { SPEAKERS, DISCRETE }; + + static std::string toString(ChannelCountMode mode); + static std::string toString(ChannelInterpretation interpretation); + + protected: + BaseAudioContext *context_; + std::unique_ptr audioBus_; + + int numberOfInputs_ = 1; + int numberOfOutputs_ = 1; + int channelCount_ = CHANNEL_COUNT; + + ChannelCountMode channelCountMode_ = ChannelCountMode::MAX; + ChannelInterpretation channelInterpretation_ = + ChannelInterpretation::SPEAKERS; + + std::vector> inputNodes_ = {}; + std::vector> outputNodes_ = {}; + + bool processAudio(AudioBus *outputBus, int framesToProcess); + + private: + void cleanup(); }; } // namespace audioapi From 4fc9044c96a1a3526fe4949bfbfb4cdac6be9eb9 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 15 Nov 2024 12:35:27 +0100 Subject: [PATCH 08/33] fix: politely obey to false-positive linter check --- packages/react-native-audio-api/common/cpp/core/AudioArray.cpp | 2 ++ packages/react-native-audio-api/common/cpp/core/AudioArray.h | 1 + packages/react-native-audio-api/common/cpp/core/AudioBus.h | 1 + 3 files changed, 4 insertions(+) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index d149e7d9..b8ee252c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -1,3 +1,5 @@ +#include + #include "AudioArray.h" #include "VectorMath.h" diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.h b/packages/react-native-audio-api/common/cpp/core/AudioArray.h index f61636db..edaa586d 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace audioapi { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.h b/packages/react-native-audio-api/common/cpp/core/AudioBus.h index 62981a5c..4a0f863c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.h @@ -2,6 +2,7 @@ #include #include +#include namespace audioapi { From 862f110fdde75d5e87e660eb94e2c9fd2fe88d2f Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 15 Nov 2024 18:03:27 +0100 Subject: [PATCH 09/33] feat: working on AudioPlayer implementation using AudioBus --- .../src/main/cpp/AudioPlayer/AudioPlayer.cpp | 16 ++++++- .../src/main/cpp/AudioPlayer/AudioPlayer.h | 6 ++- .../common/cpp/core/AudioBus.cpp | 22 ++------- .../common/cpp/core/AudioBus.h | 8 ++-- .../common/cpp/core/AudioDestinationNode.cpp | 40 ++++++++-------- .../common/cpp/core/AudioDestinationNode.h | 20 +++++--- .../common/cpp/core/AudioNode.cpp | 7 ++- .../common/cpp/core/AudioNode.h | 5 +- .../common/cpp/core/BaseAudioContext.cpp | 46 +++++++++++++------ .../common/cpp/core/BaseAudioContext.h | 45 ++++++------------ .../common/cpp/core/StereoPannerNode.cpp | 1 + .../ios/AudioPlayer/AudioPlayer.h | 3 +- .../ios/AudioPlayer/AudioPlayer.m | 20 +------- .../ios/AudioPlayer/IOSAudioPlayer.h | 9 ++-- .../ios/AudioPlayer/IOSAudioPlayer.mm | 21 +++++++-- 15 files changed, 136 insertions(+), 133 deletions(-) diff --git a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp index 9616737e..65554ad9 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp @@ -1,9 +1,11 @@ #include "AudioPlayer.h" + +#include "AudioBus.h" #include "AudioContext.h" namespace audioapi { -AudioPlayer::AudioPlayer(const std::function &renderAudio) +AudioPlayer::AudioPlayer(const std::function &renderAudio) : renderAudio_(renderAudio) { AudioStreamBuilder builder; @@ -15,6 +17,8 @@ AudioPlayer::AudioPlayer(const std::function &renderAudio) ->setSampleRateConversionQuality(SampleRateConversionQuality::Medium) ->setDataCallback(this) ->openStream(mStream_); + + mBus_ = std::make_unique(getSampleRate(), getBufferSizeInFrames(), CHANNEL_COUNT); } int AudioPlayer::getSampleRate() const { @@ -44,7 +48,15 @@ DataCallbackResult AudioPlayer::onAudioReady( void *audioData, int32_t numFrames) { auto buffer = static_cast(audioData); - renderAudio_(buffer, numFrames); + + renderAudio_(mBus_, numFrames); + + // TODO: optimize this with SIMD? + for (int32_t i = 0; i < numFrames; i += 1) { + for (int channel = 0; channel < CHANNEL_COUNT; channel += 1) { + buffer[i * CHANNEL_COUNT + channel] = mBus_->getChannel(channel)->get()[i]; + } + } return DataCallbackResult::Continue; } diff --git a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h index 41f4f860..c4aa89ef 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h +++ b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h @@ -8,10 +8,11 @@ namespace audioapi { using namespace oboe; class AudioContext; +class AudioBus; class AudioPlayer : public AudioStreamDataCallback { public: - explicit AudioPlayer(const std::function &renderAudio); + explicit AudioPlayer(const std::function &renderAudio); int getSampleRate() const; int getBufferSizeInFrames() const; @@ -24,8 +25,9 @@ class AudioPlayer : public AudioStreamDataCallback { int32_t numFrames) override; private: - std::function renderAudio_; + std::function renderAudio_; std::shared_ptr mStream_; + std::unique_ptr mBus_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index fba622ef..341803ab 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -16,27 +16,13 @@ namespace audioapi { /** * Public interfaces - memory management */ -AudioBus::AudioBus(BaseAudioContext *context) - : context_(context), numberOfChannels_(CHANNEL_COUNT) { - sampleRate_ = context_->getSampleRate(); - size_ = context_->getBufferSizeInFrames(); - - +AudioBus::AudioBus(int sampleRate, int size) + : sampleRate_(sampleRate), size_(size), numberOfChannels_(CHANNEL_COUNT) { createChannels(); } -AudioBus::AudioBus(BaseAudioContext *context, int numberOfChannels) - : context_(context), numberOfChannels_(numberOfChannels) { - sampleRate_ = context_->getSampleRate(); - size_ = context_->getBufferSizeInFrames(); - - createChannels(); -} - -AudioBus::AudioBus(BaseAudioContext *context, int numberOfChannels, int size) - : context_(context), numberOfChannels_(numberOfChannels), size_(size) { - sampleRate_ = context_->getSampleRate(); - +AudioBus::AudioBus(int sampleRate, int size, int numberOfChannels) + : sampleRate_(sampleRate), size_(size), numberOfChannels_(numberOfChannels) { createChannels(); } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.h b/packages/react-native-audio-api/common/cpp/core/AudioBus.h index 4a0f863c..1164eac8 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.h @@ -21,9 +21,9 @@ class AudioBus { ChannelSurroundRight = 5, }; - explicit AudioBus(BaseAudioContext *context); - explicit AudioBus(BaseAudioContext *context, int numberOfChannels); - explicit AudioBus(BaseAudioContext *context, int numberOfChannels, int size); + explicit AudioBus(int sampleRate, int size); + explicit AudioBus(int sampleRate, int size, int numberOfChannels); + ~AudioBus(); [[nodiscard]] int getNumberOfChannels() const; @@ -41,10 +41,8 @@ class AudioBus { void sum(const AudioBus &source); private: - BaseAudioContext *context_; std::vector> channels_; - int numberOfChannels_; int sampleRate_; int size_; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 8ab3d4e6..4cff221c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -5,25 +5,35 @@ namespace audioapi { AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) - : AudioNode(context) { + : AudioNode(context), currentSampleFrame_(0) { numberOfOutputs_ = 0; numberOfInputs_ = INT_MAX; channelCountMode_ = ChannelCountMode::EXPLICIT; } -void AudioDestinationNode::renderAudio(float *audioData, int32_t numFrames) { - // processAudio(audioData, numFrames); +unsigned AudioDestinationNode::getCurrentSampleFrame() const { + return currentSampleFrame_; } -// bool AudioDestinationNode::processAudio(float *audioData, int32_t numFrames) { -// int numSamples = numFrames * CHANNEL_COUNT; +double AudioDestinationNode::getCurrentTime() const { + return currentSampleFrame_ / context_->getSampleRate(); +} -// if (mixingBuffer == nullptr) { -// mixingBuffer = std::make_unique(numSamples); -// } +void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFrames) { + // int32_t numSamples = numFrames * CHANNEL_COUNT; + + // memset(destinationBuffer, 0.0f, sizeof(float) * numSamples); + + // if (!numFrames) { + // return; + // } + + // processAudio(numFrames); -// memset(audioData, 0.0f, sizeof(float) * numSamples); + // currentSampleFrame_ += numFrames; +} +// bool AudioDestinationNode::processAudio(float *audioData, int32_t numFrames) { // for (auto &node : inputNodes_) { // if (node && node->processAudio(mixingBuffer.get(), numFrames)) { // normalize(mixingBuffer.get(), numFrames); @@ -34,16 +44,4 @@ void AudioDestinationNode::renderAudio(float *audioData, int32_t numFrames) { // return true; // } -void AudioDestinationNode::normalize(float *audioData, int32_t numFrames) { - auto maxValue = std::max( - 1.0f, VectorMath::maximumMagnitude(audioData, numFrames * channelCount_)); - - if (maxValue == 1.0f) { - return; - } - - VectorMath::multiplyByScalar( - audioData, 1.0f / maxValue, audioData, numFrames * channelCount_); -} - } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h index 048dc3bd..4428a08a 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h @@ -1,25 +1,33 @@ #pragma once -#include -#include #include +#include +#include #include "AudioNode.h" namespace audioapi { +class AudioBus; +class BaseAudioContext; + class AudioDestinationNode : public AudioNode { public: explicit AudioDestinationNode(BaseAudioContext *context); - void renderAudio(float *audioData, int32_t numFrames); + void renderAudio(AudioBus* audioData, int32_t numFrames); + + unsigned getCurrentSampleFrame() const; + double getCurrentTime() const; protected: - // bool processAudio(float *audioData, int32_t numFrames) override; private: - std::unique_ptr mixingBuffer; + unsigned currentSampleFrame_; - void normalize(float *audioData, int32_t numFrames); + // DestinationNode is triggered by AudioContext using renderAudio + // processNode function is not necessary and is never called. + void processNode(int) final { }; }; + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 8db8b005..1ec0f816 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -5,7 +5,7 @@ namespace audioapi { AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { - audioBus_ = std::make_unique(context_, channelCount_); + audioBus_ = std::make_unique(context->getSampleRate(), context->getBufferSizeInFrames(), channelCount_); } AudioNode::~AudioNode() { @@ -75,7 +75,8 @@ std::string AudioNode::toString(ChannelInterpretation interpretation) { } } -bool AudioNode::processAudio(AudioBus *outputBus, int framesToProcess) { +void AudioNode::processAudio(int framesToProcess) { + // bool isPlaying = false; // for (auto &node : inputNodes_) { // if (node->processAudio(buffer_.get(), framesToProcess)) { @@ -86,8 +87,6 @@ bool AudioNode::processAudio(AudioBus *outputBus, int framesToProcess) { // if (isPlaying) { // outputBus->copyFrom(*buffer_.get(), 0, 0, framesToProcess); // } - - // return isPlaying; } void AudioNode::cleanup() { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index 0d3de9fa..dd175e03 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -28,7 +28,6 @@ class AudioNode : public std::enable_shared_from_this { static std::string toString(ChannelCountMode mode); static std::string toString(ChannelInterpretation interpretation); - protected: BaseAudioContext *context_; std::unique_ptr audioBus_; @@ -43,10 +42,12 @@ class AudioNode : public std::enable_shared_from_this { std::vector> inputNodes_ = {}; std::vector> outputNodes_ = {}; - bool processAudio(AudioBus *outputBus, int framesToProcess); + void processAudio(int framesToProcess); private: void cleanup(); + + virtual void processNode(int framesToProcess) = 0; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp index 596b967e..0cf8bbe5 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp @@ -1,5 +1,19 @@ +#ifdef ANDROID +#include "AudioPlayer.h" +#else +#include "IOSAudioPlayer.h" +#endif + #include "BaseAudioContext.h" +#include "GainNode.h" +#include "AudioBuffer.h" +#include "OscillatorNode.h" +#include "StereoPannerNode.h" +#include "BiquadFilterNode.h" +#include "AudioDestinationNode.h" +#include "AudioBufferSourceNode.h" + namespace audioapi { BaseAudioContext::BaseAudioContext() { @@ -12,12 +26,6 @@ BaseAudioContext::BaseAudioContext() { sampleRate_ = audioPlayer_->getSampleRate(); - auto now = std::chrono::high_resolution_clock ::now(); - contextStartTime_ = - static_cast(std::chrono::duration_cast( - now.time_since_epoch()) - .count()); - audioPlayer_->start(); } @@ -34,12 +42,7 @@ int BaseAudioContext::getBufferSizeInFrames() const { } double BaseAudioContext::getCurrentTime() const { - auto now = std::chrono::high_resolution_clock ::now(); - auto currentTime = - static_cast(std::chrono::duration_cast( - now.time_since_epoch()) - .count()); - return (currentTime - contextStartTime_) / 1e9; + return destination_->getCurrentTime(); } std::shared_ptr BaseAudioContext::getDestination() { @@ -73,13 +76,26 @@ std::shared_ptr BaseAudioContext::createBuffer( return std::make_shared(numberOfChannels, length, sampleRate); } -std::function BaseAudioContext::renderAudio() { +std::function BaseAudioContext::renderAudio() { if (state_ == State::CLOSED) { - return [](float *, int) {}; + return [](AudioBus*, int) {}; } - return [this](float *data, int frames) { + return [this](AudioBus* data, int frames) { destination_->renderAudio(data, frames); }; } + +std::string BaseAudioContext::toString(State state) { + switch (state) { + case State::SUSPENDED: + return "suspended"; + case State::RUNNING: + return "running"; + case State::CLOSED: + return "closed"; + default: + throw std::invalid_argument("Unknown context state"); + } +} } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h index c7d71af6..79a359fa 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h @@ -6,24 +6,22 @@ #include #include -#include "AudioBuffer.h" -#include "AudioBufferSourceNode.h" -#include "AudioDestinationNode.h" -#include "AudioScheduledSourceNode.h" -#include "BiquadFilterNode.h" -#include "Constants.h" -#include "GainNode.h" -#include "OscillatorNode.h" -#include "StereoPannerNode.h" +namespace audioapi { + +class AudioDestinationNode; +class OscillatorNode; +class GainNode; +class StereoPannerNode; +class BiquadFilterNode; +class AudioBufferSourceNode; +class AudioBuffer; #ifdef ANDROID -#include "AudioPlayer.h" +class AudioPlayer; #else -#include "IOSAudioPlayer.h" +class IOSAudioPlayer; #endif -namespace audioapi { - class BaseAudioContext { public: BaseAudioContext(); @@ -40,35 +38,22 @@ class BaseAudioContext { std::shared_ptr createBufferSource(); static std::shared_ptr createBuffer(int numberOfChannels, int length, int sampleRate); - std::function renderAudio(); + std::function renderAudio(); protected: enum class State { SUSPENDED, RUNNING, CLOSED }; - - static std::string toString(State state) { - switch (state) { - case State::SUSPENDED: - return "suspended"; - case State::RUNNING: - return "running"; - case State::CLOSED: - return "closed"; - default: - throw std::invalid_argument("Unknown context state"); - } - } - - protected: + static std::string toString(State state); std::shared_ptr destination_; + #ifdef ANDROID std::shared_ptr audioPlayer_; #else std::shared_ptr audioPlayer_; #endif + State state_ = State::RUNNING; int sampleRate_; int bufferSizeInFrames_; - double contextStartTime_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index d1752514..f536244e 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -1,3 +1,4 @@ +#include "Constants.h" #include "StereoPannerNode.h" #include "BaseAudioContext.h" diff --git a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h index 2d0ffff9..df6a6ebb 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h +++ b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.h @@ -3,7 +3,7 @@ #import #import -typedef void (^RenderAudioBlock)(float *audioData, int numFrames); +typedef void (^RenderAudioBlock)(AudioBufferList* outputBuffer, int numFrames); @interface AudioPlayer : NSObject @@ -12,7 +12,6 @@ typedef void (^RenderAudioBlock)(float *audioData, int numFrames); @property (nonatomic, strong) AVAudioFormat *format; @property (nonatomic, strong) AVAudioSourceNode *sourceNode; @property (nonatomic, copy) RenderAudioBlock renderAudio; -@property (nonatomic, assign) float *buffer; - (instancetype)initWithRenderAudioBlock:(RenderAudioBlock)renderAudio; diff --git a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m index 1c7f61e2..bb79ad81 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m +++ b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m @@ -12,8 +12,6 @@ - (instancetype)initWithRenderAudioBlock:(RenderAudioBlock)renderAudio self.audioSession = AVAudioSession.sharedInstance; NSError *error = nil; - NSLog(@"buffer time: %f %f", self.audioSession.IOBufferDuration, self.audioSession.IOBufferDuration * self.audioSession.sampleRate); - // TODO: // We will probably want to change it to AVAudioSessionCategoryPlayAndRecord in the future. // Eventually we to make this a dynamic setting, if user of the lib wants to use recording features. @@ -44,8 +42,6 @@ - (instancetype)initWithRenderAudioBlock:(RenderAudioBlock)renderAudio frameCount:frameCount outputData:outputData]; }]; - - self.buffer = malloc([self getBufferSizeInFrames] * 2 * sizeof(float)); } return self; @@ -96,8 +92,6 @@ - (void)cleanup self.audioEngine = nil; self.audioSession = nil; self.renderAudio = nil; - - free(_buffer); } - (OSStatus)renderCallbackWithIsSilence:(BOOL *)isSilence @@ -109,19 +103,7 @@ - (OSStatus)renderCallbackWithIsSilence:(BOOL *)isSilence return noErr; // Ensure we have stereo output } - memset(self.buffer, 0, frameCount * 2 * sizeof(float)); - - float *leftBuffer = (float *)outputData->mBuffers[0].mData; - float *rightBuffer = (float *)outputData->mBuffers[1].mData; - - // self.renderAudio(self.buffer, frameCount); - - for (int frame = 0; frame < frameCount; frame += 1) { - leftBuffer[frame] = 0; - rightBuffer[frame] = 0; - // leftBuffer[frame] = self.buffer[frame * 2]; - // rightBuffer[frame] = self.buffer[frame * 2 + 1]; - } + self.renderAudio(outputData, frameCount); return noErr; } diff --git a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h index 19832eaa..f8f0f5b1 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h +++ b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.h @@ -11,14 +11,16 @@ typedef struct objc_object AudioPlayer; namespace audioapi { class AudioContext; +class AudioBus; class IOSAudioPlayer { protected: - AudioPlayer *audioPlayer_; - std::function renderAudio_; + AudioBus* audioBus_; + AudioPlayer* audioPlayer_; + std::function renderAudio_; public: - explicit IOSAudioPlayer(const std::function &renderAudio); + explicit IOSAudioPlayer(const std::function &renderAudio); ~IOSAudioPlayer(); int getSampleRate() const; @@ -26,7 +28,6 @@ class IOSAudioPlayer { void start(); void stop(); - void renderAudio(float *audioData, int32_t numFrames); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm index c6a9a0a3..70fe15ad 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm @@ -1,11 +1,24 @@ +#import + +#include +#include +#include #include namespace audioapi { -IOSAudioPlayer::IOSAudioPlayer(const std::function &renderAudio) : renderAudio_(renderAudio) +IOSAudioPlayer::IOSAudioPlayer(const std::function &renderAudio) : renderAudio_(renderAudio) { - RenderAudioBlock renderAudioBlock = ^(float *audioData, int numFrames) { - renderAudio_(audioData, numFrames); + audioBus_ = new AudioBus(getSampleRate(), getBufferSizeInFrames(), CHANNEL_COUNT); + + RenderAudioBlock renderAudioBlock = ^(AudioBufferList* outputData, int numFrames) { + renderAudio_(audioBus_, numFrames); + + for (int i = 0; i < outputData->mNumberBuffers; i += 1) { + float *outputBuffer = (float *)outputData->mBuffers[i].mData; + + memcpy(outputBuffer, audioBus_->getChannel(i)->getData(), sizeof(float) * numFrames); + } }; audioPlayer_ = [[AudioPlayer alloc] initWithRenderAudioBlock:renderAudioBlock]; @@ -15,6 +28,7 @@ { stop(); [audioPlayer_ cleanup]; + delete audioBus_; } void IOSAudioPlayer::start() @@ -36,4 +50,5 @@ { return [audioPlayer_ getBufferSizeInFrames]; } + } // namespace audioapi From 5ce4a575b3f4383a9693a3c64d60bcc4bb42ff27 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Mon, 18 Nov 2024 18:34:44 +0100 Subject: [PATCH 10/33] feat: processNode implementations --- .../common/cpp/core/AudioBuffer.cpp | 126 ++++++------------ .../common/cpp/core/AudioBuffer.h | 4 - .../common/cpp/core/AudioBufferSourceNode.cpp | 87 +++++++----- .../common/cpp/core/AudioBufferSourceNode.h | 9 +- .../common/cpp/core/AudioDestinationNode.cpp | 36 ++--- .../common/cpp/core/AudioDestinationNode.h | 11 +- .../common/cpp/core/AudioNode.cpp | 68 ++++++++-- .../common/cpp/core/AudioNode.h | 9 +- .../cpp/core/AudioScheduledSourceNode.cpp | 28 +--- .../cpp/core/AudioScheduledSourceNode.h | 11 +- .../common/cpp/core/BaseAudioContext.cpp | 4 + .../common/cpp/core/BaseAudioContext.h | 1 + .../common/cpp/core/BiquadFilterNode.cpp | 95 +++++++------ .../common/cpp/core/BiquadFilterNode.h | 4 +- .../common/cpp/core/OscillatorNode.cpp | 77 ++++++----- .../common/cpp/core/OscillatorNode.h | 4 +- .../common/cpp/core/StereoPannerNode.cpp | 54 ++++---- .../common/cpp/core/StereoPannerNode.h | 4 +- .../ios/AudioPlayer/IOSAudioPlayer.mm | 6 +- 19 files changed, 324 insertions(+), 314 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp index bef4f1c9..1223adc4 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp @@ -1,96 +1,56 @@ +#include "AudioBus.h" +#include "AudioArray.h" #include "AudioBuffer.h" namespace audioapi { -AudioBuffer::AudioBuffer(int numberOfChannels, int length, int sampleRate) - : numberOfChannels_(numberOfChannels), - length_(length), - sampleRate_(sampleRate), - duration_(static_cast(length) / sampleRate) { +AudioBuffer::AudioBuffer(int numberOfChannels, int length, int sampleRate) { + bus_ = std::make_unique(sampleRate, length, numberOfChannels); + bus_->zero(); } -} // namespace audioapi - - -// AudioBuffer::AudioBuffer(int numberOfChannels, int length, int sampleRate) -// : numberOfChannels_(numberOfChannels), -// length_(length), -// sampleRate_(sampleRate), -// duration_(static_cast(length) / sampleRate) { -// channels_ = new float *[numberOfChannels]; - -// for (int i = 0; i < numberOfChannels; i++) { -// channels_[i] = new float[length]; - -// for (int j = 0; j < length; j++) { -// channels_[i][j] = 0.0f; -// } -// } -// } - -// int AudioBuffer::getNumberOfChannels() const { -// return numberOfChannels_; -// } - -// int AudioBuffer::getLength() const { -// return length_; -// } - -// int AudioBuffer::getSampleRate() const { -// return sampleRate_; -// } - -// double AudioBuffer::getDuration() const { -// return duration_; -// } +int AudioBuffer::getLength() const { + return bus_->getSize(); +} -// float *AudioBuffer::getChannelData(int channel) const { -// return channels_[channel]; -// } +int AudioBuffer::getNumberOfChannels() const { + return bus_->getNumberOfChannels(); +} -// std::shared_ptr AudioBuffer::mix(int outputNumberOfChannels) { -// if (outputNumberOfChannels == numberOfChannels_) { -// return shared_from_this(); -// } +int AudioBuffer::getSampleRate() const { + return bus_->getSampleRate(); +} -// auto mixedBuffer = std::make_shared( -// outputNumberOfChannels, length_, sampleRate_); +double AudioBuffer::getDuration() const { + return static_cast(getLength()) / getSampleRate(); +} -// switch (this->numberOfChannels_) { -// case 1: -// mixedBuffer->copyToChannel(this->channels_[0], length_, 0, 0); -// mixedBuffer->copyToChannel(this->channels_[0], length_, 1, 0); -// break; -// case 2: -// for (int i = 0; i < length_; i++) { -// mixedBuffer->channels_[0][i] = -// (this->channels_[0][i] + this->channels_[1][i]) / 2; -// } -// break; -// } +float* AudioBuffer::getChannelData(int channel) const { + return bus_->getChannel(channel)->getData(); +} -// return mixedBuffer; -// } +void AudioBuffer::copyFromChannel( + float *destination, + int destinationLength, + int channelNumber, + int startInChannel) const { + memcpy( + destination, + bus_->getChannel(channelNumber)->getData() + startInChannel, + std::min(destinationLength, getLength() - startInChannel) * sizeof(float) + ); +} -// void AudioBuffer::copyFromChannel( -// float *destination, -// int destinationLength, -// int channelNumber, -// int startInChannel) const { -// std::copy( -// channels_[channelNumber] + startInChannel, -// channels_[channelNumber] + startInChannel + -// std::min(destinationLength, length_ - startInChannel), -// destination); -// } +void AudioBuffer::copyToChannel( + const float *source, + int sourceLength, + int channelNumber, + int startInChannel) { + memcpy( + bus_->getChannel(channelNumber)->getData() + startInChannel, + source, + std::min(sourceLength, getLength() - startInChannel) * sizeof(float) + ); +} -// void AudioBuffer::copyToChannel( -// const float *source, -// int sourceLength, -// int channelNumber, -// int startInChannel) { -// std::copy( -// source, -// source + std::min(sourceLength, length_ - startInChannel), -// channels_[channelNumber] + startInChannel); -// } +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h index 98df0771..c281e12c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h @@ -32,10 +32,6 @@ class AudioBuffer : public std::enable_shared_from_this { int startInChannel); private: - int numberOfChannels_; - int length_; - int sampleRate_; - double duration_; std::unique_ptr bus_; }; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index 97cbe8f7..20d37883 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -1,59 +1,74 @@ -#include "AudioBufferSourceNode.h" +#include "AudioBus.h" +#include "AudioArray.h" #include "BaseAudioContext.h" +#include "AudioBufferSourceNode.h" namespace audioapi { AudioBufferSourceNode::AudioBufferSourceNode(BaseAudioContext *context) : AudioScheduledSourceNode(context), loop_(false), bufferIndex_(0) { - // numberOfInputs_ = 0; - // buffer_ = std::shared_ptr(nullptr); + numberOfInputs_ = 0; + buffer_ = std::shared_ptr(nullptr); } bool AudioBufferSourceNode::getLoop() const { - // return loop_; + return loop_; } std::shared_ptr AudioBufferSourceNode::getBuffer() const { - // return buffer_; + return buffer_; } void AudioBufferSourceNode::setLoop(bool loop) { - // loop_ = loop; + loop_ = loop; } void AudioBufferSourceNode::setBuffer( const std::shared_ptr &buffer) { - // if (!buffer) { - // buffer_ = std::shared_ptr(nullptr); - // return; - // } - // buffer_ = buffer->mix(channelCount_); + if (!buffer) { + buffer_ = std::shared_ptr(nullptr); + return; + } + + buffer_ = buffer; +} + +void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToProcess) { + double time = context_->getCurrentTime(); + handlePlayback(time, framesToProcess); + + if (!isPlaying_ || !buffer_) { + processingBus->zero(); + return; + } + + /* + TODO: tomorrow: + 1. Add zeroRange, copyRange and sumRange to AudioArray and AudioBus + 2. Implement the following pseudo code: + + int framesToCopy = std::min(framesToProcess, buffer_->getLength() - bufferIndex_); + int framesToFill = framesToProcess - framesToCopy; + + processingBus->sumRange(buffer_->getAudioBus() + bufferIndex_, 0, framesToCopy); + + bufferIndex_ += framesToCopy; + + if (bufferIndex_ >= buffer_->getLength()) { + bufferIndex_ -= buffer_->getLength(); + } + + if (framesToFill > 0) { + if (loop_) { + processingBus->sumRange(buffer_->getAudioBus(), framesToCopy, framesToFill); + } else { + processingBus->zeroRange(framesToCopy, framesToFill); + isPlaying_ = false; + bufferIndex_ = 0; + } + } +*/ } -// bool AudioBufferSourceNode::processAudio(float *audioData, int32_t numFrames) { -// if (!isPlaying_ || !buffer_) { -// return false; -// } else { -// for (int i = 0; i < numFrames; ++i) { -// for (int j = 0; j < channelCount_; j++) { -// audioData[i * channelCount_ + j] = -// buffer_->getChannelData(j)[bufferIndex_]; -// } - -// bufferIndex_++; - -// if (bufferIndex_ >= buffer_->getLength()) { -// if (loop_) { -// bufferIndex_ = 0; -// } else { -// isPlaying_ = false; -// break; -// } -// } -// } - -// return true; -// } -// } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h index ed993357..f2810aa3 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h @@ -7,6 +7,8 @@ namespace audioapi { +class AudioBus; + class AudioBufferSourceNode : public AudioScheduledSourceNode { public: explicit AudioBufferSourceNode(BaseAudioContext *context); @@ -15,11 +17,14 @@ class AudioBufferSourceNode : public AudioScheduledSourceNode { [[nodiscard]] std::shared_ptr getBuffer() const; void setLoop(bool loop); void setBuffer(const std::shared_ptr &buffer); - // [[nodiscard]] bool processAudio(float *audioData, int32_t numFrames) override; + + protected: + void processNode(AudioBus* processingBus, int framesToProcess) override; private: bool loop_; std::shared_ptr buffer_; - int bufferIndex_; + std::size_t bufferIndex_; }; + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 4cff221c..39523640 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -1,6 +1,8 @@ -#include "AudioDestinationNode.h" -#include "BaseAudioContext.h" +#include "AudioBus.h" +#include "AudioNode.h" #include "VectorMath.h" +#include "BaseAudioContext.h" +#include "AudioDestinationNode.h" namespace audioapi { @@ -11,37 +13,23 @@ AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) channelCountMode_ = ChannelCountMode::EXPLICIT; } -unsigned AudioDestinationNode::getCurrentSampleFrame() const { +std::size_t AudioDestinationNode::getCurrentSampleFrame() const { return currentSampleFrame_; } double AudioDestinationNode::getCurrentTime() const { - return currentSampleFrame_ / context_->getSampleRate(); + return static_cast(currentSampleFrame_) / context_->getSampleRate(); } void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFrames) { - // int32_t numSamples = numFrames * CHANNEL_COUNT; + destinationBus->zero(); - // memset(destinationBuffer, 0.0f, sizeof(float) * numSamples); + if (!numFrames) { + return; + } - // if (!numFrames) { - // return; - // } - - // processAudio(numFrames); - - // currentSampleFrame_ += numFrames; + processAudio(destinationBus, numFrames); + currentSampleFrame_ += numFrames; } -// bool AudioDestinationNode::processAudio(float *audioData, int32_t numFrames) { -// for (auto &node : inputNodes_) { -// if (node && node->processAudio(mixingBuffer.get(), numFrames)) { -// normalize(mixingBuffer.get(), numFrames); -// VectorMath::add(audioData, mixingBuffer.get(), audioData, numSamples); -// } -// } - -// return true; -// } - } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h index 4428a08a..f345cdd2 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.h @@ -17,17 +17,16 @@ class AudioDestinationNode : public AudioNode { void renderAudio(AudioBus* audioData, int32_t numFrames); - unsigned getCurrentSampleFrame() const; + std::size_t getCurrentSampleFrame() const; double getCurrentTime() const; protected: - - private: - unsigned currentSampleFrame_; - // DestinationNode is triggered by AudioContext using renderAudio // processNode function is not necessary and is never called. - void processNode(int) final { }; + void processNode(AudioBus*, int) final { }; + + private: + std::size_t currentSampleFrame_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 1ec0f816..71f969a5 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -75,18 +75,62 @@ std::string AudioNode::toString(ChannelInterpretation interpretation) { } } -void AudioNode::processAudio(int framesToProcess) { - - // bool isPlaying = false; - // for (auto &node : inputNodes_) { - // if (node->processAudio(buffer_.get(), framesToProcess)) { - // isPlaying = true; - // } - // } - - // if (isPlaying) { - // outputBus->copyFrom(*buffer_.get(), 0, 0, framesToProcess); - // } +AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { + std::size_t currentSampleFrame = context_->getCurrentSampleFrame(); + + // check if the node has already been processed for this rendering quantum + bool isAlreadyProcessed = currentSampleFrame == lastRenderedFrame_; + + // Node can't use output bus if: + // - outputBus is not provided, which means that next node is doing a multi-node summing. + // - it has more than one input, which means that it has to sum all inputs using internal bus. + // - it has more than one output, so each output node can get the processed data without re-calculating the node. + bool canUseOutputBus = outputBus != 0 && inputNodes_.size() == 1 && outputNodes_.size() == 1; + + if (isAlreadyProcessed) { + // If it was already processed in the rendering quantum,return it. + return audioBus_.get(); + } + + // Update the last rendered frame before processing node and its inputs. + lastRenderedFrame_ = currentSampleFrame; + + AudioBus* processingBus = canUseOutputBus ? outputBus : audioBus_.get(); + + if (!canUseOutputBus) { + // Clear the bus before summing all connected nodes. + processingBus->zero(); + } + + if (inputNodes_.empty()) { + // If there are no connected inputs, process the node just to advance the audio params. + // The node will output silence anyway. + processNode(processingBus, framesToProcess); + return processingBus; + } + + // Process first connected node, it can be directly connected to the processingBus, + // resulting in one less summing operation in most cases. + AudioBus* firstNodeResultBus = inputNodes_.front()->processAudio(processingBus, framesToProcess); + + if (firstNodeResultBus != processingBus) { + processingBus->sum(*firstNodeResultBus); + } + + if (inputNodes_.size() > 1) { + // If there are more connected nodes, sum them together. + for (auto it = inputNodes_.begin() + 1; it != inputNodes_.end(); it += 1) { + // Enforce the summing to be done using the internal bus. + AudioBus* inputBus = it->get()->processAudio(0, framesToProcess); + + processingBus->sum(*inputBus); + } + } + + // Finally, process the node itself. + processNode(processingBus, framesToProcess); + + return processingBus; } void AudioNode::cleanup() { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index dd175e03..9de81879 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -3,6 +3,7 @@ #include #include #include +#include "Constants.h" namespace audioapi { @@ -35,6 +36,8 @@ class AudioNode : public std::enable_shared_from_this { int numberOfOutputs_ = 1; int channelCount_ = CHANNEL_COUNT; + std::size_t lastRenderedFrame_ { SIZE_MAX }; + ChannelCountMode channelCountMode_ = ChannelCountMode::MAX; ChannelInterpretation channelInterpretation_ = ChannelInterpretation::SPEAKERS; @@ -42,12 +45,10 @@ class AudioNode : public std::enable_shared_from_this { std::vector> inputNodes_ = {}; std::vector> outputNodes_ = {}; - void processAudio(int framesToProcess); - private: void cleanup(); - - virtual void processNode(int framesToProcess) = 0; + AudioBus* processAudio(AudioBus* outputBus, int framesToProcess); + virtual void processNode(AudioBus* processingBus, int framesToProcess) = 0; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp index 741e09c1..42d0fc9a 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp @@ -1,5 +1,5 @@ -#include "AudioScheduledSourceNode.h" #include "BaseAudioContext.h" +#include "AudioScheduledSourceNode.h" namespace audioapi { @@ -9,31 +9,17 @@ AudioScheduledSourceNode::AudioScheduledSourceNode(BaseAudioContext *context) } void AudioScheduledSourceNode::start(double time) { - waitAndExecute(time, [this](double time) { startPlayback(); }); + nextChangeTime_ = time; } void AudioScheduledSourceNode::stop(double time) { - waitAndExecute(time, [this](double time) { stopPlayback(); }); -} - -void AudioScheduledSourceNode::startPlayback() { - isPlaying_ = true; + nextChangeTime_ = time; } -void AudioScheduledSourceNode::stopPlayback() { - isPlaying_ = false; -} - -void AudioScheduledSourceNode::waitAndExecute( - double time, - const std::function &fun) { - std::thread([this, time, fun]() { - while (context_->getCurrentTime() < time) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - fun(time); - }).detach(); +void AudioScheduledSourceNode::handlePlayback(double time, int framesToProcess) { + if (nextChangeTime_ <= time + (framesToProcess / context_->getSampleRate())) { + isPlaying_ = !isPlaying_; + } } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h index 85691aef..472e3234 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h @@ -1,11 +1,6 @@ #pragma once #include -#include -#include -#include -#include -#include #include "AudioNode.h" @@ -20,11 +15,9 @@ class AudioScheduledSourceNode : public AudioNode { protected: std::atomic isPlaying_; + std::atomic nextChangeTime_ { -1.0 }; - private: - void startPlayback(); - void stopPlayback(); - void waitAndExecute(double time, const std::function &fun); + void handlePlayback(double time, int framesToProcess); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp index 0cf8bbe5..c58f4ba6 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp @@ -41,6 +41,10 @@ int BaseAudioContext::getBufferSizeInFrames() const { return bufferSizeInFrames_; } +std::size_t BaseAudioContext::getCurrentSampleFrame() const { + return destination_->getCurrentSampleFrame(); +} + double BaseAudioContext::getCurrentTime() const { return destination_->getCurrentTime(); } diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h index 79a359fa..d02e32fb 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h @@ -29,6 +29,7 @@ class BaseAudioContext { [[nodiscard]] int getSampleRate() const; [[nodiscard]] double getCurrentTime() const; [[nodiscard]] int getBufferSizeInFrames() const; + [[nodiscard]] std::size_t getCurrentSampleFrame() const; std::shared_ptr getDestination(); std::shared_ptr createOscillator(); diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index fabb61e9..ed8f162e 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -1,3 +1,5 @@ +#include "AudioBus.h" +#include "AudioArray.h" #include "BiquadFilterNode.h" #include "BaseAudioContext.h" @@ -92,12 +94,12 @@ void BiquadFilterNode::setNormalizedCoefficients( float a0, float a1, float a2) { - auto a0Inversed = 1.0f / a0; - b0_ = b0 * a0Inversed; - b1_ = b1 * a0Inversed; - b2_ = b2 * a0Inversed; - a1_ = a1 * a0Inversed; - a2_ = a2 * a0Inversed; + auto a0Inverted = 1.0f / a0; + b0_ = b0 * a0Inverted; + b1_ = b1 * a0Inverted; + b2_ = b2 * a0Inverted; + a1_ = a1 * a0Inverted; + a2_ = a2 * a0Inverted; } void BiquadFilterNode::setLowpassCoefficients(float frequency, float Q) { @@ -116,11 +118,11 @@ void BiquadFilterNode::setLowpassCoefficients(float frequency, float Q) { float theta = M_PI * frequency; float alpha = std::sin(theta) / (2 * g); - float cosw = std::cos(theta); - float beta = (1 - cosw) / 2; + float cosW = std::cos(theta); + float beta = (1 - cosW) / 2; setNormalizedCoefficients( - beta, 2 * beta, beta, 1 + alpha, -2 * cosw, 1 - alpha); + beta, 2 * beta, beta, 1 + alpha, -2 * cosW, 1 - alpha); } void BiquadFilterNode::setHighpassCoefficients(float frequency, float Q) { @@ -139,11 +141,11 @@ void BiquadFilterNode::setHighpassCoefficients(float frequency, float Q) { float theta = M_PI * frequency; float alpha = std::sin(theta) / (2 * g); - float cosw = std::cos(theta); - float beta = (1 - cosw) / 2; + float cosW = std::cos(theta); + float beta = (1 - cosW) / 2; setNormalizedCoefficients( - beta, -2 * beta, beta, 1 + alpha, -2 * cosw, 1 - alpha); + beta, -2 * beta, beta, 1 + alpha, -2 * cosW, 1 - alpha); } void BiquadFilterNode::setBandpassCoefficients(float frequency, float Q) { @@ -351,40 +353,37 @@ void BiquadFilterNode::applyFilter() { } } -// bool BiquadFilterNode::processAudio(float *audioData, int32_t numFrames) { -// if (!AudioNode::processAudio(audioData, numFrames)) { -// return false; -// } - -// resetCoefficients(); -// applyFilter(); - -// float x1 = x1_; -// float x2 = x2_; -// float y1 = y1_; -// float y2 = y2_; - -// float b0 = b0_; -// float b1 = b1_; -// float b2 = b2_; -// float a1 = a1_; -// float a2 = a2_; - -// for (int i = 0; i < numFrames; i++) { -// auto input = audioData[i * channelCount_]; -// auto output = -// static_cast(b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2); - -// for (int j = 0; j < channelCount_; j++) { -// audioData[i * channelCount_ + j] = output; -// } - -// x2 = x1; -// x1 = input; -// y2 = y1; -// y1 = output; -// } - -// return true; -// } +void BiquadFilterNode::processNode(AudioBus* processingBus, int framesToProcess) { + resetCoefficients(); + applyFilter(); + + for (int i = 0; i < processingBus->getNumberOfChannels(); i += 1) { + AudioArray* inputChannel = processingBus->getChannel(i); + + // Reset the coefficients for each channel + float x1 = x1_; + float x2 = x2_; + float y1 = y1_; + float y2 = y2_; + + float b0 = b0_; + float b1 = b1_; + float b2 = b2_; + float a1 = a1_; + float a2 = a2_; + + for (int f = 0; i < framesToProcess; f += 1) { + float input = (*inputChannel)[f]; + float output = b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; + + (*inputChannel)[f] = output; + + x2 = x1; + x1 = input; + y2 = y1; + y1 = output; + } + } +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h index 7265d089..3891343b 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.h @@ -13,6 +13,8 @@ namespace audioapi { +class AudioBus; + class BiquadFilterNode : public AudioNode { public: explicit BiquadFilterNode(BaseAudioContext *context); @@ -29,7 +31,7 @@ class BiquadFilterNode : public AudioNode { std::vector &phaseResponseOutput); protected: - // bool processAudio(float *audioData, int32_t numFrames) override; + void processNode(AudioBus *processingBus, int framesToProcess) override; private: enum class FilterType { diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index dc790bf2..89190637 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -1,3 +1,5 @@ +#include "AudioBus.h" +#include "AudioArray.h" #include "OscillatorNode.h" #include "BaseAudioContext.h" @@ -27,40 +29,43 @@ void OscillatorNode::setType(const std::string &type) { type_ = OscillatorNode::fromString(type); } -// bool OscillatorNode::processAudio(float *audioData, int32_t numFrames) { -// if (!isPlaying_) { -// return false; -// } else { -// auto time = context_->getCurrentTime(); -// auto deltaTime = 1.0 / context_->getSampleRate(); - -// for (int i = 0; i < numFrames; ++i) { -// auto detuneRatio = -// std::pow(2.0f, detuneParam_->getValueAtTime(time) / 1200.0f); -// auto detunedFrequency = -// round(frequencyParam_->getValueAtTime(time) * detuneRatio); -// auto phaseIncrement = static_cast( -// 2 * M_PI * detunedFrequency / context_->getSampleRate()); - -// float value = OscillatorNode::getWaveBufferElement(phase_, type_); - -// for (int j = 0; j < channelCount_; j++) { -// audioData[i * channelCount_ + j] = value; -// } - -// phase_ += phaseIncrement; -// time += deltaTime; - -// if (phase_ >= 2 * M_PI) { -// phase_ -= 2 * M_PI; -// } - -// if (phase_ < 0) { -// phase_ += 2 * M_PI; -// } -// } - -// return true; -// } -// } +void OscillatorNode::processNode(AudioBus* processingBus, int framesToProcess) { + double time = context_->getCurrentTime(); + handlePlayback(time, framesToProcess); + + if (!isPlaying_) { + processingBus->zero(); + return; + } + + double deltaTime = 1.0 / context_->getSampleRate(); + + for (int i = 0; i < framesToProcess; i += 1) { + auto detuneRatio = + std::pow(2.0f, detuneParam_->getValueAtTime(time) / 1200.0f); + auto detunedFrequency = + round(frequencyParam_->getValueAtTime(time) * detuneRatio); + auto phaseIncrement = static_cast( + 2 * M_PI * detunedFrequency / context_->getSampleRate()); + + float value = OscillatorNode::getWaveBufferElement(phase_, type_); + + for (int j = 0; j < channelCount_; j += 1) { + // Call the [] operator directly for better readability ;) + processingBus->getChannel(j)->operator[](i) = value; + } + + phase_ += phaseIncrement; + time += deltaTime; + + if (phase_ >= 2 * M_PI) { + phase_ -= 2 * M_PI; + } + + if (phase_ < 0) { + phase_ += 2 * M_PI; + } + } +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h index 03a1b181..5f1751aa 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h @@ -9,6 +9,8 @@ namespace audioapi { +class AudioBus; + class OscillatorNode : public AudioScheduledSourceNode { public: explicit OscillatorNode(BaseAudioContext *context); @@ -19,7 +21,7 @@ class OscillatorNode : public AudioScheduledSourceNode { void setType(const std::string &type); protected: - // bool processAudio(float *audioData, int32_t numFrames) override; + void processNode(AudioBus *processingBus, int framesToProcess) override; private: enum class WaveType { SINE, SQUARE, SAWTOOTH, TRIANGLE, CUSTOM }; diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index f536244e..cad4bfb1 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -1,4 +1,6 @@ +#include "AudioBus.h" #include "Constants.h" +#include "AudioArray.h" #include "StereoPannerNode.h" #include "BaseAudioContext.h" @@ -16,36 +18,38 @@ std::shared_ptr StereoPannerNode::getPanParam() const { return panParam_; } -// bool StereoPannerNode::processAudio(float *audioData, int32_t numFrames) { -// // assumed channelCount = 2 -// if (!AudioNode::processAudio(audioData, numFrames)) { -// return false; -// } +void StereoPannerNode::processNode(AudioBus* processingBus, int framesToProcess) { + // TODO: Currently assumed channelCount is 2 + // it should: + // - support mono-channel buses + // - throw errors when trying to setup stereo panner with more than 2 channels -// auto time = context_->getCurrentTime(); -// auto deltaTime = 1.0 / context_->getSampleRate(); + double time = context_->getCurrentTime(); + double deltaTime = 1.0 / context_->getSampleRate(); -// for (int i = 0; i < numFrames; i++) { -// auto pan = panParam_->getValueAtTime(time); -// auto x = (pan <= 0 ? pan + 1 : pan) * M_PI / 2; + AudioArray* left = processingBus->getChannelByType(AudioBus::ChannelLeft); + AudioArray* right = processingBus->getChannelByType(AudioBus::ChannelRight); -// auto gainL = static_cast(cos(x)); -// auto gainR = static_cast(sin(x)); + for (int i = 0; i < framesToProcess; i += 1) { + float pan = panParam_->getValueAtTime(time); + float x = (pan <= 0 ? pan + 1 : pan) * M_PI / 2; -// auto inputL = audioData[i * 2]; -// auto inputR = audioData[i * 2 + 1]; + float gainL = static_cast(cos(x)); + float gainR = static_cast(sin(x)); -// if (pan <= 0) { -// audioData[i * 2] = inputL + inputR * gainL; -// audioData[i * 2 + 1] = inputR * gainR; -// } else { -// audioData[i * 2] = inputL * gainL; -// audioData[i * 2 + 1] = inputR + inputL * gainR; -// } + float inputL = (*left)[i]; + float inputR = (*right)[i]; -// time += deltaTime; -// } + if (pan <= 0) { + (*left)[i] = inputL + inputR * gainL; + (*right)[i] = inputR * gainR; + } else { + (*left)[i] = inputL * gainL; + (*right)[i] = inputR + inputL * gainR; + } -// return true; -// } + time += deltaTime; + } + +} } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h index d49ad1a4..b3550e3d 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.h @@ -8,6 +8,8 @@ namespace audioapi { +class AudioBus; + class StereoPannerNode : public AudioNode { public: explicit StereoPannerNode(BaseAudioContext *context); @@ -15,7 +17,7 @@ class StereoPannerNode : public AudioNode { [[nodiscard]] std::shared_ptr getPanParam() const; protected: - // bool processAudio(float *audioData, int32_t numFrames) override; + void processNode(AudioBus* processingBus, int framesToProcess) override; private: std::shared_ptr panParam_; diff --git a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm index 70fe15ad..5f2daa56 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm @@ -28,7 +28,11 @@ { stop(); [audioPlayer_ cleanup]; - delete audioBus_; + + if (audioBus_) { + delete audioBus_; + audioBus_ = 0; + } } void IOSAudioPlayer::start() From e07102dc9e726aa437c4e28f775a4805a8956058 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Mon, 18 Nov 2024 18:38:22 +0100 Subject: [PATCH 11/33] feat: self-abuse --- packages/react-native-audio-api/common/cpp/core/GainNode.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index d9438fc8..f55dbbad 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -11,6 +11,8 @@ std::shared_ptr GainNode::getGainParam() const { return gainParam_; } + +// TODO TOMORROW: YOU FORGOT TO IMPLEMENT THIS FUNCTION MORON! // bool GainNode::processAudio(float *audioData, int32_t numFrames) { // if (!AudioNode::processAudio(audioData, numFrames)) { // return false; From ec1f407e3e41b26ff975947e0182149fe81ad9f6 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 19 Nov 2024 12:25:35 +0100 Subject: [PATCH 12/33] feat: migrate gain node --- .../common/cpp/core/GainNode.cpp | 27 ++++++++----------- .../common/cpp/core/GainNode.h | 4 ++- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index f55dbbad..eb388f1e 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -1,4 +1,6 @@ #include "GainNode.h" +#include "AudioBus.h" +#include "AudioArray.h" #include "BaseAudioContext.h" namespace audioapi { @@ -11,22 +13,15 @@ std::shared_ptr GainNode::getGainParam() const { return gainParam_; } +void GainNode::processNode(AudioBus *processingBus, int framesToProcess) { + double time = context_->getCurrentTime(); + double deltaTime = 1.0 / context_->getSampleRate(); -// TODO TOMORROW: YOU FORGOT TO IMPLEMENT THIS FUNCTION MORON! -// bool GainNode::processAudio(float *audioData, int32_t numFrames) { -// if (!AudioNode::processAudio(audioData, numFrames)) { -// return false; -// } - -// auto time = context_->getCurrentTime(); -// auto deltaTime = 1.0 / context_->getSampleRate(); - -// for (int i = 0; i < numFrames * channelCount_; i++) { -// audioData[i] *= gainParam_->getValueAtTime(time); -// time += deltaTime; -// } - -// return true; -// } + for (int i = 0; i < framesToProcess; i += 1) { + for (int j = 0; j < processingBus->getNumberOfChannels(); j += 1) { + (*processingBus->getChannel(j))[i] *= gainParam_->getValueAtTime(time); + } + } +} } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.h b/packages/react-native-audio-api/common/cpp/core/GainNode.h index 65f12b8e..ee387791 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.h +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.h @@ -7,6 +7,8 @@ namespace audioapi { +class AudioBus; + class GainNode : public AudioNode { public: explicit GainNode(BaseAudioContext *context); @@ -14,7 +16,7 @@ class GainNode : public AudioNode { [[nodiscard]] std::shared_ptr getGainParam() const; protected: - // bool processAudio(float *audioData, int32_t numFrames) override; + void processNode(AudioBus *processingBus, int framesToProcess) override; private: std::shared_ptr gainParam_; From d742c8d1ab1ef5948035122a7fccc4a919420a55 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 19 Nov 2024 12:59:06 +0100 Subject: [PATCH 13/33] feat: audio bus & audio array range operators --- .../common/cpp/core/AudioArray.cpp | 52 ++++++--- .../common/cpp/core/AudioArray.h | 16 ++- .../common/cpp/core/AudioBus.cpp | 110 +++++++++++------- .../common/cpp/core/AudioBus.h | 17 ++- 4 files changed, 122 insertions(+), 73 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index b8ee252c..1ce437a9 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -29,8 +29,14 @@ const float& AudioArray::operator[](int index) const { return data_[index]; } -void AudioArray::zero() { - memset(data_, 0, size_ * sizeof(float)); +void AudioArray::normalize() { + float maxAbsValue = getMaxAbsValue(); + + if (maxAbsValue == 0.0f || maxAbsValue == 1.0f) { + return; + } + + VectorMath::multiplyByScalar(data_, 1.0f / maxAbsValue, data_, size_); } void AudioArray::resize(int size) { @@ -46,35 +52,45 @@ void AudioArray::resize(int size) { zero(); } -void AudioArray::copy(const AudioArray* source) { - if (size_ != source->getSize()) { - resize(source->getSize()); - } - - memcpy(data_, source->getData(), size_ * sizeof(float)); +void AudioArray::scale(float value) { + VectorMath::multiplyByScalar(data_, value, data_, size_); } float AudioArray::getMaxAbsValue() const { return VectorMath::maximumMagnitude(data_, size_); } -void AudioArray::normalize() { - float maxAbsValue = getMaxAbsValue(); - - if (maxAbsValue == 0.0f || maxAbsValue == 1.0f) { - return; - } - - VectorMath::multiplyByScalar(data_, 1.0f / maxAbsValue, data_, size_); +void AudioArray::zero() { + memset(data_, 0, size_ * sizeof(float)); } -void AudioArray::scale(float value) { - VectorMath::multiplyByScalar(data_, value, data_, size_); +void AudioArray::zero(int start, int length) { + memset(data_ + start, 0, length * sizeof(float)); } void AudioArray::sum(const AudioArray* source) { VectorMath::add(data_, source->getData(), data_, size_); } +void AudioArray::sum(const AudioArray* source, int start, int length) { + VectorMath::add(data_ + start, source->getData() + start, data_ + start, length); +} + +void AudioArray::sum(const AudioArray* source, int sourceStart, int destinationStart, int length) { + VectorMath::add(data_ + destinationStart, source->getData() + sourceStart, data_ + destinationStart, length); +} + +void AudioArray::copy(const AudioArray* source) { + memcpy(data_, source->getData(), size_ * sizeof(float)); +} + +void AudioArray::copy(const AudioArray* source, int start, int length) { + memcpy(data_ + start, source->getData() + start, length * sizeof(float)); +} + +void AudioArray::copy(const AudioArray* source, int sourceStart, int destinationStart, int length) { + memcpy(data_ + destinationStart, source->getData() + sourceStart, length * sizeof(float)); +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.h b/packages/react-native-audio-api/common/cpp/core/AudioArray.h index edaa586d..3c59eb90 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.h @@ -17,15 +17,21 @@ class AudioArray { float& operator[](int index); const float& operator[](int index) const; - void zero(); + void normalize(); void resize(int size); - void copy(const AudioArray* source); - + void scale(float value); float getMaxAbsValue() const; - void normalize(); - void scale(float value); + void zero(); + void zero(int start, int length); + void sum(const AudioArray* source); + void sum(const AudioArray* source, int start, int length); + void sum(const AudioArray* source, int sourceStart, int destinationStart, int length); + + void copy(const AudioArray* source); + void copy(const AudioArray* source, int start, int length); + void copy(const AudioArray* source, int sourceStart, int destinationStart, int length); private: diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index 341803ab..26515320 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -104,8 +104,12 @@ AudioArray* AudioBus::getChannelByType(int channelType) const { */ void AudioBus::zero() { + zero(0, getSize()); +} + +void AudioBus::zero(int start, int length) { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { - it->get()->zero(); + it->get()->zero(start, length); } } @@ -137,17 +141,15 @@ float AudioBus::maxAbsValue() const { return maxAbsValue; } -void AudioBus::copy(const AudioBus &source) { - if (&source == this) { - return; - } +void AudioBus::sum(const AudioBus &source) { + sum(source, 0, 0, getSize()); +} - // zero + sum is equivalent to copy, but takes care of up/down-mixing. - zero(); - sum(source); +void AudioBus::sum(const AudioBus &source, int start, int length) { + sum(source, start, start, length); } -void AudioBus::sum(const AudioBus &source) { +void AudioBus::sum(const AudioBus &source, int sourceStart, int destinationStart, int length) { if (&source == this) { return; } @@ -158,20 +160,38 @@ void AudioBus::sum(const AudioBus &source) { // TODO: consider adding ability to enforce discrete summing (if/when it will be useful). // Source channel count is smaller than current bus, we need to up-mix. if (numberOfSourceChannels < numberOfChannels) { - sumByUpMixing(source); + sumByUpMixing(source, sourceStart, destinationStart, length); return; } // Source channel count is larger than current bus, we need to down-mix. if (numberOfSourceChannels > numberOfChannels) { - sumByDownMixing(source); + sumByDownMixing(source, sourceStart, destinationStart, length); return; } // Source and destination channel counts are the same. Just sum the channels. for (int i = 0; i < numberOfChannels_; i++) { - channels_[i]->sum(source.getChannel(i)); + channels_[i]->sum(source.getChannel(i), sourceStart, destinationStart, length); + } +} + +void AudioBus::copy(const AudioBus &source) { + copy(source, 0, 0, getSize()); +} + +void AudioBus::copy(const AudioBus &source, int start, int length) { + copy(source, start, start, length); +} + +void AudioBus::copy(const AudioBus &source, int sourceStart, int destinationStart, int length) { + if (&source == this) { + return; } + + // zero + sum is equivalent to copy, but takes care of up/down-mixing. + zero(destinationStart, length); + sum(source, sourceStart, destinationStart, length); } /** @@ -187,17 +207,17 @@ void AudioBus::createChannels() { * Internal tooling - channel summing */ -void AudioBus::discreteSum(const AudioBus &source) { +void AudioBus::discreteSum(const AudioBus &source, int sourceStart, int destinationStart, int length) { int numberOfChannels = std::min(getNumberOfChannels(), source.getNumberOfChannels()); // In case of source > destination, we "down-mix" and drop the extra channels. // In case of source < destination, we "up-mix" as many channels as we have, leaving the remaining channels untouched. for (int i = 0; i < numberOfChannels; i++) { - getChannel(i)->sum(source.getChannel(i)); + getChannel(i)->sum(source.getChannel(i), sourceStart, destinationStart, length); } } -void AudioBus::sumByUpMixing(const AudioBus &source) { +void AudioBus::sumByUpMixing(const AudioBus &source, int sourceStart, int destinationStart, int length) { int numberOfSourceChannels = source.getNumberOfChannels(); int numberOfChannels = getNumberOfChannels(); @@ -205,8 +225,8 @@ void AudioBus::sumByUpMixing(const AudioBus &source) { if (numberOfSourceChannels == 1 && (numberOfChannels == 2 || numberOfChannels == 4)) { AudioArray* sourceChannel = source.getChannelByType(ChannelMono); - getChannelByType(ChannelLeft)->sum(sourceChannel); - getChannelByType(ChannelRight)->sum(sourceChannel); + getChannelByType(ChannelLeft)->sum(sourceChannel, sourceStart, destinationStart, length); + getChannelByType(ChannelRight)->sum(sourceChannel, sourceStart, destinationStart, length); return; } @@ -214,30 +234,30 @@ void AudioBus::sumByUpMixing(const AudioBus &source) { if (numberOfSourceChannels == 1 && numberOfChannels == 6) { AudioArray* sourceChannel = source.getChannel(0); - getChannelByType(ChannelCenter)->sum(sourceChannel); + getChannelByType(ChannelCenter)->sum(sourceChannel, sourceStart, destinationStart, length); return; } // Stereo 2 to stereo 4 or 5.1 (2 -> 4, 6) if (numberOfSourceChannels == 2 && (numberOfChannels == 4 || numberOfChannels == 6)) { - getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft)); - getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight)); + getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft), sourceStart, destinationStart, length); + getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight), sourceStart, destinationStart, length); return; } // Stereo 4 to 5.1 (4 -> 6) if (numberOfSourceChannels == 4 && numberOfChannels == 6) { - getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft)); - getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight)); - getChannelByType(ChannelSurroundLeft)->sum(source.getChannelByType(ChannelSurroundLeft)); - getChannelByType(ChannelSurroundRight)->sum(source.getChannelByType(ChannelSurroundRight)); + getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft), sourceStart, destinationStart, length); + getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight), sourceStart, destinationStart, length); + getChannelByType(ChannelSurroundLeft)->sum(source.getChannelByType(ChannelSurroundLeft), sourceStart, destinationStart, length); + getChannelByType(ChannelSurroundRight)->sum(source.getChannelByType(ChannelSurroundRight), sourceStart, destinationStart, length); return; } - discreteSum(source); + discreteSum(source, sourceStart, destinationStart, length); } -void AudioBus::sumByDownMixing(const AudioBus &source) { +void AudioBus::sumByDownMixing(const AudioBus &source, int sourceStart, int destinationStart, int length) { int numberOfSourceChannels = source.getNumberOfChannels(); int numberOfChannels = getNumberOfChannels(); @@ -248,8 +268,8 @@ void AudioBus::sumByDownMixing(const AudioBus &source) { float* destinationData = getChannelByType(ChannelMono)->getData(); - VectorMath::multiplyByScalarThenAddToOutput(sourceLeft, 0.5f, destinationData, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceRight, 0.5f, destinationData, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceLeft + sourceStart, 0.5f, destinationData + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceRight + sourceStart, 0.5f, destinationData + destinationStart, length); return; } @@ -262,10 +282,10 @@ void AudioBus::sumByDownMixing(const AudioBus &source) { float* destinationData = getChannelByType(ChannelMono)->getData(); - VectorMath::multiplyByScalarThenAddToOutput(sourceLeft, 0.25f, destinationData, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceRight, 0.25f, destinationData, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundLeft, 0.25f, destinationData, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundRight, 0.25f, destinationData, getSize()); + VectorMath::multiplyByScalarThenAddToOutput(sourceLeft + sourceStart, 0.25f, destinationData + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceRight + sourceStart, 0.25f, destinationData + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundLeft + sourceStart, 0.25f, destinationData + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundRight + sourceStart, 0.25f, destinationData + destinationStart, length); return; } @@ -282,13 +302,13 @@ void AudioBus::sumByDownMixing(const AudioBus &source) { float* destinationLeft = getChannelByType(ChannelLeft)->getData(); float* destinationRight = getChannelByType(ChannelRight)->getData(); - VectorMath::add(sourceLeft, destinationLeft, destinationLeft, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationLeft, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundLeft, SQRT_HALF, destinationLeft, getSize()); + VectorMath::add(sourceLeft + sourceStart, destinationLeft + destinationStart, destinationLeft + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter + sourceStart, SQRT_HALF, destinationLeft + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundLeft + sourceStart, SQRT_HALF, destinationLeft + destinationStart, length); - VectorMath::add(sourceRight, destinationRight, destinationRight, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationRight, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundRight, SQRT_HALF, destinationRight, getSize()); + VectorMath::add(sourceRight + sourceStart, destinationRight + destinationStart, destinationRight + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter + sourceStart, SQRT_HALF, destinationRight + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceSurroundRight + sourceStart, SQRT_HALF, destinationRight + destinationStart, length); return; } @@ -309,18 +329,18 @@ void AudioBus::sumByDownMixing(const AudioBus &source) { float* destinationSurroundLeft = getChannelByType(ChannelSurroundLeft)->getData(); float* destinationSurroundRight = getChannelByType(ChannelSurroundRight)->getData(); - VectorMath::add(sourceLeft, destinationLeft, destinationLeft, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationLeft, getSize()); + VectorMath::add(sourceLeft + sourceStart, destinationLeft + destinationStart, destinationLeft + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationLeft + destinationStart, length); - VectorMath::add(sourceRight, destinationRight, destinationRight, getSize()); - VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationRight, getSize()); + VectorMath::add(sourceRight + sourceStart, destinationRight + destinationStart, destinationRight + destinationStart, length); + VectorMath::multiplyByScalarThenAddToOutput(sourceCenter, SQRT_HALF, destinationRight + destinationStart, length); - VectorMath::add(sourceSurroundLeft, destinationSurroundLeft, destinationSurroundLeft, getSize()); - VectorMath::add(sourceSurroundRight, destinationSurroundRight, destinationSurroundRight, getSize()); + VectorMath::add(sourceSurroundLeft + sourceStart, destinationSurroundLeft + destinationStart, destinationSurroundLeft + destinationStart, length); + VectorMath::add(sourceSurroundRight + sourceStart, destinationSurroundRight + destinationStart, destinationSurroundRight + destinationStart, length); return; } - discreteSum(source); + discreteSum(source, sourceStart, destinationStart, length); } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.h b/packages/react-native-audio-api/common/cpp/core/AudioBus.h index 1164eac8..4feaeed1 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.h @@ -32,13 +32,20 @@ class AudioBus { AudioArray* getChannel(int index) const; AudioArray* getChannelByType(int channelType) const; - void zero(); void normalize(); void scale(float value); float maxAbsValue() const; - void copy(const AudioBus &source); + void zero(); + void zero(int start, int length); + void sum(const AudioBus &source); + void sum(const AudioBus &source, int start, int length); + void sum(const AudioBus &source, int sourceStart, int destinationStart, int length); + + void copy(const AudioBus &source); + void copy(const AudioBus &source, int start, int length); + void copy(const AudioBus &source, int sourceStart, int destinationStart, int length); private: std::vector> channels_; @@ -48,9 +55,9 @@ class AudioBus { int size_; void createChannels(); - void discreteSum(const AudioBus &source); - void sumByUpMixing(const AudioBus &source); - void sumByDownMixing(const AudioBus &source); + void discreteSum(const AudioBus &source, int sourceStart, int destinationStart, int length); + void sumByUpMixing(const AudioBus &source, int sourceStart, int destinationStart, int length); + void sumByDownMixing(const AudioBus &source, int sourceStart, int destinationStart, int length); }; } // namespace audioapi From 82c4cb9816c12876508a824b6d418c9e19edc724 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 19 Nov 2024 15:58:09 +0100 Subject: [PATCH 14/33] feat: audio buffer source node - rewrite to use audio bus and complicate the matter --- .../common/cpp/core/AudioArray.cpp | 10 +-- .../common/cpp/core/AudioBuffer.h | 2 + .../common/cpp/core/AudioBufferSourceNode.cpp | 87 ++++++++++++++----- .../common/cpp/core/AudioBufferSourceNode.h | 2 +- .../common/cpp/core/AudioBus.cpp | 8 ++ 5 files changed, 81 insertions(+), 28 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index 1ce437a9..ad5a087b 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -61,7 +61,7 @@ float AudioArray::getMaxAbsValue() const { } void AudioArray::zero() { - memset(data_, 0, size_ * sizeof(float)); + zero(0, size_); } void AudioArray::zero(int start, int length) { @@ -69,11 +69,11 @@ void AudioArray::zero(int start, int length) { } void AudioArray::sum(const AudioArray* source) { - VectorMath::add(data_, source->getData(), data_, size_); + sum(source, 0, 0, size_); } void AudioArray::sum(const AudioArray* source, int start, int length) { - VectorMath::add(data_ + start, source->getData() + start, data_ + start, length); + sum(source, start, start, length); } void AudioArray::sum(const AudioArray* source, int sourceStart, int destinationStart, int length) { @@ -81,11 +81,11 @@ void AudioArray::sum(const AudioArray* source, int sourceStart, int destinationS } void AudioArray::copy(const AudioArray* source) { - memcpy(data_, source->getData(), size_ * sizeof(float)); + copy(source, 0, size_); } void AudioArray::copy(const AudioArray* source, int start, int length) { - memcpy(data_ + start, source->getData() + start, length * sizeof(float)); + copy(source, start, start, length); } void AudioArray::copy(const AudioArray* source, int sourceStart, int destinationStart, int length) { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h index c281e12c..f9ad5b1e 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h @@ -33,6 +33,8 @@ class AudioBuffer : public std::enable_shared_from_this { private: std::unique_ptr bus_; + + friend class AudioBufferSourceNode; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index 20d37883..bac50b08 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -34,41 +34,84 @@ void AudioBufferSourceNode::setBuffer( buffer_ = buffer; } +// Note: AudioBus copy method will use memcpy if the source buffer and system processing bus have same channel count, +// otherwise it will use the summing function taking care of up/down mixing. void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToProcess) { double time = context_->getCurrentTime(); handlePlayback(time, framesToProcess); - if (!isPlaying_ || !buffer_) { + // No audio data to fill, zero the output and return. + if (!isPlaying_ || !buffer_ || buffer_->getLength() == 0) { processingBus->zero(); return; } - /* - TODO: tomorrow: - 1. Add zeroRange, copyRange and sumRange to AudioArray and AudioBus - 2. Implement the following pseudo code: + // Easiest case, the buffer is the same length as the number of frames to process, just copy the data. + if (framesToProcess == buffer_->getLength()) { + processingBus->copy(*buffer_->bus_.get()); - int framesToCopy = std::min(framesToProcess, buffer_->getLength() - bufferIndex_); - int framesToFill = framesToProcess - framesToCopy; + if (!loop_) { + isPlaying_ = false; + } - processingBus->sumRange(buffer_->getAudioBus() + bufferIndex_, 0, framesToCopy); + return; + } + + // The buffer is longer than the number of frames to process. + // We have to keep track of where we are in the buffer. + if (framesToProcess < buffer_->getLength()) { + int framesToCopy = std::min(framesToProcess, buffer_->getLength() - bufferIndex_); + + processingBus->copy(*buffer_->bus_.get(), bufferIndex_, framesToCopy); - bufferIndex_ += framesToCopy; + if (loop_) { + bufferIndex_ = (bufferIndex_ + framesToCopy) % buffer_->getLength(); + return; + } - if (bufferIndex_ >= buffer_->getLength()) { - bufferIndex_ -= buffer_->getLength(); - } + if (bufferIndex_ + framesToCopy == buffer_->getLength() - 1) { + isPlaying_ = false; + bufferIndex_ = 0; + } - if (framesToFill > 0) { - if (loop_) { - processingBus->sumRange(buffer_->getAudioBus(), framesToCopy, framesToFill); - } else { - processingBus->zeroRange(framesToCopy, framesToFill); - isPlaying_ = false; - bufferIndex_ = 0; - } - } -*/ + return; + } + + // processing bus is longer than the source buffer + if (!loop_) { + // If we don't loop the buffer, copy it once and zero the remaining processing bus frames. + processingBus->copy(*buffer_->bus_.get()); + processingBus->zero(buffer_->getLength(), framesToProcess - buffer_->getLength()); + isPlaying_ = false; + return; + } + + // If we loop the buffer, we need to loop the buffer framesToProcess / bufferSize times + // There might also be a remainder of frames to copy after the loop, + // which will also carry over some buffer frames to the next render quantum. + int processingBusPosition = 0; + int bufferSize = buffer_->getLength(); + int remainingFrames = framesToProcess - framesToProcess / bufferSize; + + // Do we have some frames left in the buffer from the previous render quantum, + // if yes copy them over and reset the buffer position. + if (bufferIndex_ > 0) { + processingBus->copy(*buffer_->bus_.get(), 0, bufferIndex_); + processingBusPosition += bufferIndex_; + bufferIndex_ = 0; + } + + // Copy the entire buffer n times to the processing bus. + while (processingBusPosition + bufferSize <= framesToProcess) { + processingBus->copy(*buffer_->bus_.get()); + processingBusPosition += bufferSize; + } + + // Fill in the remaining frames from the processing buffer and update buffer index for next render quantum. + if (remainingFrames > 0) { + processingBus->copy(*buffer_->bus_.get(), 0, processingBusPosition, remainingFrames); + bufferIndex_ = remainingFrames; + } } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h index f2810aa3..66feac08 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.h @@ -24,7 +24,7 @@ class AudioBufferSourceNode : public AudioScheduledSourceNode { private: bool loop_; std::shared_ptr buffer_; - std::size_t bufferIndex_; + int bufferIndex_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index 26515320..eb97344c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -189,6 +189,14 @@ void AudioBus::copy(const AudioBus &source, int sourceStart, int destinationStar return; } + if (source.getNumberOfChannels() == getNumberOfChannels()) { + for (int i = 0; i < getNumberOfChannels(); i += 1) { + getChannel(i)->copy(source.getChannel(i), sourceStart, destinationStart, length); + } + + return; + } + // zero + sum is equivalent to copy, but takes care of up/down-mixing. zero(destinationStart, length); sum(source, sourceStart, destinationStart, length); From 13d1d0da50c0644da40e0bc009d0866f9129004d Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 19 Nov 2024 17:17:06 +0100 Subject: [PATCH 15/33] fix: some memory alloc fixes --- .../common/cpp/core/AudioArray.cpp | 13 ++++++++++--- .../common/cpp/core/AudioBus.cpp | 14 +++++++------- .../common/cpp/core/AudioBus.h | 2 +- .../common/cpp/core/AudioContext.cpp | 7 +++++++ .../common/cpp/core/BaseAudioContext.cpp | 2 ++ .../common/cpp/core/BaseAudioContext.h | 12 ++++++------ .../common/cpp/core/GainNode.cpp | 2 ++ .../ios/AudioPlayer/IOSAudioPlayer.mm | 10 +++++----- 8 files changed, 40 insertions(+), 22 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index ad5a087b..ecded254 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -5,12 +5,15 @@ namespace audioapi { -AudioArray::AudioArray(int size) : size_(size) { - resize(size); +AudioArray::AudioArray(int size) : size_(size), data_(0) { + data_ = new float[size]; } AudioArray::~AudioArray() { - delete[] data_; + if (data_) { + delete[] data_; + data_ = 0; + } } int AudioArray::getSize() const { @@ -41,6 +44,10 @@ void AudioArray::normalize() { void AudioArray::resize(int size) { if (size == size_) { + if (!data_) { + data_ = new float[size]; + } + zero(); return; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index eb97344c..49c5e99c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -47,7 +47,7 @@ int AudioBus::getSize() const { } AudioArray* AudioBus::getChannel(int index) const { - return channels_[index].get(); + return (AudioArray*)&channels_[index]; } AudioArray* AudioBus::getChannelByType(int channelType) const { @@ -109,7 +109,7 @@ void AudioBus::zero() { void AudioBus::zero(int start, int length) { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { - it->get()->zero(start, length); + it->zero(start, length); } } @@ -126,7 +126,7 @@ void AudioBus::normalize() { void AudioBus::scale(float value) { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { - it->get()->scale(value); + it->scale(value); } } @@ -134,7 +134,7 @@ float AudioBus::maxAbsValue() const { float maxAbsValue = 0.0f; for (auto it = channels_.begin(); it != channels_.end(); it += 1) { - float channelMaxAbsValue = it->get()->getMaxAbsValue(); + float channelMaxAbsValue = it->getMaxAbsValue(); maxAbsValue = std::max(maxAbsValue, channelMaxAbsValue); } @@ -171,8 +171,8 @@ void AudioBus::sum(const AudioBus &source, int sourceStart, int destinationStart } // Source and destination channel counts are the same. Just sum the channels. - for (int i = 0; i < numberOfChannels_; i++) { - channels_[i]->sum(source.getChannel(i), sourceStart, destinationStart, length); + for (int i = 0; i < numberOfChannels_; i += 1) { + channels_[i].sum(source.getChannel(i), sourceStart, destinationStart, length); } } @@ -208,7 +208,7 @@ void AudioBus::copy(const AudioBus &source, int sourceStart, int destinationStar void AudioBus::createChannels() { // TODO: verify if its correct way of memory allocation - channels_ = std::vector>(numberOfChannels_, std::make_unique(size_)); + channels_ = std::vector(numberOfChannels_, AudioArray(size_)); } /** diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.h b/packages/react-native-audio-api/common/cpp/core/AudioBus.h index 4feaeed1..40504f09 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.h @@ -48,7 +48,7 @@ class AudioBus { void copy(const AudioBus &source, int sourceStart, int destinationStart, int length); private: - std::vector> channels_; + std::vector channels_; int numberOfChannels_; int sampleRate_; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/AudioContext.cpp index 9c92567d..80231067 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioContext.cpp @@ -1,5 +1,12 @@ +#ifdef ANDROID +#include "AudioPlayer.h" +#else +#include "IOSAudioPlayer.h" +#endif + #include "AudioContext.h" + namespace audioapi { AudioContext::AudioContext() : BaseAudioContext() {} diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp index c58f4ba6..87c6f882 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp @@ -7,6 +7,8 @@ #include "BaseAudioContext.h" #include "GainNode.h" +#include "AudioBus.h" +#include "AudioArray.h" #include "AudioBuffer.h" #include "OscillatorNode.h" #include "StereoPannerNode.h" diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h index d02e32fb..e8eec2d1 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h @@ -8,13 +8,14 @@ namespace audioapi { -class AudioDestinationNode; -class OscillatorNode; +class AudioBus; class GainNode; +class AudioBuffer; +class OscillatorNode; class StereoPannerNode; class BiquadFilterNode; +class AudioDestinationNode; class AudioBufferSourceNode; -class AudioBuffer; #ifdef ANDROID class AudioPlayer; @@ -37,9 +38,8 @@ class BaseAudioContext { std::shared_ptr createStereoPanner(); std::shared_ptr createBiquadFilter(); std::shared_ptr createBufferSource(); - static std::shared_ptr - createBuffer(int numberOfChannels, int length, int sampleRate); - std::function renderAudio(); + static std::shared_ptr createBuffer(int numberOfChannels, int length, int sampleRate); + std::function renderAudio(); protected: enum class State { SUSPENDED, RUNNING, CLOSED }; diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index eb388f1e..88bc6f50 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -21,6 +21,8 @@ void GainNode::processNode(AudioBus *processingBus, int framesToProcess) { for (int j = 0; j < processingBus->getNumberOfChannels(); j += 1) { (*processingBus->getChannel(j))[i] *= gainParam_->getValueAtTime(time); } + + time += deltaTime; } } diff --git a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm index 5f2daa56..ba3e68a4 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm @@ -7,21 +7,21 @@ namespace audioapi { -IOSAudioPlayer::IOSAudioPlayer(const std::function &renderAudio) : renderAudio_(renderAudio) +IOSAudioPlayer::IOSAudioPlayer(const std::function &renderAudio) : renderAudio_(renderAudio), audioBus_(0) { - audioBus_ = new AudioBus(getSampleRate(), getBufferSizeInFrames(), CHANNEL_COUNT); - RenderAudioBlock renderAudioBlock = ^(AudioBufferList* outputData, int numFrames) { renderAudio_(audioBus_, numFrames); - + for (int i = 0; i < outputData->mNumberBuffers; i += 1) { float *outputBuffer = (float *)outputData->mBuffers[i].mData; - + memcpy(outputBuffer, audioBus_->getChannel(i)->getData(), sizeof(float) * numFrames); } }; audioPlayer_ = [[AudioPlayer alloc] initWithRenderAudioBlock:renderAudioBlock]; + audioBus_ = new AudioBus(getSampleRate(), getBufferSizeInFrames(), CHANNEL_COUNT); + NSLog(@"info: %d %d", getSampleRate(), getBufferSizeInFrames()); } IOSAudioPlayer::~IOSAudioPlayer() From d4e948cbea6ec20e3b5d5f295202d81ab3f7caac Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 19 Nov 2024 20:24:52 +0100 Subject: [PATCH 16/33] fix: get back to previous scheduling impl --- .../src/examples/DrumMachine/usePlayer.tsx | 1 + .../src/examples/Oscillator/Oscillator.tsx | 19 ++- .../src/main/cpp/AudioPlayer/AudioPlayer.cpp | 2 +- .../src/main/cpp/AudioPlayer/AudioPlayer.h | 2 +- .../common/cpp/core/AudioArray.cpp | 7 +- .../common/cpp/core/AudioBuffer.cpp | 3 +- .../common/cpp/core/AudioBuffer.h | 2 +- .../common/cpp/core/AudioBufferSourceNode.cpp | 13 +- .../common/cpp/core/AudioBus.cpp | 111 +++++++++--------- .../common/cpp/core/AudioBus.h | 20 ++-- .../common/cpp/core/AudioDestinationNode.cpp | 10 +- .../common/cpp/core/AudioNode.cpp | 26 ++-- .../common/cpp/core/AudioNode.h | 2 +- .../cpp/core/AudioScheduledSourceNode.cpp | 27 ++++- .../cpp/core/AudioScheduledSourceNode.h | 11 +- .../common/cpp/core/BaseAudioContext.cpp | 3 +- .../common/cpp/core/OscillatorNode.cpp | 5 +- .../ios/AudioPlayer/IOSAudioPlayer.mm | 3 +- 18 files changed, 147 insertions(+), 120 deletions(-) diff --git a/apps/common-app/src/examples/DrumMachine/usePlayer.tsx b/apps/common-app/src/examples/DrumMachine/usePlayer.tsx index 6f977556..47d75777 100644 --- a/apps/common-app/src/examples/DrumMachine/usePlayer.tsx +++ b/apps/common-app/src/examples/DrumMachine/usePlayer.tsx @@ -134,6 +134,7 @@ export default function usePlayer(options: PlayerOptions) { } return () => { + console.log('Closing audio context'); audioContext.close(); }; // \/ Shared values are not necessary in deps array diff --git a/apps/common-app/src/examples/Oscillator/Oscillator.tsx b/apps/common-app/src/examples/Oscillator/Oscillator.tsx index e6055693..b4509714 100644 --- a/apps/common-app/src/examples/Oscillator/Oscillator.tsx +++ b/apps/common-app/src/examples/Oscillator/Oscillator.tsx @@ -42,16 +42,15 @@ const Oscillator: FC = () => { oscillatorRef.current.frequency.value = frequency; oscillatorRef.current.detune.value = detune; oscillatorRef.current.type = oscillatorType; - - gainRef.current = audioContextRef.current.createGain(); - gainRef.current.gain.value = gain; - - panRef.current = audioContextRef.current.createStereoPanner(); - panRef.current.pan.value = pan; - - oscillatorRef.current.connect(gainRef.current); - gainRef.current.connect(panRef.current); - panRef.current.connect(audioContextRef.current.destination); + oscillatorRef.current.connect(audioContextRef.current.destination); + + // gainRef.current = audioContextRef.current.createGain(); + // gainRef.current.gain.value = gain; + // panRef.current = audioContextRef.current.createStereoPanner(); + // panRef.current.pan.value = pan; + // oscillatorRef.current.connect(gainRef.current); + // gainRef.current.connect(panRef.current); + // panRef.current.connect(audioContextRef.current.destination); }; const handleGainChange = (newValue: number) => { diff --git a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp index 65554ad9..34a37a52 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp @@ -18,7 +18,7 @@ AudioPlayer::AudioPlayer(const std::function &renderAudio) ->setDataCallback(this) ->openStream(mStream_); - mBus_ = std::make_unique(getSampleRate(), getBufferSizeInFrames(), CHANNEL_COUNT); + mBus_ = std::make_shared(getSampleRate(), getBufferSizeInFrames(), CHANNEL_COUNT); } int AudioPlayer::getSampleRate() const { diff --git a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h index c4aa89ef..26383f07 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h +++ b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.h @@ -27,7 +27,7 @@ class AudioPlayer : public AudioStreamDataCallback { private: std::function renderAudio_; std::shared_ptr mStream_; - std::unique_ptr mBus_; + std::shared_ptr mBus_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index ecded254..e3b606dd 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -6,7 +6,8 @@ namespace audioapi { AudioArray::AudioArray(int size) : size_(size), data_(0) { - data_ = new float[size]; + printf("AudioArray::AudioArray(%d)\n", size); + resize(size); } AudioArray::~AudioArray() { @@ -48,7 +49,7 @@ void AudioArray::resize(int size) { data_ = new float[size]; } - zero(); + zero(0, size); return; } @@ -56,7 +57,7 @@ void AudioArray::resize(int size) { size_ = size; data_ = new float[size_]; - zero(); + zero(0, size_); } void AudioArray::scale(float value) { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp index 1223adc4..4027abed 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.cpp @@ -5,8 +5,7 @@ namespace audioapi { AudioBuffer::AudioBuffer(int numberOfChannels, int length, int sampleRate) { - bus_ = std::make_unique(sampleRate, length, numberOfChannels); - bus_->zero(); + bus_ = std::make_shared(sampleRate, length, numberOfChannels); } int AudioBuffer::getLength() const { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h index f9ad5b1e..bca9e193 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBuffer.h @@ -32,7 +32,7 @@ class AudioBuffer : public std::enable_shared_from_this { int startInChannel); private: - std::unique_ptr bus_; + std::shared_ptr bus_; friend class AudioBufferSourceNode; }; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index bac50b08..3aed052f 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -38,7 +38,6 @@ void AudioBufferSourceNode::setBuffer( // otherwise it will use the summing function taking care of up/down mixing. void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToProcess) { double time = context_->getCurrentTime(); - handlePlayback(time, framesToProcess); // No audio data to fill, zero the output and return. if (!isPlaying_ || !buffer_ || buffer_->getLength() == 0) { @@ -48,7 +47,7 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro // Easiest case, the buffer is the same length as the number of frames to process, just copy the data. if (framesToProcess == buffer_->getLength()) { - processingBus->copy(*buffer_->bus_.get()); + processingBus->copy(buffer_->bus_.get()); if (!loop_) { isPlaying_ = false; @@ -62,7 +61,7 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro if (framesToProcess < buffer_->getLength()) { int framesToCopy = std::min(framesToProcess, buffer_->getLength() - bufferIndex_); - processingBus->copy(*buffer_->bus_.get(), bufferIndex_, framesToCopy); + processingBus->copy(buffer_->bus_.get(), bufferIndex_, framesToCopy); if (loop_) { bufferIndex_ = (bufferIndex_ + framesToCopy) % buffer_->getLength(); @@ -80,7 +79,7 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro // processing bus is longer than the source buffer if (!loop_) { // If we don't loop the buffer, copy it once and zero the remaining processing bus frames. - processingBus->copy(*buffer_->bus_.get()); + processingBus->copy(buffer_->bus_.get()); processingBus->zero(buffer_->getLength(), framesToProcess - buffer_->getLength()); isPlaying_ = false; return; @@ -96,20 +95,20 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro // Do we have some frames left in the buffer from the previous render quantum, // if yes copy them over and reset the buffer position. if (bufferIndex_ > 0) { - processingBus->copy(*buffer_->bus_.get(), 0, bufferIndex_); + processingBus->copy(buffer_->bus_.get(), 0, bufferIndex_); processingBusPosition += bufferIndex_; bufferIndex_ = 0; } // Copy the entire buffer n times to the processing bus. while (processingBusPosition + bufferSize <= framesToProcess) { - processingBus->copy(*buffer_->bus_.get()); + processingBus->copy(buffer_->bus_.get()); processingBusPosition += bufferSize; } // Fill in the remaining frames from the processing buffer and update buffer index for next render quantum. if (remainingFrames > 0) { - processingBus->copy(*buffer_->bus_.get(), 0, processingBusPosition, remainingFrames); + processingBus->copy(buffer_->bus_.get(), 0, processingBusPosition, remainingFrames); bufferIndex_ = remainingFrames; } } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index 49c5e99c..a6386aaf 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -47,7 +47,7 @@ int AudioBus::getSize() const { } AudioArray* AudioBus::getChannel(int index) const { - return (AudioArray*)&channels_[index]; + return channels_[index].get(); } AudioArray* AudioBus::getChannelByType(int channelType) const { @@ -109,7 +109,7 @@ void AudioBus::zero() { void AudioBus::zero(int start, int length) { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { - it->zero(start, length); + it->get()->zero(start, length); } } @@ -126,35 +126,35 @@ void AudioBus::normalize() { void AudioBus::scale(float value) { for (auto it = channels_.begin(); it != channels_.end(); it += 1) { - it->scale(value); + it->get()->scale(value); } } float AudioBus::maxAbsValue() const { - float maxAbsValue = 0.0f; + float maxAbsValue = 1.0f; for (auto it = channels_.begin(); it != channels_.end(); it += 1) { - float channelMaxAbsValue = it->getMaxAbsValue(); + float channelMaxAbsValue = it->get()->getMaxAbsValue(); maxAbsValue = std::max(maxAbsValue, channelMaxAbsValue); } return maxAbsValue; } -void AudioBus::sum(const AudioBus &source) { +void AudioBus::sum(const AudioBus *source) { sum(source, 0, 0, getSize()); } -void AudioBus::sum(const AudioBus &source, int start, int length) { +void AudioBus::sum(const AudioBus *source, int start, int length) { sum(source, start, start, length); } -void AudioBus::sum(const AudioBus &source, int sourceStart, int destinationStart, int length) { - if (&source == this) { +void AudioBus::sum(const AudioBus *source, int sourceStart, int destinationStart, int length) { + if (source == this) { return; } - int numberOfSourceChannels = source.getNumberOfChannels(); + int numberOfSourceChannels = source->getNumberOfChannels(); int numberOfChannels = getNumberOfChannels(); // TODO: consider adding ability to enforce discrete summing (if/when it will be useful). @@ -172,30 +172,30 @@ void AudioBus::sum(const AudioBus &source, int sourceStart, int destinationStart // Source and destination channel counts are the same. Just sum the channels. for (int i = 0; i < numberOfChannels_; i += 1) { - channels_[i].sum(source.getChannel(i), sourceStart, destinationStart, length); + getChannel(i)->sum(source->getChannel(i), sourceStart, destinationStart, length); } } -void AudioBus::copy(const AudioBus &source) { +void AudioBus::copy(const AudioBus *source) { copy(source, 0, 0, getSize()); } -void AudioBus::copy(const AudioBus &source, int start, int length) { +void AudioBus::copy(const AudioBus *source, int start, int length) { copy(source, start, start, length); } -void AudioBus::copy(const AudioBus &source, int sourceStart, int destinationStart, int length) { - if (&source == this) { +void AudioBus::copy(const AudioBus *source, int sourceStart, int destinationStart, int length) { + if (source == this) { return; } - if (source.getNumberOfChannels() == getNumberOfChannels()) { - for (int i = 0; i < getNumberOfChannels(); i += 1) { - getChannel(i)->copy(source.getChannel(i), sourceStart, destinationStart, length); - } + // if (source->getNumberOfChannels() == getNumberOfChannels()) { + // for (int i = 0; i < getNumberOfChannels(); i += 1) { + // getChannel(i)->copy(source->getChannel(i), sourceStart, destinationStart, length); + // } - return; - } + // return; + // } // zero + sum is equivalent to copy, but takes care of up/down-mixing. zero(destinationStart, length); @@ -207,31 +207,34 @@ void AudioBus::copy(const AudioBus &source, int sourceStart, int destinationStar */ void AudioBus::createChannels() { - // TODO: verify if its correct way of memory allocation - channels_ = std::vector(numberOfChannels_, AudioArray(size_)); + channels_ = std::vector>(numberOfChannels_); + + for (int i = 0; i < numberOfChannels_; i += 1) { + channels_[i] = std::make_shared(size_); + } } /** * Internal tooling - channel summing */ -void AudioBus::discreteSum(const AudioBus &source, int sourceStart, int destinationStart, int length) { - int numberOfChannels = std::min(getNumberOfChannels(), source.getNumberOfChannels()); +void AudioBus::discreteSum(const AudioBus *source, int sourceStart, int destinationStart, int length) { + int numberOfChannels = std::min(getNumberOfChannels(), source->getNumberOfChannels()); // In case of source > destination, we "down-mix" and drop the extra channels. // In case of source < destination, we "up-mix" as many channels as we have, leaving the remaining channels untouched. for (int i = 0; i < numberOfChannels; i++) { - getChannel(i)->sum(source.getChannel(i), sourceStart, destinationStart, length); + getChannel(i)->sum(source->getChannel(i), sourceStart, destinationStart, length); } } -void AudioBus::sumByUpMixing(const AudioBus &source, int sourceStart, int destinationStart, int length) { - int numberOfSourceChannels = source.getNumberOfChannels(); +void AudioBus::sumByUpMixing(const AudioBus *source, int sourceStart, int destinationStart, int length) { + int numberOfSourceChannels = source->getNumberOfChannels(); int numberOfChannels = getNumberOfChannels(); // Mono to stereo (1 -> 2, 4) if (numberOfSourceChannels == 1 && (numberOfChannels == 2 || numberOfChannels == 4)) { - AudioArray* sourceChannel = source.getChannelByType(ChannelMono); + AudioArray* sourceChannel = source->getChannelByType(ChannelMono); getChannelByType(ChannelLeft)->sum(sourceChannel, sourceStart, destinationStart, length); getChannelByType(ChannelRight)->sum(sourceChannel, sourceStart, destinationStart, length); @@ -240,7 +243,7 @@ void AudioBus::sumByUpMixing(const AudioBus &source, int sourceStart, int destin // Mono to 5.1 (1 -> 6) if (numberOfSourceChannels == 1 && numberOfChannels == 6) { - AudioArray* sourceChannel = source.getChannel(0); + AudioArray* sourceChannel = source->getChannel(0); getChannelByType(ChannelCenter)->sum(sourceChannel, sourceStart, destinationStart, length); return; @@ -248,31 +251,31 @@ void AudioBus::sumByUpMixing(const AudioBus &source, int sourceStart, int destin // Stereo 2 to stereo 4 or 5.1 (2 -> 4, 6) if (numberOfSourceChannels == 2 && (numberOfChannels == 4 || numberOfChannels == 6)) { - getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft), sourceStart, destinationStart, length); - getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight), sourceStart, destinationStart, length); + getChannelByType(ChannelLeft)->sum(source->getChannelByType(ChannelLeft), sourceStart, destinationStart, length); + getChannelByType(ChannelRight)->sum(source->getChannelByType(ChannelRight), sourceStart, destinationStart, length); return; } // Stereo 4 to 5.1 (4 -> 6) if (numberOfSourceChannels == 4 && numberOfChannels == 6) { - getChannelByType(ChannelLeft)->sum(source.getChannelByType(ChannelLeft), sourceStart, destinationStart, length); - getChannelByType(ChannelRight)->sum(source.getChannelByType(ChannelRight), sourceStart, destinationStart, length); - getChannelByType(ChannelSurroundLeft)->sum(source.getChannelByType(ChannelSurroundLeft), sourceStart, destinationStart, length); - getChannelByType(ChannelSurroundRight)->sum(source.getChannelByType(ChannelSurroundRight), sourceStart, destinationStart, length); + getChannelByType(ChannelLeft)->sum(source->getChannelByType(ChannelLeft), sourceStart, destinationStart, length); + getChannelByType(ChannelRight)->sum(source->getChannelByType(ChannelRight), sourceStart, destinationStart, length); + getChannelByType(ChannelSurroundLeft)->sum(source->getChannelByType(ChannelSurroundLeft), sourceStart, destinationStart, length); + getChannelByType(ChannelSurroundRight)->sum(source->getChannelByType(ChannelSurroundRight), sourceStart, destinationStart, length); return; } discreteSum(source, sourceStart, destinationStart, length); } -void AudioBus::sumByDownMixing(const AudioBus &source, int sourceStart, int destinationStart, int length) { - int numberOfSourceChannels = source.getNumberOfChannels(); +void AudioBus::sumByDownMixing(const AudioBus *source, int sourceStart, int destinationStart, int length) { + int numberOfSourceChannels = source->getNumberOfChannels(); int numberOfChannels = getNumberOfChannels(); // Stereo to mono (2 -> 1): output += 0.5 * (input.left + input.right). if (numberOfSourceChannels == 2 && numberOfChannels == 1) { - float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); - float* sourceRight = source.getChannelByType(ChannelRight)->getData(); + float* sourceLeft = source->getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source->getChannelByType(ChannelRight)->getData(); float* destinationData = getChannelByType(ChannelMono)->getData(); @@ -283,10 +286,10 @@ void AudioBus::sumByDownMixing(const AudioBus &source, int sourceStart, int dest // Stereo 4 to mono: output += 0.25 * (input.left + input.right + input.surroundLeft + input.surroundRight) if (numberOfSourceChannels == 4 && numberOfChannels == 1) { - float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); - float* sourceRight = source.getChannelByType(ChannelRight)->getData(); - float* sourceSurroundLeft = source.getChannelByType(ChannelSurroundLeft)->getData(); - float* sourceSurroundRight = source.getChannelByType(ChannelSurroundRight)->getData(); + float* sourceLeft = source->getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source->getChannelByType(ChannelRight)->getData(); + float* sourceSurroundLeft = source->getChannelByType(ChannelSurroundLeft)->getData(); + float* sourceSurroundRight = source->getChannelByType(ChannelSurroundRight)->getData(); float* destinationData = getChannelByType(ChannelMono)->getData(); @@ -301,11 +304,11 @@ void AudioBus::sumByDownMixing(const AudioBus &source, int sourceStart, int dest // output.left += input.left + sqrt(1/2) * (input.center + input.surroundLeft) // output.right += input.right + sqrt(1/2) * (input.center + input.surroundRight) if (numberOfSourceChannels == 6 && numberOfChannels == 2) { - float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); - float* sourceRight = source.getChannelByType(ChannelRight)->getData(); - float* sourceCenter = source.getChannelByType(ChannelCenter)->getData(); - float* sourceSurroundLeft = source.getChannelByType(ChannelSurroundLeft)->getData(); - float* sourceSurroundRight = source.getChannelByType(ChannelSurroundRight)->getData(); + float* sourceLeft = source->getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source->getChannelByType(ChannelRight)->getData(); + float* sourceCenter = source->getChannelByType(ChannelCenter)->getData(); + float* sourceSurroundLeft = source->getChannelByType(ChannelSurroundLeft)->getData(); + float* sourceSurroundRight = source->getChannelByType(ChannelSurroundRight)->getData(); float* destinationLeft = getChannelByType(ChannelLeft)->getData(); float* destinationRight = getChannelByType(ChannelRight)->getData(); @@ -326,11 +329,11 @@ void AudioBus::sumByDownMixing(const AudioBus &source, int sourceStart, int dest // output.surroundLeft += input.surroundLeft // output.surroundRight += input.surroundRight if (numberOfSourceChannels == 6 && numberOfChannels == 4) { - float* sourceLeft = source.getChannelByType(ChannelLeft)->getData(); - float* sourceRight = source.getChannelByType(ChannelRight)->getData(); - float* sourceCenter = source.getChannelByType(ChannelCenter)->getData(); - float* sourceSurroundLeft = source.getChannelByType(ChannelSurroundLeft)->getData(); - float* sourceSurroundRight = source.getChannelByType(ChannelSurroundRight)->getData(); + float* sourceLeft = source->getChannelByType(ChannelLeft)->getData(); + float* sourceRight = source->getChannelByType(ChannelRight)->getData(); + float* sourceCenter = source->getChannelByType(ChannelCenter)->getData(); + float* sourceSurroundLeft = source->getChannelByType(ChannelSurroundLeft)->getData(); + float* sourceSurroundRight = source->getChannelByType(ChannelSurroundRight)->getData(); float* destinationLeft = getChannelByType(ChannelLeft)->getData(); float* destinationRight = getChannelByType(ChannelRight)->getData(); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.h b/packages/react-native-audio-api/common/cpp/core/AudioBus.h index 40504f09..b54dbf60 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.h @@ -39,25 +39,25 @@ class AudioBus { void zero(); void zero(int start, int length); - void sum(const AudioBus &source); - void sum(const AudioBus &source, int start, int length); - void sum(const AudioBus &source, int sourceStart, int destinationStart, int length); + void sum(const AudioBus *source); + void sum(const AudioBus *source, int start, int length); + void sum(const AudioBus *source, int sourceStart, int destinationStart, int length); - void copy(const AudioBus &source); - void copy(const AudioBus &source, int start, int length); - void copy(const AudioBus &source, int sourceStart, int destinationStart, int length); + void copy(const AudioBus *source); + void copy(const AudioBus *source, int start, int length); + void copy(const AudioBus *source, int sourceStart, int destinationStart, int length); private: - std::vector channels_; + std::vector> channels_; int numberOfChannels_; int sampleRate_; int size_; void createChannels(); - void discreteSum(const AudioBus &source, int sourceStart, int destinationStart, int length); - void sumByUpMixing(const AudioBus &source, int sourceStart, int destinationStart, int length); - void sumByDownMixing(const AudioBus &source, int sourceStart, int destinationStart, int length); + void discreteSum(const AudioBus *source, int sourceStart, int destinationStart, int length); + void sumByUpMixing(const AudioBus *source, int sourceStart, int destinationStart, int length); + void sumByDownMixing(const AudioBus *source, int sourceStart, int destinationStart, int length); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 39523640..be98aff6 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -8,6 +8,7 @@ namespace audioapi { AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) : AudioNode(context), currentSampleFrame_(0) { + printf("AudioDestinationNode::AudioDestinationNode\n"); numberOfOutputs_ = 0; numberOfInputs_ = INT_MAX; channelCountMode_ = ChannelCountMode::EXPLICIT; @@ -28,7 +29,14 @@ void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFram return; } - processAudio(destinationBus, numFrames); + AudioBus* processedBus = processAudio(destinationBus, numFrames); + + if (processedBus != destinationBus) { + destinationBus->copy(processedBus); + } + + destinationBus->normalize(); + currentSampleFrame_ += numFrames; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 71f969a5..cf269912 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -5,10 +5,12 @@ namespace audioapi { AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { - audioBus_ = std::make_unique(context->getSampleRate(), context->getBufferSizeInFrames(), channelCount_); + printf("AudioNode::AudioNode %d %d\n", context->getSampleRate(), context->getBufferSizeInFrames()); + audioBus_ = std::make_shared(context->getSampleRate(), context->getBufferSizeInFrames(), channelCount_); } AudioNode::~AudioNode() { + printf("AudioNode::~AudioNode\n"); cleanup(); } @@ -85,7 +87,7 @@ AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { // - outputBus is not provided, which means that next node is doing a multi-node summing. // - it has more than one input, which means that it has to sum all inputs using internal bus. // - it has more than one output, so each output node can get the processed data without re-calculating the node. - bool canUseOutputBus = outputBus != 0 && inputNodes_.size() == 1 && outputNodes_.size() == 1; + bool canUseOutputBus = outputBus != 0 && inputNodes_.size() < 2 && outputNodes_.size() < 2; if (isAlreadyProcessed) { // If it was already processed in the rendering quantum,return it. @@ -109,21 +111,15 @@ AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { return processingBus; } - // Process first connected node, it can be directly connected to the processingBus, - // resulting in one less summing operation in most cases. - AudioBus* firstNodeResultBus = inputNodes_.front()->processAudio(processingBus, framesToProcess); - - if (firstNodeResultBus != processingBus) { - processingBus->sum(*firstNodeResultBus); - } - - if (inputNodes_.size() > 1) { - // If there are more connected nodes, sum them together. - for (auto it = inputNodes_.begin() + 1; it != inputNodes_.end(); it += 1) { + for (auto it = inputNodes_.begin(); it != inputNodes_.end(); it += 1) { + // Process first connected node, it can be directly connected to the processingBus, + // resulting in one less summing operation.q + if (it == inputNodes_.begin()) { + it->get()->processAudio(processingBus, framesToProcess); + } else { // Enforce the summing to be done using the internal bus. AudioBus* inputBus = it->get()->processAudio(0, framesToProcess); - - processingBus->sum(*inputBus); + processingBus->sum(inputBus); } } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index 9de81879..4b4acf4b 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -30,7 +30,7 @@ class AudioNode : public std::enable_shared_from_this { static std::string toString(ChannelInterpretation interpretation); BaseAudioContext *context_; - std::unique_ptr audioBus_; + std::shared_ptr audioBus_; int numberOfInputs_ = 1; int numberOfOutputs_ = 1; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp index 42d0fc9a..e1d4861d 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp @@ -5,21 +5,36 @@ namespace audioapi { AudioScheduledSourceNode::AudioScheduledSourceNode(BaseAudioContext *context) : AudioNode(context), isPlaying_(false) { + printf("AudioScheduledSourceNode::AudioScheduledSourceNode\n"); numberOfInputs_ = 0; } void AudioScheduledSourceNode::start(double time) { - nextChangeTime_ = time; + waitAndExecute(time, [this](double time) { startPlayback(); }); } void AudioScheduledSourceNode::stop(double time) { - nextChangeTime_ = time; + waitAndExecute(time, [this](double time) { stopPlayback(); }); } -void AudioScheduledSourceNode::handlePlayback(double time, int framesToProcess) { - if (nextChangeTime_ <= time + (framesToProcess / context_->getSampleRate())) { - isPlaying_ = !isPlaying_; - } +void AudioScheduledSourceNode::startPlayback() { + isPlaying_ = true; +} + +void AudioScheduledSourceNode::stopPlayback() { + isPlaying_ = false; +} + +void AudioScheduledSourceNode::waitAndExecute( + double time, + const std::function &fun) { + std::thread([this, time, fun]() { + while (context_->getCurrentTime() < time) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + fun(time); + }).detach(); } } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h index 472e3234..85691aef 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h @@ -1,6 +1,11 @@ #pragma once #include +#include +#include +#include +#include +#include #include "AudioNode.h" @@ -15,9 +20,11 @@ class AudioScheduledSourceNode : public AudioNode { protected: std::atomic isPlaying_; - std::atomic nextChangeTime_ { -1.0 }; - void handlePlayback(double time, int framesToProcess); + private: + void startPlayback(); + void stopPlayback(); + void waitAndExecute(double time, const std::function &fun); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp index 87c6f882..d9ef5336 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp @@ -24,10 +24,11 @@ BaseAudioContext::BaseAudioContext() { #else audioPlayer_ = std::make_shared(this->renderAudio()); #endif - destination_ = std::make_shared(this); sampleRate_ = audioPlayer_->getSampleRate(); + bufferSizeInFrames_ = audioPlayer_->getBufferSizeInFrames(); + destination_ = std::make_shared(this); audioPlayer_->start(); } diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index 89190637..f65ad0aa 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -7,6 +7,7 @@ namespace audioapi { OscillatorNode::OscillatorNode(BaseAudioContext *context) : AudioScheduledSourceNode(context) { + printf("OscillatorNode::OscillatorNode\n"); frequencyParam_ = std::make_shared( context, 444.0, -NYQUIST_FREQUENCY, NYQUIST_FREQUENCY); detuneParam_ = @@ -31,7 +32,6 @@ void OscillatorNode::setType(const std::string &type) { void OscillatorNode::processNode(AudioBus* processingBus, int framesToProcess) { double time = context_->getCurrentTime(); - handlePlayback(time, framesToProcess); if (!isPlaying_) { processingBus->zero(); @@ -51,8 +51,7 @@ void OscillatorNode::processNode(AudioBus* processingBus, int framesToProcess) { float value = OscillatorNode::getWaveBufferElement(phase_, type_); for (int j = 0; j < channelCount_; j += 1) { - // Call the [] operator directly for better readability ;) - processingBus->getChannel(j)->operator[](i) = value; + (*processingBus->getChannel(j))[i] = value; } phase_ += phaseIncrement; diff --git a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm index ba3e68a4..7c40ab66 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/AudioPlayer/IOSAudioPlayer.mm @@ -21,14 +21,13 @@ audioPlayer_ = [[AudioPlayer alloc] initWithRenderAudioBlock:renderAudioBlock]; audioBus_ = new AudioBus(getSampleRate(), getBufferSizeInFrames(), CHANNEL_COUNT); - NSLog(@"info: %d %d", getSampleRate(), getBufferSizeInFrames()); } IOSAudioPlayer::~IOSAudioPlayer() { stop(); [audioPlayer_ cleanup]; - + if (audioBus_) { delete audioBus_; audioBus_ = 0; From 3822884320f37f9ba92bde3977557e35685baa0d Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 19 Nov 2024 20:57:53 +0100 Subject: [PATCH 17/33] fix: cleanup logs --- apps/common-app/src/examples/Oscillator/Oscillator.tsx | 1 + packages/react-native-audio-api/common/cpp/core/AudioArray.cpp | 1 - .../common/cpp/core/AudioDestinationNode.cpp | 1 - packages/react-native-audio-api/common/cpp/core/AudioNode.cpp | 2 -- .../common/cpp/core/AudioScheduledSourceNode.cpp | 1 - .../react-native-audio-api/common/cpp/core/OscillatorNode.cpp | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/common-app/src/examples/Oscillator/Oscillator.tsx b/apps/common-app/src/examples/Oscillator/Oscillator.tsx index b4509714..c591257a 100644 --- a/apps/common-app/src/examples/Oscillator/Oscillator.tsx +++ b/apps/common-app/src/examples/Oscillator/Oscillator.tsx @@ -88,6 +88,7 @@ const Oscillator: FC = () => { const handlePlayPause = () => { if (isPlaying) { oscillatorRef.current?.stop(0); + oscillatorRef.current = null; } else { setup(); oscillatorRef.current?.start(0); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp index e3b606dd..439b9f04 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioArray.cpp @@ -6,7 +6,6 @@ namespace audioapi { AudioArray::AudioArray(int size) : size_(size), data_(0) { - printf("AudioArray::AudioArray(%d)\n", size); resize(size); } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index be98aff6..2df933d2 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -8,7 +8,6 @@ namespace audioapi { AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) : AudioNode(context), currentSampleFrame_(0) { - printf("AudioDestinationNode::AudioDestinationNode\n"); numberOfOutputs_ = 0; numberOfInputs_ = INT_MAX; channelCountMode_ = ChannelCountMode::EXPLICIT; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index cf269912..89406459 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -5,12 +5,10 @@ namespace audioapi { AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { - printf("AudioNode::AudioNode %d %d\n", context->getSampleRate(), context->getBufferSizeInFrames()); audioBus_ = std::make_shared(context->getSampleRate(), context->getBufferSizeInFrames(), channelCount_); } AudioNode::~AudioNode() { - printf("AudioNode::~AudioNode\n"); cleanup(); } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp index e1d4861d..c177cab5 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp @@ -5,7 +5,6 @@ namespace audioapi { AudioScheduledSourceNode::AudioScheduledSourceNode(BaseAudioContext *context) : AudioNode(context), isPlaying_(false) { - printf("AudioScheduledSourceNode::AudioScheduledSourceNode\n"); numberOfInputs_ = 0; } diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index f65ad0aa..e74180d4 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -7,7 +7,6 @@ namespace audioapi { OscillatorNode::OscillatorNode(BaseAudioContext *context) : AudioScheduledSourceNode(context) { - printf("OscillatorNode::OscillatorNode\n"); frequencyParam_ = std::make_shared( context, 444.0, -NYQUIST_FREQUENCY, NYQUIST_FREQUENCY); detuneParam_ = From ea951c98b215745224ce6dbae560b680c35fab85 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Wed, 20 Nov 2024 11:29:01 +0100 Subject: [PATCH 18/33] feat: update translations --- packages/react-native-audio-api/android/CMakeLists.txt | 5 ----- packages/react-native-audio-api/ios/AudioAPIModule.h | 4 ++++ .../ios/AudioPlayer/AudioPlayer.m | 10 +++++++++- .../src/core/AudioScheduledSourceNode.ts | 1 - 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/react-native-audio-api/android/CMakeLists.txt b/packages/react-native-audio-api/android/CMakeLists.txt index b91d27fe..9e779096 100644 --- a/packages/react-native-audio-api/android/CMakeLists.txt +++ b/packages/react-native-audio-api/android/CMakeLists.txt @@ -4,11 +4,6 @@ project(react-native-audio-api) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_CXX_STANDARD 20) -# Detect the operating system -if(APPLE) - set(HAVE_ACCELERATE TRUE) -endif() - # Detect the processor and SIMD support if(CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") set(HAVE_ARM_NEON_INTRINSICS TRUE) diff --git a/packages/react-native-audio-api/ios/AudioAPIModule.h b/packages/react-native-audio-api/ios/AudioAPIModule.h index 61a47468..7760a435 100644 --- a/packages/react-native-audio-api/ios/AudioAPIModule.h +++ b/packages/react-native-audio-api/ios/AudioAPIModule.h @@ -8,6 +8,10 @@ #import #endif // RCT_NEW_ARCH_ENABLED +#ifndef HAVE_ACCELERATE +#define HAVE_ACCELERATE +#endif + #import @interface AudioAPIModule : RCTEventEmitter diff --git a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m index bb79ad81..9042163c 100644 --- a/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m +++ b/packages/react-native-audio-api/ios/AudioPlayer/AudioPlayer.m @@ -54,7 +54,15 @@ - (int)getSampleRate - (int)getBufferSizeInFrames { - return (int)(self.audioSession.IOBufferDuration * self.audioSession.sampleRate); + // Note: might be important in the future. + // For some reason audioSession.IOBufferDuration is always 0.01, which for sample rate of 48k + // gives exactly 480 frames, while at the same time frameCount requested by AVAudioSourceEngine + // might vary f.e. between 555-560. + // preferredIOBufferDuration seems to be double the value (resulting in 960 frames), + // which is safer to base our internal AudioBus sizes. + // Buut no documentation => no guarantee :) + // If something is crackling when it should play silence, start here 📻 + return (int)(self.audioSession.preferredIOBufferDuration * self.audioSession.sampleRate); } - (void)start diff --git a/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts b/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts index eee8dd85..7c1051c0 100644 --- a/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts +++ b/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts @@ -23,7 +23,6 @@ export default class AudioScheduledSourceNode extends AudioNode { this.hasBeenStarted = true; (this.node as IAudioScheduledSourceNode).start(when); - this.hasBeenStarted = true; } public stop(when: number = 0): void { From 90ac5adb1b277d13e0f60b944cdb9d43878c706a Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Wed, 20 Nov 2024 11:42:25 +0100 Subject: [PATCH 19/33] fix: uncomment all oscilator example params --- .../src/examples/Oscillator/Oscillator.tsx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/common-app/src/examples/Oscillator/Oscillator.tsx b/apps/common-app/src/examples/Oscillator/Oscillator.tsx index c591257a..ec384ffb 100644 --- a/apps/common-app/src/examples/Oscillator/Oscillator.tsx +++ b/apps/common-app/src/examples/Oscillator/Oscillator.tsx @@ -42,15 +42,16 @@ const Oscillator: FC = () => { oscillatorRef.current.frequency.value = frequency; oscillatorRef.current.detune.value = detune; oscillatorRef.current.type = oscillatorType; - oscillatorRef.current.connect(audioContextRef.current.destination); - - // gainRef.current = audioContextRef.current.createGain(); - // gainRef.current.gain.value = gain; - // panRef.current = audioContextRef.current.createStereoPanner(); - // panRef.current.pan.value = pan; - // oscillatorRef.current.connect(gainRef.current); - // gainRef.current.connect(panRef.current); - // panRef.current.connect(audioContextRef.current.destination); + + gainRef.current = audioContextRef.current.createGain(); + gainRef.current.gain.value = gain; + + panRef.current = audioContextRef.current.createStereoPanner(); + panRef.current.pan.value = pan; + + oscillatorRef.current.connect(gainRef.current); + gainRef.current.connect(panRef.current); + panRef.current.connect(audioContextRef.current.destination); }; const handleGainChange = (newValue: number) => { From fb4c245afbc5f9b067ae5accaff531529e01580e Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Wed, 20 Nov 2024 12:50:16 +0100 Subject: [PATCH 20/33] fix: more fixes --- .../src/examples/DrumMachine/DrumMachine.tsx | 4 +- .../src/examples/DrumMachine/usePlayer.tsx | 8 +-- .../common/cpp/core/AudioBufferSourceNode.cpp | 2 + .../common/cpp/core/AudioBus.cpp | 12 ++--- .../common/cpp/core/AudioDestinationNode.cpp | 8 ++- .../common/cpp/core/AudioNode.cpp | 12 ++++- .../common/cpp/core/AudioNode.h | 2 + .../common/cpp/core/BiquadFilterNode.cpp | 49 ++++++++++--------- .../common/cpp/core/GainNode.cpp | 2 + .../common/cpp/core/OscillatorNode.cpp | 2 + .../common/cpp/core/StereoPannerNode.cpp | 2 + 11 files changed, 64 insertions(+), 39 deletions(-) diff --git a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx index e179a73f..67bbf041 100644 --- a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx +++ b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx @@ -22,7 +22,7 @@ const defaultPreset = 'Empty'; function setupPlayer(audioCtx: AudioContext) { const kick = new Kick(audioCtx); - const clap = new Clap(audioCtx); + // const clap = new Clap(audioCtx); const hiHat = new HiHat(audioCtx); const playNote = (name: InstrumentName, time: number) => { @@ -31,7 +31,7 @@ function setupPlayer(audioCtx: AudioContext) { kick.play(time); break; case 'clap': - clap.play(time); + // clap.play(time); break; case 'hi-hat': hiHat.play(time); diff --git a/apps/common-app/src/examples/DrumMachine/usePlayer.tsx b/apps/common-app/src/examples/DrumMachine/usePlayer.tsx index 47d75777..597809f9 100644 --- a/apps/common-app/src/examples/DrumMachine/usePlayer.tsx +++ b/apps/common-app/src/examples/DrumMachine/usePlayer.tsx @@ -133,10 +133,10 @@ export default function usePlayer(options: PlayerOptions) { playingInstruments.value = getPlayingInstruments(); } - return () => { - console.log('Closing audio context'); - audioContext.close(); - }; + // return () => { + // console.log('Closing audio context'); + // audioContext.close(); + // }; // \/ Shared values are not necessary in deps array // eslint-disable-next-line react-hooks/exhaustive-deps }, [isPlaying, setup]); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index 3aed052f..0e59b888 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -9,6 +9,8 @@ AudioBufferSourceNode::AudioBufferSourceNode(BaseAudioContext *context) : AudioScheduledSourceNode(context), loop_(false), bufferIndex_(0) { numberOfInputs_ = 0; buffer_ = std::shared_ptr(nullptr); + debugName_ = "AudioBufferSourceNode"; + isInitialized_ = true; } bool AudioBufferSourceNode::getLoop() const { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index a6386aaf..5e6c3cdb 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -189,13 +189,13 @@ void AudioBus::copy(const AudioBus *source, int sourceStart, int destinationStar return; } - // if (source->getNumberOfChannels() == getNumberOfChannels()) { - // for (int i = 0; i < getNumberOfChannels(); i += 1) { - // getChannel(i)->copy(source->getChannel(i), sourceStart, destinationStart, length); - // } + if (source->getNumberOfChannels() == getNumberOfChannels()) { + for (int i = 0; i < getNumberOfChannels(); i += 1) { + getChannel(i)->copy(source->getChannel(i), sourceStart, destinationStart, length); + } - // return; - // } + return; + } // zero + sum is equivalent to copy, but takes care of up/down-mixing. zero(destinationStart, length); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 2df933d2..391bb4f3 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -11,6 +11,8 @@ AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) numberOfOutputs_ = 0; numberOfInputs_ = INT_MAX; channelCountMode_ = ChannelCountMode::EXPLICIT; + debugName_ = "AudioDestinationNode"; + isInitialized_ = true; } std::size_t AudioDestinationNode::getCurrentSampleFrame() const { @@ -22,6 +24,10 @@ double AudioDestinationNode::getCurrentTime() const { } void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFrames) { + if (!isInitialized_) { + return; + } + destinationBus->zero(); if (!numFrames) { @@ -30,7 +36,7 @@ void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFram AudioBus* processedBus = processAudio(destinationBus, numFrames); - if (processedBus != destinationBus) { + if (processedBus && processedBus != destinationBus) { destinationBus->copy(processedBus); } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 89406459..168dc147 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -9,6 +9,7 @@ AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { } AudioNode::~AudioNode() { + isInitialized_ = false; cleanup(); } @@ -76,6 +77,11 @@ std::string AudioNode::toString(ChannelInterpretation interpretation) { } AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { + debugName_; + if (!isInitialized_) { + return outputBus; + } + std::size_t currentSampleFrame = context_->getCurrentSampleFrame(); // check if the node has already been processed for this rendering quantum @@ -114,10 +120,12 @@ AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { // resulting in one less summing operation.q if (it == inputNodes_.begin()) { it->get()->processAudio(processingBus, framesToProcess); - } else { + } else if (it->get()) { // Enforce the summing to be done using the internal bus. AudioBus* inputBus = it->get()->processAudio(0, framesToProcess); - processingBus->sum(inputBus); + if (inputBus) { + processingBus->sum(inputBus); + } } } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index 4b4acf4b..a1c645d1 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -35,6 +35,8 @@ class AudioNode : public std::enable_shared_from_this { int numberOfInputs_ = 1; int numberOfOutputs_ = 1; int channelCount_ = CHANNEL_COUNT; + std::string debugName_; + bool isInitialized_ = false; std::size_t lastRenderedFrame_ { SIZE_MAX }; diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index ed8f162e..7b92afcb 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -19,6 +19,8 @@ BiquadFilterNode::BiquadFilterNode(BaseAudioContext *context) gainParam_ = std::make_shared( context, 0.0, MIN_FILTER_GAIN, MAX_FILTER_GAIN); type_ = FilterType::LOWPASS; + debugName_ = "BiquadFilterNode"; + isInitialized_ = true; } std::string BiquadFilterNode::getType() const { @@ -354,35 +356,34 @@ void BiquadFilterNode::applyFilter() { } void BiquadFilterNode::processNode(AudioBus* processingBus, int framesToProcess) { + printf("BiquadFilterNode::processNode\n"); resetCoefficients(); applyFilter(); + float x1 = x1_; + float x2 = x2_; + float y1 = y1_; + float y2 = y2_; + + float b0 = b0_; + float b1 = b1_; + float b2 = b2_; + float a1 = a1_; + float a2 = a2_; + for (int i = 0; i < processingBus->getNumberOfChannels(); i += 1) { - AudioArray* inputChannel = processingBus->getChannel(i); - - // Reset the coefficients for each channel - float x1 = x1_; - float x2 = x2_; - float y1 = y1_; - float y2 = y2_; - - float b0 = b0_; - float b1 = b1_; - float b2 = b2_; - float a1 = a1_; - float a2 = a2_; - - for (int f = 0; i < framesToProcess; f += 1) { - float input = (*inputChannel)[f]; - float output = b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; - - (*inputChannel)[f] = output; - - x2 = x1; - x1 = input; - y2 = y1; - y1 = output; + float input = (*processingBus->getChannel(0))[i]; + float output = b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; + + for (int j = 0; j < channelCount_; j += 1) { + (*processingBus->getChannel(j))[i] = output; } + + + x2 = x1; + x1 = input; + y2 = y1; + y1 = output; } } diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index 88bc6f50..b4c16594 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -7,6 +7,8 @@ namespace audioapi { GainNode::GainNode(BaseAudioContext *context) : AudioNode(context) { gainParam_ = std::make_shared(context, 1.0, -MAX_GAIN, MAX_GAIN); + debugName_ = "GainNode"; + isInitialized_ = true; } std::shared_ptr GainNode::getGainParam() const { diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index e74180d4..99ca68cb 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -11,6 +11,8 @@ OscillatorNode::OscillatorNode(BaseAudioContext *context) context, 444.0, -NYQUIST_FREQUENCY, NYQUIST_FREQUENCY); detuneParam_ = std::make_shared(context, 0.0, -MAX_DETUNE, MAX_DETUNE); + debugName_ = "OscillatorNode"; + isInitialized_ = true; } std::shared_ptr OscillatorNode::getFrequencyParam() const { diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index cad4bfb1..09d9fe89 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -12,6 +12,8 @@ StereoPannerNode::StereoPannerNode(BaseAudioContext *context) : AudioNode(context) { channelCountMode_ = ChannelCountMode::CLAMPED_MAX; panParam_ = std::make_shared(context, 0.0, -MAX_PAN, MAX_PAN); + debugName_ = "StereoPannerNode"; + isInitialized_ = true; } std::shared_ptr StereoPannerNode::getPanParam() const { From 8afa7b98bb5a4c93a200a46ee1a48db235367a78 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 22 Nov 2024 11:05:54 +0100 Subject: [PATCH 21/33] feat: locker + audio node manager wip --- .../src/examples/DrumMachine/DrumMachine.tsx | 4 +- .../common/cpp/core/AudioDestinationNode.cpp | 3 +- .../common/cpp/core/AudioNode.cpp | 20 ++++-- .../common/cpp/core/AudioNode.h | 9 ++- .../common/cpp/core/AudioNodeManager.cpp | 63 +++++++++++++++++++ .../common/cpp/core/AudioNodeManager.h | 36 +++++++++++ .../common/cpp/core/BaseAudioContext.cpp | 8 ++- .../common/cpp/core/BaseAudioContext.h | 4 ++ .../common/cpp/core/BiquadFilterNode.cpp | 3 +- .../common/cpp/utils/Locker.hpp | 47 ++++++++++++++ 10 files changed, 186 insertions(+), 11 deletions(-) create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp create mode 100644 packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h create mode 100644 packages/react-native-audio-api/common/cpp/utils/Locker.hpp diff --git a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx index 67bbf041..e179a73f 100644 --- a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx +++ b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx @@ -22,7 +22,7 @@ const defaultPreset = 'Empty'; function setupPlayer(audioCtx: AudioContext) { const kick = new Kick(audioCtx); - // const clap = new Clap(audioCtx); + const clap = new Clap(audioCtx); const hiHat = new HiHat(audioCtx); const playNote = (name: InstrumentName, time: number) => { @@ -31,7 +31,7 @@ function setupPlayer(audioCtx: AudioContext) { kick.play(time); break; case 'clap': - // clap.play(time); + clap.play(time); break; case 'hi-hat': hiHat.play(time); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 391bb4f3..c97f1d29 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -40,7 +40,8 @@ void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFram destinationBus->copy(processedBus); } - destinationBus->normalize(); + // destinationBus->normalize(); + destinationBus->zero(); currentSampleFrame_ += numFrames; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 168dc147..97e950b4 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -1,5 +1,6 @@ #include "AudioBus.h" #include "AudioNode.h" +#include "AudioNodeManager.h" #include "BaseAudioContext.h" namespace audioapi { @@ -34,11 +35,19 @@ std::string AudioNode::getChannelInterpretation() const { } void AudioNode::connect(const std::shared_ptr &node) { + context_->getNodeManager()->addPendingConnection(shared_from_this(), node, AudioNodeManager::ConnectionType::CONNECT); +} + +void AudioNode::connectNode(std::shared_ptr &node) { outputNodes_.push_back(node); node->inputNodes_.push_back(shared_from_this()); } void AudioNode::disconnect(const std::shared_ptr &node) { + context_->getNodeManager()->addPendingConnection(shared_from_this(), node, AudioNodeManager::ConnectionType::DISCONNECT); +} + +void AudioNode::disconnectNode(std::shared_ptr &node) { outputNodes_.erase( std::remove(outputNodes_.begin(), outputNodes_.end(), node), outputNodes_.end()); @@ -51,6 +60,10 @@ void AudioNode::disconnect(const std::shared_ptr &node) { } } +bool AudioNode::isInitialized() const { + return isInitialized_; +} + std::string AudioNode::toString(ChannelCountMode mode) { switch (mode) { @@ -77,7 +90,6 @@ std::string AudioNode::toString(ChannelInterpretation interpretation) { } AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { - debugName_; if (!isInitialized_) { return outputBus; } @@ -115,12 +127,12 @@ AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { return processingBus; } - for (auto it = inputNodes_.begin(); it != inputNodes_.end(); it += 1) { + for (auto it = inputNodes_.begin(); it != inputNodes_.end(); ++it) { // Process first connected node, it can be directly connected to the processingBus, - // resulting in one less summing operation.q + // resulting in one less summing operation. if (it == inputNodes_.begin()) { it->get()->processAudio(processingBus, framesToProcess); - } else if (it->get()) { + } else { // Enforce the summing to be done using the internal bus. AudioBus* inputBus = it->get()->processAudio(0, framesToProcess); if (inputBus) { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index a1c645d1..829377e2 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -3,6 +3,7 @@ #include #include #include +#include "AudioNode.h" #include "Constants.h" namespace audioapi { @@ -22,7 +23,11 @@ class AudioNode : public std::enable_shared_from_this { void connect(const std::shared_ptr &node); void disconnect(const std::shared_ptr &node); + bool isInitialized() const; + protected: + friend class AudioNodeManager; + enum class ChannelCountMode { MAX, CLAMPED_MAX, EXPLICIT }; enum class ChannelInterpretation { SPEAKERS, DISCRETE }; @@ -47,10 +52,12 @@ class AudioNode : public std::enable_shared_from_this { std::vector> inputNodes_ = {}; std::vector> outputNodes_ = {}; - void cleanup(); AudioBus* processAudio(AudioBus* outputBus, int framesToProcess); virtual void processNode(AudioBus* processingBus, int framesToProcess) = 0; + + void connectNode(std::shared_ptr &node); + void disconnectNode(std::shared_ptr &node); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp new file mode 100644 index 00000000..3977ef32 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp @@ -0,0 +1,63 @@ + +#include "Locker.hpp" +#include "AudioNode.h" +#include "AudioNodeManager.h" + +namespace audioapi { + +AudioNodeManager::AudioNodeManager() {} + +AudioNodeManager::~AudioNodeManager() { + audioNodesToConnect_.clear(); + audioNodesToDelete_.clear(); +} + +void AudioNodeManager::addPendingConnection(std::shared_ptr from, std::shared_ptr to, ConnectionType type) { + Locker lock(getGraphLock()); + + audioNodesToConnect_.push_back(std::make_tuple(from, to, type)); +} + +void AudioNodeManager::setNodeToDelete(std::shared_ptr node) { + Locker lock(getGraphLock()); + + audioNodesToDelete_.push_back(node); +} + +void AudioNodeManager::preProcessGraph() { + if (!Locker::tryLock(getGraphLock())) { + return; + } +} + +void AudioNodeManager::postProcessGraph() { + if (!Locker::tryLock(getGraphLock())) { + return; + } +} + +std::mutex& AudioNodeManager::getGraphLock() { + return graphLock_; +} + +void AudioNodeManager::settlePendingConnections() { + for (auto& connection : audioNodesToConnect_) { + std::shared_ptr from = std::get<0>(connection); + std::shared_ptr to = std::get<1>(connection); + ConnectionType type = std::get<2>(connection); + + if (type == ConnectionType::CONNECT) { + from->connectNode(to); + } else { + from->disconnectNode(to); + } + } + + audioNodesToConnect_.clear(); +} + +void AudioNodeManager::settlePendingDeletions() { + audioNodesToDelete_.clear(); +} + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h new file mode 100644 index 00000000..030aac16 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include + +namespace audioapi { + +class AudioNode; + +class AudioNodeManager { + public: + enum class ConnectionType { CONNECT, DISCONNECT }; + AudioNodeManager(); + ~AudioNodeManager(); + + void preProcessGraph(); + void postProcessGraph(); + void addPendingConnection(std::shared_ptr from, std::shared_ptr to, ConnectionType type); + + void setNodeToDelete(std::shared_ptr node); + + std::mutex& getGraphLock(); + + private: + std::mutex graphLock_; + + std::vector, std::shared_ptr, ConnectionType>> audioNodesToConnect_; + std::vector> audioNodesToDelete_; + + void settlePendingConnections(); + void settlePendingDeletions(); +}; + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp index d9ef5336..1f266bc8 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp @@ -13,6 +13,7 @@ #include "OscillatorNode.h" #include "StereoPannerNode.h" #include "BiquadFilterNode.h" +#include "AudioNodeManager.h" #include "AudioDestinationNode.h" #include "AudioBufferSourceNode.h" @@ -28,8 +29,9 @@ BaseAudioContext::BaseAudioContext() { sampleRate_ = audioPlayer_->getSampleRate(); bufferSizeInFrames_ = audioPlayer_->getBufferSizeInFrames(); - destination_ = std::make_shared(this); audioPlayer_->start(); + nodeManager_ = std::make_shared(); + destination_ = std::make_shared(this); } std::string BaseAudioContext::getState() { @@ -93,6 +95,10 @@ std::function BaseAudioContext::renderAudio() { }; } +AudioNodeManager* BaseAudioContext::getNodeManager() { + return nodeManager_.get(); +} + std::string BaseAudioContext::toString(State state) { switch (state) { case State::SUSPENDED: diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h index e8eec2d1..39d15dab 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h @@ -16,6 +16,7 @@ class StereoPannerNode; class BiquadFilterNode; class AudioDestinationNode; class AudioBufferSourceNode; +class AudioNodeManager; #ifdef ANDROID class AudioPlayer; @@ -41,6 +42,8 @@ class BaseAudioContext { static std::shared_ptr createBuffer(int numberOfChannels, int length, int sampleRate); std::function renderAudio(); + AudioNodeManager* getNodeManager(); + protected: enum class State { SUSPENDED, RUNNING, CLOSED }; static std::string toString(State state); @@ -55,6 +58,7 @@ class BaseAudioContext { State state_ = State::RUNNING; int sampleRate_; int bufferSizeInFrames_; + std::shared_ptr nodeManager_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index 7b92afcb..1e00960f 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -356,7 +356,6 @@ void BiquadFilterNode::applyFilter() { } void BiquadFilterNode::processNode(AudioBus* processingBus, int framesToProcess) { - printf("BiquadFilterNode::processNode\n"); resetCoefficients(); applyFilter(); @@ -371,7 +370,7 @@ void BiquadFilterNode::processNode(AudioBus* processingBus, int framesToProcess) float a1 = a1_; float a2 = a2_; - for (int i = 0; i < processingBus->getNumberOfChannels(); i += 1) { + for (int i = 0; i < framesToProcess; i += 1) { float input = (*processingBus->getChannel(0))[i]; float output = b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; diff --git a/packages/react-native-audio-api/common/cpp/utils/Locker.hpp b/packages/react-native-audio-api/common/cpp/utils/Locker.hpp new file mode 100644 index 00000000..2f13adc0 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/utils/Locker.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include + +namespace audioapi { + +// Small easy interface to manage locking +class Locker { + public: + Locker(): lockPtr_(0) {} + Locker(std::mutex& lockPtr): lockPtr_(&lockPtr) { + lock(); + } + + ~Locker() { + unlock(); + } + + explicit operator bool() const { return !!lockPtr_; } + + void lock() { + if (lockPtr_) { + lockPtr_->lock(); + } + } + + void unlock() { + if (lockPtr_) { + lockPtr_->unlock(); + } + } + + static Locker tryLock(std::mutex& lock) { + Locker result = Locker(); + + if (lock.try_lock()) { + result.lockPtr_ = &lock; + } + + return result; + } + + private: + std::mutex* lockPtr_; +}; + +} // namespace audioapi From 48d1c34cefd80259a3225b149c6d6bb050134028 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Fri, 22 Nov 2024 11:26:00 +0100 Subject: [PATCH 22/33] fix: remove silencer + add post and preproc --- .../common/cpp/core/AudioDestinationNode.cpp | 8 ++++++-- .../common/cpp/core/AudioNodeManager.cpp | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index c97f1d29..8d332207 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -1,6 +1,7 @@ #include "AudioBus.h" #include "AudioNode.h" #include "VectorMath.h" +#include "AudioNodeManager.h" #include "BaseAudioContext.h" #include "AudioDestinationNode.h" @@ -28,6 +29,8 @@ void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFram return; } + context_->getNodeManager()->preProcessGraph(); + destinationBus->zero(); if (!numFrames) { @@ -40,8 +43,9 @@ void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFram destinationBus->copy(processedBus); } - // destinationBus->normalize(); - destinationBus->zero(); + destinationBus->normalize(); + + context_->getNodeManager()->postProcessGraph(); currentSampleFrame_ += numFrames; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp index 3977ef32..a33b35df 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp @@ -28,12 +28,18 @@ void AudioNodeManager::preProcessGraph() { if (!Locker::tryLock(getGraphLock())) { return; } + + settlePendingConnections(); + settlePendingDeletions(); } void AudioNodeManager::postProcessGraph() { if (!Locker::tryLock(getGraphLock())) { return; } + + settlePendingConnections(); + settlePendingDeletions(); } std::mutex& AudioNodeManager::getGraphLock() { From f7e9da0efabcb5405befe1c7c69c1e8545639041 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Mon, 25 Nov 2024 10:21:14 +0100 Subject: [PATCH 23/33] feat: working on decontruction --- .../common/cpp/core/AudioBufferSourceNode.cpp | 1 - .../common/cpp/core/AudioDestinationNode.cpp | 1 - .../common/cpp/core/AudioNode.cpp | 19 ++++++++++ .../common/cpp/core/AudioNode.h | 36 ++++++++++++++++++- .../common/cpp/core/AudioNodeManager.cpp | 10 ++++-- .../common/cpp/core/AudioNodeManager.h | 6 ++-- .../cpp/core/AudioScheduledSourceNode.cpp | 1 + .../common/cpp/core/BiquadFilterNode.cpp | 1 - .../common/cpp/core/GainNode.cpp | 1 - .../common/cpp/core/OscillatorNode.cpp | 1 - .../common/cpp/core/StereoPannerNode.cpp | 1 - 11 files changed, 67 insertions(+), 11 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index 0e59b888..f766c380 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -9,7 +9,6 @@ AudioBufferSourceNode::AudioBufferSourceNode(BaseAudioContext *context) : AudioScheduledSourceNode(context), loop_(false), bufferIndex_(0) { numberOfInputs_ = 0; buffer_ = std::shared_ptr(nullptr); - debugName_ = "AudioBufferSourceNode"; isInitialized_ = true; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 8d332207..be56334f 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -12,7 +12,6 @@ AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) numberOfOutputs_ = 0; numberOfInputs_ = INT_MAX; channelCountMode_ = ChannelCountMode::EXPLICIT; - debugName_ = "AudioDestinationNode"; isInitialized_ = true; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 97e950b4..6921b9a8 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -41,6 +41,8 @@ void AudioNode::connect(const std::shared_ptr &node) { void AudioNode::connectNode(std::shared_ptr &node) { outputNodes_.push_back(node); node->inputNodes_.push_back(shared_from_this()); + numberOfConnections_ += 1; + node->numberOfConnections_ += 1; } void AudioNode::disconnect(const std::shared_ptr &node) { @@ -58,12 +60,25 @@ void AudioNode::disconnectNode(std::shared_ptr &node) { node->inputNodes_.begin(), node->inputNodes_.end(), sharedThis), node->inputNodes_.end()); } + numberOfConnections_ -= 1; + node->numberOfConnections_ -= 1; } bool AudioNode::isInitialized() const { return isInitialized_; } +bool AudioNode::isEnabled() const { + return isEnabled_; +} + +void AudioNode::enable() { + isEnabled_ = true; +} + +void AudioNode::disable() { + isEnabled_ = false; +} std::string AudioNode::toString(ChannelCountMode mode) { switch (mode) { @@ -128,6 +143,10 @@ AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { } for (auto it = inputNodes_.begin(); it != inputNodes_.end(); ++it) { + if (!(*it)->isEnabled()) { + continue; + } + // Process first connected node, it can be directly connected to the processingBus, // resulting in one less summing operation. if (it == inputNodes_.begin()) { diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index 829377e2..02227b18 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -25,6 +25,10 @@ class AudioNode : public std::enable_shared_from_this { bool isInitialized() const; + bool isEnabled() const; + void enable(); + void disable(); + protected: friend class AudioNodeManager; @@ -40,8 +44,9 @@ class AudioNode : public std::enable_shared_from_this { int numberOfInputs_ = 1; int numberOfOutputs_ = 1; int channelCount_ = CHANNEL_COUNT; - std::string debugName_; + int numberOfConnections_ = 0; bool isInitialized_ = false; + bool isEnabled_ = true; std::size_t lastRenderedFrame_ { SIZE_MAX }; @@ -61,3 +66,32 @@ class AudioNode : public std::enable_shared_from_this { }; } // namespace audioapi + +/* +Audio node management and deletion + +1. AudioSourceNode finishes playing, then: + - it disables itself (so it won't by processed anymore) + - if there is no reference from JS side, we can mark self for deletion + - node that is marked for deletion should be disconnected from the graph first + - it should "notify" connected nodes that it has been disabled (and potentially prepares for deletion) + +2. Node gets "notified" that one of its sources is disabled: + - it lowers the count of enabled sources + - if the count of enabled sources is 0, disable itself + - if there is no reference from JS side, we can mark self for deletion + - node that is marked for deletion should be disconnected from the graph first + - it should "notify" connected nodes that it has been disabled (and potentially prepares for deletion) + +Translating into more technical terms: +We use shared pointers for keeping output nodes +We use shared pointers in audio node manager to keep track of all source nodes +when audio source node finished playing it: + - disables itself and tells all output nodes that it has been disabled + - each node up to destination, checks their input nodes and if was its only active input node, it disables itself. + - source node tells audio node manager to dereference it (only if it is the last reference to the source node) + - audio manager in pre-process or post-process will remove the reference + - deletion of the node will dereference all connected nodes, resulting in destroy'ing them if they are not referenced from JS side + + +*/ diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp index a33b35df..d7cf226c 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp @@ -12,18 +12,24 @@ AudioNodeManager::~AudioNodeManager() { audioNodesToDelete_.clear(); } -void AudioNodeManager::addPendingConnection(std::shared_ptr from, std::shared_ptr to, ConnectionType type) { +void AudioNodeManager::addPendingConnection(const std::shared_ptr &from, const std::shared_ptr &to, ConnectionType type) { Locker lock(getGraphLock()); audioNodesToConnect_.push_back(std::make_tuple(from, to, type)); } -void AudioNodeManager::setNodeToDelete(std::shared_ptr node) { +void AudioNodeManager::setNodeToDelete(const std::shared_ptr &node) { Locker lock(getGraphLock()); audioNodesToDelete_.push_back(node); } +void AudioNodeManager::addSourceNode(const std::shared_ptr &node) { + Locker lock(getGraphLock()); + + sourceNodes_.push_back(node); +} + void AudioNodeManager::preProcessGraph() { if (!Locker::tryLock(getGraphLock())) { return; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h index 030aac16..3684094e 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h @@ -17,9 +17,10 @@ class AudioNodeManager { void preProcessGraph(); void postProcessGraph(); - void addPendingConnection(std::shared_ptr from, std::shared_ptr to, ConnectionType type); + void addPendingConnection(const std::shared_ptr &from, const std::shared_ptr &to, ConnectionType type); - void setNodeToDelete(std::shared_ptr node); + void setNodeToDelete(const std::shared_ptr &node); + void addSourceNode(const std::shared_ptr &node); std::mutex& getGraphLock(); @@ -28,6 +29,7 @@ class AudioNodeManager { std::vector, std::shared_ptr, ConnectionType>> audioNodesToConnect_; std::vector> audioNodesToDelete_; + std::vector> sourceNodes_; void settlePendingConnections(); void settlePendingDeletions(); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp index c177cab5..5a2e194b 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp @@ -1,4 +1,5 @@ #include "BaseAudioContext.h" +#include "AudioNodeManager.h" #include "AudioScheduledSourceNode.h" namespace audioapi { diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index 1e00960f..9b5aea07 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -19,7 +19,6 @@ BiquadFilterNode::BiquadFilterNode(BaseAudioContext *context) gainParam_ = std::make_shared( context, 0.0, MIN_FILTER_GAIN, MAX_FILTER_GAIN); type_ = FilterType::LOWPASS; - debugName_ = "BiquadFilterNode"; isInitialized_ = true; } diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index b4c16594..9e43ff14 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -7,7 +7,6 @@ namespace audioapi { GainNode::GainNode(BaseAudioContext *context) : AudioNode(context) { gainParam_ = std::make_shared(context, 1.0, -MAX_GAIN, MAX_GAIN); - debugName_ = "GainNode"; isInitialized_ = true; } diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index 99ca68cb..30750424 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -11,7 +11,6 @@ OscillatorNode::OscillatorNode(BaseAudioContext *context) context, 444.0, -NYQUIST_FREQUENCY, NYQUIST_FREQUENCY); detuneParam_ = std::make_shared(context, 0.0, -MAX_DETUNE, MAX_DETUNE); - debugName_ = "OscillatorNode"; isInitialized_ = true; } diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index 09d9fe89..e3210516 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -12,7 +12,6 @@ StereoPannerNode::StereoPannerNode(BaseAudioContext *context) : AudioNode(context) { channelCountMode_ = ChannelCountMode::CLAMPED_MAX; panParam_ = std::make_shared(context, 0.0, -MAX_PAN, MAX_PAN); - debugName_ = "StereoPannerNode"; isInitialized_ = true; } From cc1ad1128732427c520f864671052f50441f52e2 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Mon, 25 Nov 2024 21:09:00 +0100 Subject: [PATCH 24/33] feat: something is working --- .../src/examples/DrumMachine/DrumMachine.tsx | 8 +- .../src/examples/Oscillator/Oscillator.tsx | 1 - .../SharedUtils/soundEngines/HiHat.ts | 21 +++-- apps/fabric-example/ios/Podfile.lock | 4 +- .../common/cpp/core/AudioBufferSourceNode.cpp | 9 +- .../common/cpp/core/AudioDestinationNode.cpp | 2 + .../common/cpp/core/AudioNode.cpp | 84 +++++++++++++++---- .../common/cpp/core/AudioNode.h | 22 +++-- .../common/cpp/core/AudioNodeManager.cpp | 23 ++++- .../common/cpp/core/AudioNodeManager.h | 7 +- .../cpp/core/AudioScheduledSourceNode.cpp | 18 +++- .../cpp/core/AudioScheduledSourceNode.h | 7 +- .../common/cpp/core/BaseAudioContext.cpp | 19 ++--- .../common/cpp/core/BaseAudioContext.h | 16 ++-- .../common/cpp/core/BiquadFilterNode.cpp | 45 +++++----- .../common/cpp/core/OscillatorNode.cpp | 2 +- .../common/cpp/core/OscillatorNode.h | 4 +- 17 files changed, 195 insertions(+), 97 deletions(-) diff --git a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx index e179a73f..a6561085 100644 --- a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx +++ b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx @@ -22,7 +22,7 @@ const defaultPreset = 'Empty'; function setupPlayer(audioCtx: AudioContext) { const kick = new Kick(audioCtx); - const clap = new Clap(audioCtx); + // const clap = new Clap(audioCtx); const hiHat = new HiHat(audioCtx); const playNote = (name: InstrumentName, time: number) => { @@ -30,9 +30,9 @@ function setupPlayer(audioCtx: AudioContext) { case 'kick': kick.play(time); break; - case 'clap': - clap.play(time); - break; + // case 'clap': + // clap.play(time); + // break; case 'hi-hat': hiHat.play(time); break; diff --git a/apps/common-app/src/examples/Oscillator/Oscillator.tsx b/apps/common-app/src/examples/Oscillator/Oscillator.tsx index 5cd1565b..658c7ff2 100644 --- a/apps/common-app/src/examples/Oscillator/Oscillator.tsx +++ b/apps/common-app/src/examples/Oscillator/Oscillator.tsx @@ -89,7 +89,6 @@ const Oscillator: FC = () => { const handlePlayPause = () => { if (isPlaying) { oscillatorRef.current?.stop(0); - oscillatorRef.current = null; } else { setup(); oscillatorRef.current?.start(0); diff --git a/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts b/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts index 34e79efe..e8d4150b 100644 --- a/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts +++ b/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts @@ -26,9 +26,10 @@ class HiHat implements SoundEngine { const oscillator = this.audioContext.createOscillator(); oscillator.type = 'square'; oscillator.frequency.value = this.tone * ratio; + const bandpassFilter = this.audioContext.createBiquadFilter(); const highpassFilter = this.audioContext.createBiquadFilter(); - const gain = this.audioContext.createGain(); + // const gain = this.audioContext.createGain(); bandpassFilter.type = 'bandpass'; bandpassFilter.frequency.value = this.bandpassFilterFrequency; @@ -36,16 +37,18 @@ class HiHat implements SoundEngine { highpassFilter.type = 'highpass'; highpassFilter.frequency.value = this.highpassFilterFrequency; - gain.gain.setValueAtTime(0.0001, time); - gain.gain.exponentialRampToValueAtTime(this.volume, time + 0.02); - gain.gain.exponentialRampToValueAtTime(this.volume * 0.33, time + 0.03); - gain.gain.exponentialRampToValueAtTime(this.volume * 0.0001, time + 0.3); - gain.gain.setValueAtTime(0, time + 0.3 + 0.001); + // gain.gain.setValueAtTime(0.0001, time); + // gain.gain.exponentialRampToValueAtTime(this.volume, time + 0.02); + // gain.gain.exponentialRampToValueAtTime(this.volume * 0.33, time + 0.03); + // gain.gain.exponentialRampToValueAtTime(this.volume * 0.0001, time + 0.3); + // gain.gain.setValueAtTime(0, time + 0.3 + 0.001); oscillator.connect(bandpassFilter); - bandpassFilter.connect(highpassFilter); - highpassFilter.connect(gain); - gain.connect(this.audioContext.destination!); + // bandpassFilter.connect(highpassFilter); + // highpassFilter.connect(gain); + // gain.connect(this.audioContext.destination!); + + bandpassFilter.connect(this.audioContext.destination); oscillator.start(time); oscillator.stop(time + this.decay); }); diff --git a/apps/fabric-example/ios/Podfile.lock b/apps/fabric-example/ios/Podfile.lock index a9c40bae..96960d34 100644 --- a/apps/fabric-example/ios/Podfile.lock +++ b/apps/fabric-example/ios/Podfile.lock @@ -2063,8 +2063,8 @@ SPEC CHECKSUMS: RNReanimated: 77242c6d67416988a2fd9f5cf574bb3e60016362 RNScreens: e389d6a6a66a4f0d3662924ecae803073ccce8ec SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 - Yoga: f8ec45ce98bba1bc93dd28f2ee37215180e6d2b6 + Yoga: 1d66db49f38fd9e576a1d7c3b081e46ab4c28b9e PODFILE CHECKSUM: 75ad38075e71875257a2590065853ea6a608b897 -COCOAPODS: 1.15.2 +COCOAPODS: 1.16.2 diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index f766c380..c7c95c47 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -41,7 +41,7 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro double time = context_->getCurrentTime(); // No audio data to fill, zero the output and return. - if (!isPlaying_ || !buffer_ || buffer_->getLength() == 0) { + if (!isPlaying() || !buffer_ || buffer_->getLength() == 0) { processingBus->zero(); return; } @@ -51,7 +51,7 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro processingBus->copy(buffer_->bus_.get()); if (!loop_) { - isPlaying_ = false; + playbackState_ = PlaybackState::FINISHED; } return; @@ -70,7 +70,7 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro } if (bufferIndex_ + framesToCopy == buffer_->getLength() - 1) { - isPlaying_ = false; + playbackState_ = PlaybackState::FINISHED; bufferIndex_ = 0; } @@ -82,7 +82,8 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro // If we don't loop the buffer, copy it once and zero the remaining processing bus frames. processingBus->copy(buffer_->bus_.get()); processingBus->zero(buffer_->getLength(), framesToProcess - buffer_->getLength()); - isPlaying_ = false; + playbackState_ = PlaybackState::FINISHED; + return; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index be56334f..6f960479 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -24,6 +24,8 @@ double AudioDestinationNode::getCurrentTime() const { } void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFrames) { + printf("connected nodes: %d\n", inputNodes_.size()); + if (!isInitialized_) { return; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 53508ee0..0f15914a 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -42,9 +42,8 @@ void AudioNode::connect(const std::shared_ptr &node) { void AudioNode::connectNode(std::shared_ptr &node) { outputNodes_.push_back(node); - node->inputNodes_.push_back(shared_from_this()); - numberOfConnections_ += 1; - node->numberOfConnections_ += 1; + + node->onInputConnected(this); } void AudioNode::disconnect(const std::shared_ptr &node) { @@ -52,18 +51,13 @@ void AudioNode::disconnect(const std::shared_ptr &node) { } void AudioNode::disconnectNode(std::shared_ptr &node) { - outputNodes_.erase( - std::remove(outputNodes_.begin(), outputNodes_.end(), node), - outputNodes_.end()); + node->onInputDisconnected(this); + + auto position = std::find(outputNodes_.begin(), outputNodes_.end(), node); - if (auto sharedThis = shared_from_this()) { - node->inputNodes_.erase( - std::remove( - node->inputNodes_.begin(), node->inputNodes_.end(), sharedThis), - node->inputNodes_.end()); + if (position != outputNodes_.end()) { + outputNodes_.erase(position); } - numberOfConnections_ -= 1; - node->numberOfConnections_ -= 1; } bool AudioNode::isInitialized() const { @@ -76,10 +70,18 @@ bool AudioNode::isEnabled() const { void AudioNode::enable() { isEnabled_ = true; + + for (auto it = outputNodes_.begin(); it != outputNodes_.end(); ++it) { + it->get()->onInputEnabled(); + } } void AudioNode::disable() { isEnabled_ = false; + + for (auto it = outputNodes_.begin(); it != outputNodes_.end(); ++it) { + it->get()->onInputDisabled(); + } } std::string AudioNode::toString(ChannelCountMode mode) { @@ -123,7 +125,7 @@ AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { bool canUseOutputBus = outputBus != 0 && inputNodes_.size() < 2 && outputNodes_.size() < 2; if (isAlreadyProcessed) { - // If it was already processed in the rendering quantum,return it. + // If it was already processed in the rendering quantum, return it. return audioBus_.get(); } @@ -152,10 +154,14 @@ AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { // Process first connected node, it can be directly connected to the processingBus, // resulting in one less summing operation. if (it == inputNodes_.begin()) { - it->get()->processAudio(processingBus, framesToProcess); + AudioBus* inputBus = (*it)->processAudio(processingBus, framesToProcess); + + if (inputBus != processingBus) { + processingBus->sum(inputBus); + } } else { // Enforce the summing to be done using the internal bus. - AudioBus* inputBus = it->get()->processAudio(0, framesToProcess); + AudioBus* inputBus = (*it)->processAudio(0, framesToProcess); if (inputBus) { processingBus->sum(inputBus); } @@ -173,4 +179,50 @@ void AudioNode::cleanup() { inputNodes_.clear(); } +void AudioNode::onInputEnabled() { + numberOfEnabledInputNodes_ += 1; + + if (!isEnabled()) { + enable(); + } +} + +void AudioNode::onInputDisabled() { + numberOfEnabledInputNodes_ -= 1; + + if (isEnabled() && numberOfEnabledInputNodes_ == 0) { + disable(); + } +} + +void AudioNode::onInputConnected(AudioNode *node) { + inputNodes_.push_back(node); + + if (node->isEnabled()) { + onInputEnabled(); + } +} + +void AudioNode::onInputDisconnected(AudioNode *node) { + auto position = std::find(inputNodes_.begin(), inputNodes_.end(), node); + + if (position == inputNodes_.end()) { + return; + } + + inputNodes_.erase(position); + + if (inputNodes_.size() > 0 || outputNodes_.size() == 0) { + return; + } + + if (isEnabled()) { + node->onInputDisabled(); + } + + for (auto outputNode : outputNodes_) { + disconnectNode(outputNode); + } +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index a3991350..80434ede 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -38,10 +38,13 @@ class AudioNode : public std::enable_shared_from_this { BaseAudioContext *context_; std::shared_ptr audioBus_; + int channelCount_ = CHANNEL_COUNT; + int numberOfInputs_ = 1; int numberOfOutputs_ = 1; - int channelCount_ = CHANNEL_COUNT; - int numberOfConnections_ = 0; + int numberOfEnabledInputNodes_ = 0; + + bool isInitialized_ = false; bool isEnabled_ = true; @@ -51,7 +54,7 @@ class AudioNode : public std::enable_shared_from_this { ChannelInterpretation channelInterpretation_ = ChannelInterpretation::SPEAKERS; - std::vector> inputNodes_ = {}; + std::vector inputNodes_ = {}; std::vector> outputNodes_ = {}; private: @@ -64,6 +67,11 @@ class AudioNode : public std::enable_shared_from_this { void connectNode(std::shared_ptr &node); void disconnectNode(std::shared_ptr &node); + + void onInputEnabled(); + void onInputDisabled(); + void onInputConnected(AudioNode *node); + void onInputDisconnected(AudioNode *node); }; } // namespace audioapi @@ -90,9 +98,9 @@ We use shared pointers in audio node manager to keep track of all source nodes when audio source node finished playing it: - disables itself and tells all output nodes that it has been disabled - each node up to destination, checks their input nodes and if was its only active input node, it disables itself. - - source node tells audio node manager to dereference it (only if it is the last reference to the source node) - - audio manager in pre-process or post-process will remove the reference - - deletion of the node will dereference all connected nodes, resulting in destroy'ing them if they are not referenced from JS side - + - source node tells audio node manager to dereference it (only if it is the last reference to the source node). + - audio manager in pre-process or post-process will remove the reference. + - audio manager in pre-process or post-process will also have to check for source nodes with only one reference and delete them if already stopped. + - deletion of the node will dereference all connected nodes, resulting in destroy'ing them if they are not referenced from JS side. */ diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp index d7cf226c..aad8a632 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp @@ -35,8 +35,9 @@ void AudioNodeManager::preProcessGraph() { return; } - settlePendingConnections(); settlePendingDeletions(); + settlePendingConnections(); + removeFinishedSourceNodes(); } void AudioNodeManager::postProcessGraph() { @@ -44,8 +45,9 @@ void AudioNodeManager::postProcessGraph() { return; } - settlePendingConnections(); settlePendingDeletions(); + settlePendingConnections(); + removeFinishedSourceNodes(); } std::mutex& AudioNodeManager::getGraphLock() { @@ -72,4 +74,21 @@ void AudioNodeManager::settlePendingDeletions() { audioNodesToDelete_.clear(); } +void AudioNodeManager::removeFinishedSourceNodes() { + for (auto it = sourceNodes_.begin(); it != sourceNodes_.end();) { + auto currentNode = it->get(); + // Release the source node if use count is equal to 1 (this vector) + if (!currentNode->isEnabled() && it->use_count() == 1) { + + for (auto& outputNode : currentNode->outputNodes_) { + currentNode->disconnectNode(outputNode); + } + + it = sourceNodes_.erase(it); + } else { + ++it; + } + } +} + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h index 3684094e..35bc7a69 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h @@ -27,12 +27,13 @@ class AudioNodeManager { private: std::mutex graphLock_; - std::vector, std::shared_ptr, ConnectionType>> audioNodesToConnect_; - std::vector> audioNodesToDelete_; std::vector> sourceNodes_; + std::vector> audioNodesToDelete_; + std::vector, std::shared_ptr, ConnectionType>> audioNodesToConnect_; - void settlePendingConnections(); void settlePendingDeletions(); + void settlePendingConnections(); + void removeFinishedSourceNodes(); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp index 5a2e194b..c9b4275d 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp @@ -5,11 +5,14 @@ namespace audioapi { AudioScheduledSourceNode::AudioScheduledSourceNode(BaseAudioContext *context) - : AudioNode(context), isPlaying_(false) { + : AudioNode(context), playbackState_(PlaybackState::UNSCHEDULED) { numberOfInputs_ = 0; } void AudioScheduledSourceNode::start(double time) { + context_->getNodeManager()->addSourceNode(shared_from_this()); + + playbackState_ = PlaybackState::SCHEDULED; waitAndExecute(time, [this](double time) { startPlayback(); }); } @@ -17,12 +20,21 @@ void AudioScheduledSourceNode::stop(double time) { waitAndExecute(time, [this](double time) { stopPlayback(); }); } +bool AudioScheduledSourceNode::isPlaying() { + return playbackState_ == PlaybackState::PLAYING; +} + +bool AudioScheduledSourceNode::isFinished() { + return playbackState_ == PlaybackState::FINISHED; +} + void AudioScheduledSourceNode::startPlayback() { - isPlaying_ = true; + playbackState_ = PlaybackState::PLAYING; } void AudioScheduledSourceNode::stopPlayback() { - isPlaying_ = false; + playbackState_ = PlaybackState::FINISHED; + disable(); } void AudioScheduledSourceNode::waitAndExecute( diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h index 85691aef..028e945a 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h @@ -13,13 +13,18 @@ namespace audioapi { class AudioScheduledSourceNode : public AudioNode { public: + enum class PlaybackState { UNSCHEDULED, SCHEDULED, PLAYING, FINISHED }; explicit AudioScheduledSourceNode(BaseAudioContext *context); void start(double time); void stop(double time); + bool isFinished(); + bool isPlaying(); + protected: - std::atomic isPlaying_; + std::atomic playbackState_; + private: void startPlayback(); diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp index 969b8f69..367372bf 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.cpp @@ -10,6 +10,7 @@ #include "AudioBus.h" #include "AudioArray.h" #include "AudioBuffer.h" +#include "ContextState.h" #include "OscillatorNode.h" #include "StereoPannerNode.h" #include "BiquadFilterNode.h" @@ -85,12 +86,6 @@ std::shared_ptr BaseAudioContext::createBuffer( return std::make_shared(numberOfChannels, length, sampleRate); } -std::function BaseAudioContext::renderAudio() { - if (state_ == State::CLOSED) { - return [](AudioBus*, int) {}; - } -} - std::shared_ptr BaseAudioContext::createPeriodicWave( float *real, float *imag, @@ -101,9 +96,9 @@ std::shared_ptr BaseAudioContext::createPeriodicWave( sampleRate_, real, imag, length, disableNormalization); } -std::function BaseAudioContext::renderAudio() { +std::function BaseAudioContext::renderAudio() { if (state_ == ContextState::CLOSED) { - return [](float *, int) {}; + return [](AudioBus *, int) {}; } return [this](AudioBus* data, int frames) { @@ -115,13 +110,13 @@ AudioNodeManager* BaseAudioContext::getNodeManager() { return nodeManager_.get(); } -std::string BaseAudioContext::toString(State state) { +std::string BaseAudioContext::toString(ContextState state) { switch (state) { - case State::SUSPENDED: + case ContextState::SUSPENDED: return "suspended"; - case State::RUNNING: + case ContextState::RUNNING: return "running"; - case State::CLOSED: + case ContextState::CLOSED: return "closed"; default: throw std::invalid_argument("Unknown context state"); diff --git a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h index abcb2adf..fb9052cf 100644 --- a/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h +++ b/packages/react-native-audio-api/common/cpp/core/BaseAudioContext.h @@ -1,23 +1,26 @@ #pragma once -#include #include #include -#include #include +#include +#include + +#include "ContextState.h" +#include "OscillatorType.h" namespace audioapi { class AudioBus; class GainNode; class AudioBuffer; +class PeriodicWave; class OscillatorNode; class StereoPannerNode; +class AudioNodeManager; class BiquadFilterNode; class AudioDestinationNode; class AudioBufferSourceNode; -class AudioNodeManager; -class PeriodicWave; #ifdef ANDROID class AudioPlayer; @@ -53,8 +56,7 @@ class BaseAudioContext { AudioNodeManager* getNodeManager(); protected: - enum class State { SUSPENDED, RUNNING, CLOSED }; - static std::string toString(State state); + static std::string toString(ContextState state); std::shared_ptr destination_; #ifdef ANDROID @@ -63,9 +65,9 @@ class BaseAudioContext { std::shared_ptr audioPlayer_; #endif - State state_ = State::RUNNING; int sampleRate_; int bufferSizeInFrames_; + ContextState state_ = ContextState::RUNNING; std::shared_ptr nodeManager_; private: diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index 380921c5..32af31bf 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -355,30 +355,29 @@ void BiquadFilterNode::processNode(AudioBus* processingBus, int framesToProcess) resetCoefficients(); applyFilter(); - float x1 = x1_; - float x2 = x2_; - float y1 = y1_; - float y2 = y2_; - - float b0 = b0_; - float b1 = b1_; - float b2 = b2_; - float a1 = a1_; - float a2 = a2_; - - for (int i = 0; i < framesToProcess; i += 1) { - float input = (*processingBus->getChannel(0))[i]; - float output = b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; - - for (int j = 0; j < channelCount_; j += 1) { - (*processingBus->getChannel(j))[i] = output; + for (int c = 0; c < framesToProcess; c += 1) { + float x1 = x1_; + float x2 = x2_; + float y1 = y1_; + float y2 = y2_; + + float b0 = b0_; + float b1 = b1_; + float b2 = b2_; + float a1 = a1_; + float a2 = a2_; + + for (int i = 0; i < framesToProcess; i += 1) { + float input = (*processingBus->getChannel(0))[i]; + float output = b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; + + (*processingBus->getChannel(c))[i] = output; + + x2 = x1; + x1 = input; + y2 = y1; + y1 = output; } - - - x2 = x1; - x1 = input; - y2 = y1; - y1 = output; } } diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index 184e4342..88db6728 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -40,7 +40,7 @@ void OscillatorNode::setPeriodicWave( } void OscillatorNode::processNode(AudioBus* processingBus, int framesToProcess) { - if (!isPlaying_) { + if (!isPlaying()) { processingBus->zero(); return; } diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h index e8d579db..e87a9a44 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.h @@ -5,9 +5,9 @@ #include #include "AudioParam.h" -#include "AudioScheduledSourceNode.h" -#include "OscillatorType.h" #include "PeriodicWave.h" +#include "OscillatorType.h" +#include "AudioScheduledSourceNode.h" namespace audioapi { From c997218d6754ca7bf6a59adcacb2f9383db51721 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Mon, 25 Nov 2024 21:58:24 +0100 Subject: [PATCH 25/33] fix: hi-hat sound --- .../src/examples/DrumMachine/DrumMachine.tsx | 8 ++++---- .../examples/SharedUtils/soundEngines/Clap.ts | 2 +- .../SharedUtils/soundEngines/HiHat.ts | 19 +++++++++--------- .../common/cpp/core/AudioBufferSourceNode.cpp | 20 ++++++++++--------- .../common/cpp/core/AudioDestinationNode.cpp | 7 ------- .../common/cpp/core/AudioNode.cpp | 9 --------- .../common/cpp/core/AudioNode.h | 4 ---- .../common/cpp/core/BiquadFilterNode.cpp | 4 ++-- .../common/cpp/core/GainNode.cpp | 1 - .../common/cpp/core/OscillatorNode.cpp | 1 - .../common/cpp/core/StereoPannerNode.cpp | 1 - 11 files changed, 27 insertions(+), 49 deletions(-) diff --git a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx index a6561085..e179a73f 100644 --- a/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx +++ b/apps/common-app/src/examples/DrumMachine/DrumMachine.tsx @@ -22,7 +22,7 @@ const defaultPreset = 'Empty'; function setupPlayer(audioCtx: AudioContext) { const kick = new Kick(audioCtx); - // const clap = new Clap(audioCtx); + const clap = new Clap(audioCtx); const hiHat = new HiHat(audioCtx); const playNote = (name: InstrumentName, time: number) => { @@ -30,9 +30,9 @@ function setupPlayer(audioCtx: AudioContext) { case 'kick': kick.play(time); break; - // case 'clap': - // clap.play(time); - // break; + case 'clap': + clap.play(time); + break; case 'hi-hat': hiHat.play(time); break; diff --git a/apps/common-app/src/examples/SharedUtils/soundEngines/Clap.ts b/apps/common-app/src/examples/SharedUtils/soundEngines/Clap.ts index 77cc6940..6a7aa5e8 100644 --- a/apps/common-app/src/examples/SharedUtils/soundEngines/Clap.ts +++ b/apps/common-app/src/examples/SharedUtils/soundEngines/Clap.ts @@ -20,7 +20,7 @@ class Clap implements SoundEngine { } createNoiseBuffer() { - const bufferSize = this.audioContext.sampleRate / 10; + const bufferSize = this.audioContext.sampleRate / 5; const buffer = this.audioContext.createBuffer( 1, bufferSize, diff --git a/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts b/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts index e8d4150b..72f8401a 100644 --- a/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts +++ b/apps/common-app/src/examples/SharedUtils/soundEngines/HiHat.ts @@ -29,7 +29,7 @@ class HiHat implements SoundEngine { const bandpassFilter = this.audioContext.createBiquadFilter(); const highpassFilter = this.audioContext.createBiquadFilter(); - // const gain = this.audioContext.createGain(); + const gain = this.audioContext.createGain(); bandpassFilter.type = 'bandpass'; bandpassFilter.frequency.value = this.bandpassFilterFrequency; @@ -37,18 +37,17 @@ class HiHat implements SoundEngine { highpassFilter.type = 'highpass'; highpassFilter.frequency.value = this.highpassFilterFrequency; - // gain.gain.setValueAtTime(0.0001, time); - // gain.gain.exponentialRampToValueAtTime(this.volume, time + 0.02); - // gain.gain.exponentialRampToValueAtTime(this.volume * 0.33, time + 0.03); - // gain.gain.exponentialRampToValueAtTime(this.volume * 0.0001, time + 0.3); - // gain.gain.setValueAtTime(0, time + 0.3 + 0.001); + gain.gain.setValueAtTime(0.0001, time); + gain.gain.exponentialRampToValueAtTime(this.volume, time + 0.02); + gain.gain.exponentialRampToValueAtTime(this.volume * 0.33, time + 0.03); + gain.gain.exponentialRampToValueAtTime(this.volume * 0.0001, time + 0.3); + gain.gain.setValueAtTime(0, time + 0.3 + 0.001); oscillator.connect(bandpassFilter); - // bandpassFilter.connect(highpassFilter); - // highpassFilter.connect(gain); - // gain.connect(this.audioContext.destination!); + bandpassFilter.connect(highpassFilter); + highpassFilter.connect(gain); + gain.connect(this.audioContext.destination!); - bandpassFilter.connect(this.audioContext.destination); oscillator.start(time); oscillator.stop(time + this.decay); }); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index c7c95c47..46baf544 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -9,7 +9,6 @@ AudioBufferSourceNode::AudioBufferSourceNode(BaseAudioContext *context) : AudioScheduledSourceNode(context), loop_(false), bufferIndex_(0) { numberOfInputs_ = 0; buffer_ = std::shared_ptr(nullptr); - isInitialized_ = true; } bool AudioBufferSourceNode::getLoop() const { @@ -60,18 +59,21 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro // The buffer is longer than the number of frames to process. // We have to keep track of where we are in the buffer. if (framesToProcess < buffer_->getLength()) { - int framesToCopy = std::min(framesToProcess, buffer_->getLength() - bufferIndex_); + int processingBufferPosition = 0; + int framesToCopy = 0; - processingBus->copy(buffer_->bus_.get(), bufferIndex_, framesToCopy); + while (processingBufferPosition < framesToProcess) + { + framesToCopy = std::min(framesToProcess, buffer_->getLength() - bufferIndex_); - if (loop_) { + processingBus->copy(buffer_->bus_.get(), processingBufferPosition, bufferIndex_, framesToCopy); + processingBufferPosition += framesToCopy; bufferIndex_ = (bufferIndex_ + framesToCopy) % buffer_->getLength(); - return; - } - if (bufferIndex_ + framesToCopy == buffer_->getLength() - 1) { - playbackState_ = PlaybackState::FINISHED; - bufferIndex_ = 0; + if (!loop_ && processingBufferPosition >= framesToCopy) { + playbackState_ = PlaybackState::FINISHED; + bufferIndex_ = 0; + } } return; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 6f960479..233bd33b 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -12,7 +12,6 @@ AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) numberOfOutputs_ = 0; numberOfInputs_ = INT_MAX; channelCountMode_ = ChannelCountMode::EXPLICIT; - isInitialized_ = true; } std::size_t AudioDestinationNode::getCurrentSampleFrame() const { @@ -24,12 +23,6 @@ double AudioDestinationNode::getCurrentTime() const { } void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFrames) { - printf("connected nodes: %d\n", inputNodes_.size()); - - if (!isInitialized_) { - return; - } - context_->getNodeManager()->preProcessGraph(); destinationBus->zero(); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index 0f15914a..bc8283f8 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -12,7 +12,6 @@ AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { } AudioNode::~AudioNode() { - isInitialized_ = false; cleanup(); } @@ -60,10 +59,6 @@ void AudioNode::disconnectNode(std::shared_ptr &node) { } } -bool AudioNode::isInitialized() const { - return isInitialized_; -} - bool AudioNode::isEnabled() const { return isEnabled_; } @@ -109,10 +104,6 @@ std::string AudioNode::toString(ChannelInterpretation interpretation) { } AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { - if (!isInitialized_) { - return outputBus; - } - std::size_t currentSampleFrame = context_->getCurrentSampleFrame(); // check if the node has already been processed for this rendering quantum diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index 80434ede..f2bdfa7b 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -25,8 +25,6 @@ class AudioNode : public std::enable_shared_from_this { void connect(const std::shared_ptr &node); void disconnect(const std::shared_ptr &node); - bool isInitialized() const; - bool isEnabled() const; void enable(); void disable(); @@ -44,8 +42,6 @@ class AudioNode : public std::enable_shared_from_this { int numberOfOutputs_ = 1; int numberOfEnabledInputNodes_ = 0; - - bool isInitialized_ = false; bool isEnabled_ = true; std::size_t lastRenderedFrame_ { SIZE_MAX }; diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index 32af31bf..1279587f 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -355,7 +355,7 @@ void BiquadFilterNode::processNode(AudioBus* processingBus, int framesToProcess) resetCoefficients(); applyFilter(); - for (int c = 0; c < framesToProcess; c += 1) { + for (int c = 0; c < processingBus->getNumberOfChannels(); c += 1) { float x1 = x1_; float x2 = x2_; float y1 = y1_; @@ -368,7 +368,7 @@ void BiquadFilterNode::processNode(AudioBus* processingBus, int framesToProcess) float a2 = a2_; for (int i = 0; i < framesToProcess; i += 1) { - float input = (*processingBus->getChannel(0))[i]; + float input = (*processingBus->getChannel(c))[i]; float output = b0 * input + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; (*processingBus->getChannel(c))[i] = output; diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index 9e43ff14..88bc6f50 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -7,7 +7,6 @@ namespace audioapi { GainNode::GainNode(BaseAudioContext *context) : AudioNode(context) { gainParam_ = std::make_shared(context, 1.0, -MAX_GAIN, MAX_GAIN); - isInitialized_ = true; } std::shared_ptr GainNode::getGainParam() const { diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index 88db6728..d23595dc 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -11,7 +11,6 @@ OscillatorNode::OscillatorNode(BaseAudioContext *context) context, 444.0, -NYQUIST_FREQUENCY, NYQUIST_FREQUENCY); detuneParam_ = std::make_shared(context, 0.0, -MAX_DETUNE, MAX_DETUNE); - isInitialized_ = true; type_ = OscillatorType::SINE; periodicWave_ = context_->getBasicWaveForm(type_); } diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index e3210516..cad4bfb1 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -12,7 +12,6 @@ StereoPannerNode::StereoPannerNode(BaseAudioContext *context) : AudioNode(context) { channelCountMode_ = ChannelCountMode::CLAMPED_MAX; panParam_ = std::make_shared(context, 0.0, -MAX_PAN, MAX_PAN); - isInitialized_ = true; } std::shared_ptr StereoPannerNode::getPanParam() const { From 6d61fbd38c10a1cc7d16de9fd00b371812b214cd Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 26 Nov 2024 14:43:29 +0100 Subject: [PATCH 26/33] feat: proper deconstruction --- .../src/examples/DrumMachine/usePlayer.tsx | 7 ++-- .../common/cpp/core/AudioBufferSourceNode.cpp | 35 +++++++++++++------ .../common/cpp/core/AudioDestinationNode.cpp | 4 +-- .../common/cpp/core/AudioNode.cpp | 17 +++++---- .../common/cpp/core/AudioNode.h | 34 ++---------------- .../common/cpp/core/AudioNodeManager.cpp | 24 +------------ .../common/cpp/core/AudioNodeManager.h | 4 --- .../cpp/core/AudioScheduledSourceNode.cpp | 1 + .../common/cpp/core/BiquadFilterNode.cpp | 1 + .../common/cpp/core/GainNode.cpp | 1 + .../common/cpp/core/OscillatorNode.cpp | 1 + .../common/cpp/core/StereoPannerNode.cpp | 1 + 12 files changed, 48 insertions(+), 82 deletions(-) diff --git a/apps/common-app/src/examples/DrumMachine/usePlayer.tsx b/apps/common-app/src/examples/DrumMachine/usePlayer.tsx index 597809f9..6f977556 100644 --- a/apps/common-app/src/examples/DrumMachine/usePlayer.tsx +++ b/apps/common-app/src/examples/DrumMachine/usePlayer.tsx @@ -133,10 +133,9 @@ export default function usePlayer(options: PlayerOptions) { playingInstruments.value = getPlayingInstruments(); } - // return () => { - // console.log('Closing audio context'); - // audioContext.close(); - // }; + return () => { + audioContext.close(); + }; // \/ Shared values are not necessary in deps array // eslint-disable-next-line react-hooks/exhaustive-deps }, [isPlaying, setup]); diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index 46baf544..adaf5fcf 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -9,6 +9,7 @@ AudioBufferSourceNode::AudioBufferSourceNode(BaseAudioContext *context) : AudioScheduledSourceNode(context), loop_(false), bufferIndex_(0) { numberOfInputs_ = 0; buffer_ = std::shared_ptr(nullptr); + isInitialized_ = true; } bool AudioBufferSourceNode::getLoop() const { @@ -51,6 +52,7 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro if (!loop_) { playbackState_ = PlaybackState::FINISHED; + disable(); } return; @@ -59,20 +61,31 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro // The buffer is longer than the number of frames to process. // We have to keep track of where we are in the buffer. if (framesToProcess < buffer_->getLength()) { - int processingBufferPosition = 0; + int outputBusIndex = 0; int framesToCopy = 0; - while (processingBufferPosition < framesToProcess) - { - framesToCopy = std::min(framesToProcess, buffer_->getLength() - bufferIndex_); + while (framesToProcess - outputBusIndex > 0) { + framesToCopy = std::min(framesToProcess - outputBusIndex, buffer_->getLength() - bufferIndex_); - processingBus->copy(buffer_->bus_.get(), processingBufferPosition, bufferIndex_, framesToCopy); - processingBufferPosition += framesToCopy; - bufferIndex_ = (bufferIndex_ + framesToCopy) % buffer_->getLength(); + processingBus->copy(buffer_->bus_.get(), bufferIndex_, outputBusIndex, framesToCopy); - if (!loop_ && processingBufferPosition >= framesToCopy) { - playbackState_ = PlaybackState::FINISHED; - bufferIndex_ = 0; + bufferIndex_ += framesToCopy; + outputBusIndex += framesToCopy; + + if (bufferIndex_ < buffer_->getLength()) { + continue; + } + + + bufferIndex_ %= buffer_->getLength(); + + if (!loop_) { + playbackState_ = PlaybackState::FINISHED; + disable(); + + if (framesToProcess - outputBusIndex > 0) { + processingBus->zero(outputBusIndex, framesToProcess - outputBusIndex); + } } } @@ -84,7 +97,9 @@ void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToPro // If we don't loop the buffer, copy it once and zero the remaining processing bus frames. processingBus->copy(buffer_->bus_.get()); processingBus->zero(buffer_->getLength(), framesToProcess - buffer_->getLength()); + playbackState_ = PlaybackState::FINISHED; + disable(); return; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp index 233bd33b..898d2258 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioDestinationNode.cpp @@ -12,6 +12,7 @@ AudioDestinationNode::AudioDestinationNode(BaseAudioContext *context) numberOfOutputs_ = 0; numberOfInputs_ = INT_MAX; channelCountMode_ = ChannelCountMode::EXPLICIT; + isInitialized_ = true; } std::size_t AudioDestinationNode::getCurrentSampleFrame() const { @@ -24,7 +25,6 @@ double AudioDestinationNode::getCurrentTime() const { void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFrames) { context_->getNodeManager()->preProcessGraph(); - destinationBus->zero(); if (!numFrames) { @@ -39,8 +39,6 @@ void AudioDestinationNode::renderAudio(AudioBus *destinationBus, int32_t numFram destinationBus->normalize(); - context_->getNodeManager()->postProcessGraph(); - currentSampleFrame_ += numFrames; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp index bc8283f8..be7d7f19 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.cpp @@ -12,6 +12,7 @@ AudioNode::AudioNode(BaseAudioContext *context) : context_(context) { } AudioNode::~AudioNode() { + isInitialized_ = false; cleanup(); } @@ -39,9 +40,8 @@ void AudioNode::connect(const std::shared_ptr &node) { context_->getNodeManager()->addPendingConnection(shared_from_this(), node, AudioNodeManager::ConnectionType::CONNECT); } -void AudioNode::connectNode(std::shared_ptr &node) { +void AudioNode::connectNode(const std::shared_ptr &node) { outputNodes_.push_back(node); - node->onInputConnected(this); } @@ -49,7 +49,7 @@ void AudioNode::disconnect(const std::shared_ptr &node) { context_->getNodeManager()->addPendingConnection(shared_from_this(), node, AudioNodeManager::ConnectionType::DISCONNECT); } -void AudioNode::disconnectNode(std::shared_ptr &node) { +void AudioNode::disconnectNode(const std::shared_ptr &node) { node->onInputDisconnected(this); auto position = std::find(outputNodes_.begin(), outputNodes_.end(), node); @@ -104,6 +104,10 @@ std::string AudioNode::toString(ChannelInterpretation interpretation) { } AudioBus* AudioNode::processAudio(AudioBus* outputBus, int framesToProcess) { + if (!isInitialized_) { + return outputBus; + } + std::size_t currentSampleFrame = context_->getCurrentSampleFrame(); // check if the node has already been processed for this rendering quantum @@ -197,13 +201,12 @@ void AudioNode::onInputConnected(AudioNode *node) { void AudioNode::onInputDisconnected(AudioNode *node) { auto position = std::find(inputNodes_.begin(), inputNodes_.end(), node); - if (position == inputNodes_.end()) { - return; + if (position != inputNodes_.end()) { + inputNodes_.erase(position); } - inputNodes_.erase(position); - if (inputNodes_.size() > 0 || outputNodes_.size() == 0) { + if (inputNodes_.size() > 0) { return; } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNode.h b/packages/react-native-audio-api/common/cpp/core/AudioNode.h index f2bdfa7b..e769c8ad 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNode.h @@ -42,6 +42,7 @@ class AudioNode : public std::enable_shared_from_this { int numberOfOutputs_ = 1; int numberOfEnabledInputNodes_ = 0; + bool isInitialized_ = false; bool isEnabled_ = true; std::size_t lastRenderedFrame_ { SIZE_MAX }; @@ -61,8 +62,8 @@ class AudioNode : public std::enable_shared_from_this { AudioBus* processAudio(AudioBus* outputBus, int framesToProcess); virtual void processNode(AudioBus* processingBus, int framesToProcess) = 0; - void connectNode(std::shared_ptr &node); - void disconnectNode(std::shared_ptr &node); + void connectNode(const std::shared_ptr &node); + void disconnectNode(const std::shared_ptr &node); void onInputEnabled(); void onInputDisabled(); @@ -71,32 +72,3 @@ class AudioNode : public std::enable_shared_from_this { }; } // namespace audioapi - -/* -Audio node management and deletion - -1. AudioSourceNode finishes playing, then: - - it disables itself (so it won't by processed anymore) - - if there is no reference from JS side, we can mark self for deletion - - node that is marked for deletion should be disconnected from the graph first - - it should "notify" connected nodes that it has been disabled (and potentially prepares for deletion) - -2. Node gets "notified" that one of its sources is disabled: - - it lowers the count of enabled sources - - if the count of enabled sources is 0, disable itself - - if there is no reference from JS side, we can mark self for deletion - - node that is marked for deletion should be disconnected from the graph first - - it should "notify" connected nodes that it has been disabled (and potentially prepares for deletion) - -Translating into more technical terms: -We use shared pointers for keeping output nodes -We use shared pointers in audio node manager to keep track of all source nodes -when audio source node finished playing it: - - disables itself and tells all output nodes that it has been disabled - - each node up to destination, checks their input nodes and if was its only active input node, it disables itself. - - source node tells audio node manager to dereference it (only if it is the last reference to the source node). - - audio manager in pre-process or post-process will remove the reference. - - audio manager in pre-process or post-process will also have to check for source nodes with only one reference and delete them if already stopped. - - deletion of the node will dereference all connected nodes, resulting in destroy'ing them if they are not referenced from JS side. - -*/ diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp index aad8a632..ee7b5b45 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp @@ -9,7 +9,7 @@ AudioNodeManager::AudioNodeManager() {} AudioNodeManager::~AudioNodeManager() { audioNodesToConnect_.clear(); - audioNodesToDelete_.clear(); + sourceNodes_.clear(); } void AudioNodeManager::addPendingConnection(const std::shared_ptr &from, const std::shared_ptr &to, ConnectionType type) { @@ -18,12 +18,6 @@ void AudioNodeManager::addPendingConnection(const std::shared_ptr &fr audioNodesToConnect_.push_back(std::make_tuple(from, to, type)); } -void AudioNodeManager::setNodeToDelete(const std::shared_ptr &node) { - Locker lock(getGraphLock()); - - audioNodesToDelete_.push_back(node); -} - void AudioNodeManager::addSourceNode(const std::shared_ptr &node) { Locker lock(getGraphLock()); @@ -35,17 +29,6 @@ void AudioNodeManager::preProcessGraph() { return; } - settlePendingDeletions(); - settlePendingConnections(); - removeFinishedSourceNodes(); -} - -void AudioNodeManager::postProcessGraph() { - if (!Locker::tryLock(getGraphLock())) { - return; - } - - settlePendingDeletions(); settlePendingConnections(); removeFinishedSourceNodes(); } @@ -70,16 +53,11 @@ void AudioNodeManager::settlePendingConnections() { audioNodesToConnect_.clear(); } -void AudioNodeManager::settlePendingDeletions() { - audioNodesToDelete_.clear(); -} - void AudioNodeManager::removeFinishedSourceNodes() { for (auto it = sourceNodes_.begin(); it != sourceNodes_.end();) { auto currentNode = it->get(); // Release the source node if use count is equal to 1 (this vector) if (!currentNode->isEnabled() && it->use_count() == 1) { - for (auto& outputNode : currentNode->outputNodes_) { currentNode->disconnectNode(outputNode); } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h index 35bc7a69..21a4219e 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h @@ -16,10 +16,8 @@ class AudioNodeManager { ~AudioNodeManager(); void preProcessGraph(); - void postProcessGraph(); void addPendingConnection(const std::shared_ptr &from, const std::shared_ptr &to, ConnectionType type); - void setNodeToDelete(const std::shared_ptr &node); void addSourceNode(const std::shared_ptr &node); std::mutex& getGraphLock(); @@ -28,10 +26,8 @@ class AudioNodeManager { std::mutex graphLock_; std::vector> sourceNodes_; - std::vector> audioNodesToDelete_; std::vector, std::shared_ptr, ConnectionType>> audioNodesToConnect_; - void settlePendingDeletions(); void settlePendingConnections(); void removeFinishedSourceNodes(); }; diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp index c9b4275d..90926f24 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.cpp @@ -7,6 +7,7 @@ namespace audioapi { AudioScheduledSourceNode::AudioScheduledSourceNode(BaseAudioContext *context) : AudioNode(context), playbackState_(PlaybackState::UNSCHEDULED) { numberOfInputs_ = 0; + isInitialized_ = true; } void AudioScheduledSourceNode::start(double time) { diff --git a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp index 1279587f..79b357ac 100644 --- a/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/BiquadFilterNode.cpp @@ -19,6 +19,7 @@ BiquadFilterNode::BiquadFilterNode(BaseAudioContext *context) gainParam_ = std::make_shared( context, 0.0, MIN_FILTER_GAIN, MAX_FILTER_GAIN); type_ = BiquadFilterType::LOWPASS; + isInitialized_ = true; } std::string BiquadFilterNode::getType() { diff --git a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp index 88bc6f50..9e43ff14 100644 --- a/packages/react-native-audio-api/common/cpp/core/GainNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/GainNode.cpp @@ -7,6 +7,7 @@ namespace audioapi { GainNode::GainNode(BaseAudioContext *context) : AudioNode(context) { gainParam_ = std::make_shared(context, 1.0, -MAX_GAIN, MAX_GAIN); + isInitialized_ = true; } std::shared_ptr GainNode::getGainParam() const { diff --git a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp index d23595dc..f8bbe403 100644 --- a/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/OscillatorNode.cpp @@ -13,6 +13,7 @@ OscillatorNode::OscillatorNode(BaseAudioContext *context) std::make_shared(context, 0.0, -MAX_DETUNE, MAX_DETUNE); type_ = OscillatorType::SINE; periodicWave_ = context_->getBasicWaveForm(type_); + isInitialized_ = true; } std::shared_ptr OscillatorNode::getFrequencyParam() const { diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index cad4bfb1..e3210516 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -12,6 +12,7 @@ StereoPannerNode::StereoPannerNode(BaseAudioContext *context) : AudioNode(context) { channelCountMode_ = ChannelCountMode::CLAMPED_MAX; panParam_ = std::make_shared(context, 0.0, -MAX_PAN, MAX_PAN); + isInitialized_ = true; } std::shared_ptr StereoPannerNode::getPanParam() const { From d0fcd3e78679b8df67e1f977a9805a4e8dfb30ab Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 26 Nov 2024 15:21:22 +0100 Subject: [PATCH 27/33] fix: unused variable, android player pointer usage --- .../android/src/main/cpp/AudioPlayer/AudioPlayer.cpp | 8 +++++--- .../common/cpp/core/AudioBufferSourceNode.cpp | 2 -- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp index 34a37a52..f0954670 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/AudioPlayer/AudioPlayer.cpp @@ -1,6 +1,8 @@ -#include "AudioPlayer.h" #include "AudioBus.h" +#include "Constants.h" +#include "AudioArray.h" +#include "AudioPlayer.h" #include "AudioContext.h" namespace audioapi { @@ -49,12 +51,12 @@ DataCallbackResult AudioPlayer::onAudioReady( int32_t numFrames) { auto buffer = static_cast(audioData); - renderAudio_(mBus_, numFrames); + renderAudio_(mBus_.get(), numFrames); // TODO: optimize this with SIMD? for (int32_t i = 0; i < numFrames; i += 1) { for (int channel = 0; channel < CHANNEL_COUNT; channel += 1) { - buffer[i * CHANNEL_COUNT + channel] = mBus_->getChannel(channel)->get()[i]; + buffer[i * CHANNEL_COUNT + channel] = mBus_->getChannel(channel)->getData()[i]; } } diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index adaf5fcf..d0032cc6 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -38,8 +38,6 @@ void AudioBufferSourceNode::setBuffer( // Note: AudioBus copy method will use memcpy if the source buffer and system processing bus have same channel count, // otherwise it will use the summing function taking care of up/down mixing. void AudioBufferSourceNode::processNode(AudioBus* processingBus, int framesToProcess) { - double time = context_->getCurrentTime(); - // No audio data to fill, zero the output and return. if (!isPlaying() || !buffer_ || buffer_->getLength() == 0) { processingBus->zero(); From 46e0796a468f284c029b32d33bc825f6ecaec863 Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 26 Nov 2024 15:38:06 +0100 Subject: [PATCH 28/33] fix: linter --- .../common/cpp/core/AudioBufferSourceNode.cpp | 2 ++ .../common/cpp/core/AudioNodeManager.h | 28 +++++++++---------- .../cpp/core/AudioScheduledSourceNode.h | 3 +- .../common/cpp/core/StereoPannerNode.cpp | 2 +- .../common/cpp/utils/Locker.hpp | 4 +-- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp index d0032cc6..9ba0ba57 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBufferSourceNode.cpp @@ -1,3 +1,5 @@ +#include + #include "AudioBus.h" #include "AudioArray.h" #include "BaseAudioContext.h" diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h index 21a4219e..6a06de99 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.h @@ -10,26 +10,26 @@ namespace audioapi { class AudioNode; class AudioNodeManager { - public: - enum class ConnectionType { CONNECT, DISCONNECT }; - AudioNodeManager(); - ~AudioNodeManager(); + public: + enum class ConnectionType { CONNECT, DISCONNECT }; + AudioNodeManager(); + ~AudioNodeManager(); - void preProcessGraph(); - void addPendingConnection(const std::shared_ptr &from, const std::shared_ptr &to, ConnectionType type); + void preProcessGraph(); + void addPendingConnection(const std::shared_ptr &from, const std::shared_ptr &to, ConnectionType type); - void addSourceNode(const std::shared_ptr &node); + void addSourceNode(const std::shared_ptr &node); - std::mutex& getGraphLock(); + std::mutex& getGraphLock(); - private: - std::mutex graphLock_; + private: + std::mutex graphLock_; - std::vector> sourceNodes_; - std::vector, std::shared_ptr, ConnectionType>> audioNodesToConnect_; + std::vector> sourceNodes_; + std::vector, std::shared_ptr, ConnectionType>> audioNodesToConnect_; - void settlePendingConnections(); - void removeFinishedSourceNodes(); + void settlePendingConnections(); + void removeFinishedSourceNodes(); }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h index 028e945a..0f6ad72a 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h +++ b/packages/react-native-audio-api/common/cpp/core/AudioScheduledSourceNode.h @@ -13,7 +13,7 @@ namespace audioapi { class AudioScheduledSourceNode : public AudioNode { public: - enum class PlaybackState { UNSCHEDULED, SCHEDULED, PLAYING, FINISHED }; + enum class PlaybackState { UNSCHEDULED, SCHEDULED, PLAYING, FINISHED }; explicit AudioScheduledSourceNode(BaseAudioContext *context); void start(double time); @@ -25,7 +25,6 @@ class AudioScheduledSourceNode : public AudioNode { protected: std::atomic playbackState_; - private: void startPlayback(); void stopPlayback(); diff --git a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp index e3210516..1b89d805 100644 --- a/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp +++ b/packages/react-native-audio-api/common/cpp/core/StereoPannerNode.cpp @@ -51,6 +51,6 @@ void StereoPannerNode::processNode(AudioBus* processingBus, int framesToProcess) time += deltaTime; } - } + } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/utils/Locker.hpp b/packages/react-native-audio-api/common/cpp/utils/Locker.hpp index 2f13adc0..1334c83d 100644 --- a/packages/react-native-audio-api/common/cpp/utils/Locker.hpp +++ b/packages/react-native-audio-api/common/cpp/utils/Locker.hpp @@ -7,8 +7,8 @@ namespace audioapi { // Small easy interface to manage locking class Locker { public: - Locker(): lockPtr_(0) {} - Locker(std::mutex& lockPtr): lockPtr_(&lockPtr) { + explicit Locker(): lockPtr_(0) {} + explicit Locker(std::mutex& lockPtr): lockPtr_(&lockPtr) { lock(); } From 8e68895711bded6401a9b9eeccbb08a266f4760e Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 26 Nov 2024 15:43:21 +0100 Subject: [PATCH 29/33] fix: linter2 - the return of the explicit? --- .../react-native-audio-api/common/cpp/core/AudioNodeManager.cpp | 2 +- .../common/cpp/utils/{Locker.hpp => Locker.h} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename packages/react-native-audio-api/common/cpp/utils/{Locker.hpp => Locker.h} (94%) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp index ee7b5b45..f790e193 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioNodeManager.cpp @@ -1,5 +1,5 @@ -#include "Locker.hpp" +#include "Locker.h" #include "AudioNode.h" #include "AudioNodeManager.h" diff --git a/packages/react-native-audio-api/common/cpp/utils/Locker.hpp b/packages/react-native-audio-api/common/cpp/utils/Locker.h similarity index 94% rename from packages/react-native-audio-api/common/cpp/utils/Locker.hpp rename to packages/react-native-audio-api/common/cpp/utils/Locker.h index 1334c83d..8764be38 100644 --- a/packages/react-native-audio-api/common/cpp/utils/Locker.hpp +++ b/packages/react-native-audio-api/common/cpp/utils/Locker.h @@ -7,7 +7,7 @@ namespace audioapi { // Small easy interface to manage locking class Locker { public: - explicit Locker(): lockPtr_(0) {} + Locker(): lockPtr_(0) {} explicit Locker(std::mutex& lockPtr): lockPtr_(&lockPtr) { lock(); } From 625cf45bc76d1be178b31388c45295a7c33a724d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20S=C4=99k?= Date: Tue, 26 Nov 2024 16:09:49 +0100 Subject: [PATCH 30/33] Update packages/react-native-audio-api/common/cpp/core/AudioBus.cpp --- packages/react-native-audio-api/common/cpp/core/AudioBus.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index 5e6c3cdb..20494ab6 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -125,7 +125,7 @@ void AudioBus::normalize() { } void AudioBus::scale(float value) { - for (auto it = channels_.begin(); it != channels_.end(); it += 1) { + for (auto it = channels_.begin(); it != channels_.end(); ++it) { it->get()->scale(value); } } From 9b3ebd260df101c95e9ad8d235f4993a0bff984d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20S=C4=99k?= Date: Tue, 26 Nov 2024 16:09:55 +0100 Subject: [PATCH 31/33] Update packages/react-native-audio-api/common/cpp/core/AudioBus.cpp --- packages/react-native-audio-api/common/cpp/core/AudioBus.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp index 20494ab6..4385a4b2 100644 --- a/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp +++ b/packages/react-native-audio-api/common/cpp/core/AudioBus.cpp @@ -133,7 +133,7 @@ void AudioBus::scale(float value) { float AudioBus::maxAbsValue() const { float maxAbsValue = 1.0f; - for (auto it = channels_.begin(); it != channels_.end(); it += 1) { + for (auto it = channels_.begin(); it != channels_.end(); ++it) { float channelMaxAbsValue = it->get()->getMaxAbsValue(); maxAbsValue = std::max(maxAbsValue, channelMaxAbsValue); } From 8150210df1d123c24d05a61a05dffa5644846cef Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 26 Nov 2024 16:10:15 +0100 Subject: [PATCH 32/33] fix: cleanup --- .../react-native-audio-api/src/core/AudioScheduledSourceNode.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts b/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts index 7c1051c0..74ce7787 100644 --- a/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts +++ b/packages/react-native-audio-api/src/core/AudioScheduledSourceNode.ts @@ -38,7 +38,6 @@ export default class AudioScheduledSourceNode extends AudioNode { ); } - // TODO: should we reset hasBeenStarted here? :thunk: (this.node as IAudioScheduledSourceNode).stop(when); } } From 12e569b14177092ac9bc18d2616e9fb2384cca3c Mon Sep 17 00:00:00 2001 From: Michal Sek Date: Tue, 26 Nov 2024 16:13:22 +0100 Subject: [PATCH 33/33] fix: redundant HAVE_ACCELERATE directive --- packages/react-native-audio-api/ios/AudioAPIModule.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/react-native-audio-api/ios/AudioAPIModule.h b/packages/react-native-audio-api/ios/AudioAPIModule.h index 7760a435..61a47468 100644 --- a/packages/react-native-audio-api/ios/AudioAPIModule.h +++ b/packages/react-native-audio-api/ios/AudioAPIModule.h @@ -8,10 +8,6 @@ #import #endif // RCT_NEW_ARCH_ENABLED -#ifndef HAVE_ACCELERATE -#define HAVE_ACCELERATE -#endif - #import @interface AudioAPIModule : RCTEventEmitter