From 2b0ba8ed1da3d8e3a8ae1a7db92c1ac53f8d0c70 Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Tue, 10 Dec 2024 20:40:08 +1100 Subject: [PATCH] Add AudioEncoderCocoa class, an AudioToolbox implementation of WebCodec's AudioEncoder https://bugs.webkit.org/show_bug.cgi?id=284019 rdar://140889671 Reviewed by NOBODY (OOPS!). We add AudioEncoderCocoa, AudioToolbox implementation of WebCodec's AudioEncoder. We only support encoding to Opus and AAC as the framework doesn't provide encoder for Flac, mp3 and vorbis (decoder only) Enabled WPT tests. * LayoutTests/TestExpectations: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt: * LayoutTests/platform/glib/TestExpectations: Remove passing expectations, add the remaining ones that do fail. * LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt: Removed. * LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt: Removed. * LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt: Added. The only failure remaining is related to different default for Opus encoding which yield slightly different values to what expected. The test performs very rough float comparisons. * LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt: Added. * LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt: Added. * LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt: Added. * Source/WebCore/Modules/webcodecs/OpusEncoderConfig.h: * Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp: (WebCore::isSupportedEncoderCodec): Add additional checks to ensure that the values provided in the Opus config are sane. While this isn't probably the best place to do so, as this is common for both GStreamer and Cocoa AudioEncoder, it's the simplest. (WebCore::isValidEncoderConfig): Make sure sampleRate and numberOfChannels aren't 0. (WebCore::createAudioEncoderConfig): Increase code readability by using struct's named member initialisation. (WebCore::WebCodecsAudioEncoder::configure): (WebCore::WebCodecsAudioEncoder::encode): Both Firefox and Chromes rejects AudioData that has a different sampleRate or numberOfChannels. While we did the same, we didn't process the errors as per spec which requires that we also change the state to Closed just before queueing the error. (WebCore::WebCodecsAudioEncoder::isConfigSupported): * Source/WebCore/PlatformMac.cmake: * Source/WebCore/SourcesCocoa.txt: * Source/WebCore/WebCore.xcodeproj/project.pbxproj: * Source/WebCore/platform/AudioEncoder.cpp: (WebCore::AudioEncoder::create): Plumb AudioEncoderCocoa. * Source/WebCore/platform/AudioEncoder.h: * Source/WebCore/platform/AudioEncoderActiveConfiguration.h: Add default initialiser to comply with more recent clang version. It allows to make the struct member initialisation optional. * Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.cpp: (WebCore::InternalAudioDecoderCocoa::initialize): Change default AudioData creation to use interleaved frames instead of planar. A few tests has this expectations, even though this isn't mandated by the specs. * Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.cpp: Added. (WebCore::InternalAudioEncoderCocoa::create): (WebCore::InternalAudioEncoderCocoa::reset): (WebCore::InternalAudioEncoderCocoa::converter): (WebCore::InternalAudioEncoderCocoa::queueSingleton): (WebCore::AudioEncoderCocoa::create): (WebCore::AudioEncoderCocoa::AudioEncoderCocoa): (WebCore::AudioEncoderCocoa::~AudioEncoderCocoa): (WebCore::AudioEncoderCocoa::encode): (WebCore::AudioEncoderCocoa::flush): (WebCore::AudioEncoderCocoa::reset): (WebCore::AudioEncoderCocoa::close): (WebCore::InternalAudioEncoderCocoa::InternalAudioEncoderCocoa): (WebCore::InternalAudioEncoderCocoa::initialize): (WebCore::InternalAudioEncoderCocoa::compressedAudioOutputBufferCallback): (WebCore::InternalAudioEncoderCocoa::generateDecoderDescriptionFromSample const): (WebCore::InternalAudioEncoderCocoa::activeConfiguration const): (WebCore::InternalAudioEncoderCocoa::processEncodedOutputs): (WebCore::InternalAudioEncoderCocoa::encode): (WebCore::InternalAudioEncoderCocoa::flush): (WebCore::InternalAudioEncoderCocoa::close): * Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.h: Copied from Source/WebCore/platform/AudioEncoderActiveConfiguration.h. --- LayoutTests/TestExpectations | 4 - ...oder-codec-specific.https.any-expected.txt | 2 +- ...udio-encoder-config.https.any-expected.txt | 32 +- ...coder-config.https.any.worker-expected.txt | 32 +- .../audio-encoder.https.any-expected.txt | 26 +- LayoutTests/platform/glib/TestExpectations | 12 +- ...udio-encoder-config.https.any-expected.txt | 53 --- ...coder-config.https.any.worker-expected.txt | 53 --- ...oder-codec-specific.https.any-expected.txt | 5 + .../audio-encoder.https.any-expected.txt | 26 ++ LayoutTests/platform/mac/TestExpectations | 2 + ...oder-codec-specific.https.any-expected.txt | 5 + .../audio-encoder.https.any-expected.txt | 26 ++ .../Modules/webcodecs/OpusEncoderConfig.h | 8 +- .../webcodecs/WebCodecsAudioEncoder.cpp | 73 +++- Source/WebCore/PlatformMac.cmake | 2 + Source/WebCore/SourcesCocoa.txt | 1 + .../WebCore/WebCore.xcodeproj/project.pbxproj | 6 + Source/WebCore/platform/AudioEncoder.cpp | 12 +- Source/WebCore/platform/AudioEncoder.h | 14 +- .../AudioEncoderActiveConfiguration.h | 10 +- .../audio/cocoa/AudioDecoderCocoa.cpp | 11 +- .../platform/audio/cocoa/AudioDecoderCocoa.h | 6 + .../audio/cocoa/AudioEncoderCocoa.cpp | 386 ++++++++++++++++++ .../platform/audio/cocoa/AudioEncoderCocoa.h | 59 +++ 25 files changed, 650 insertions(+), 216 deletions(-) delete mode 100644 LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt delete mode 100644 LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt create mode 100644 LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt create mode 100644 LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt create mode 100644 LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt create mode 100644 LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt create mode 100644 Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.cpp create mode 100644 Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.h diff --git a/LayoutTests/TestExpectations b/LayoutTests/TestExpectations index eb1c212bc2e01..7172bf2d74c47 100644 --- a/LayoutTests/TestExpectations +++ b/LayoutTests/TestExpectations @@ -6410,10 +6410,6 @@ imported/w3c/web-platform-tests/html/dom/render-blocking/element-render-blocking webkit.org/b/278192 imported/w3c/web-platform-tests/html/dom/render-blocking/element-render-blocking-033.html [ Failure ] -# AudioEncoder implementation missing. -imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any.html [ Failure ] -imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any.html [ Failure ] - webkit.org/b/258192 imported/w3c/web-platform-tests/webcodecs/full-cycle-test.https.any.worker.html?h264_avc [ Pass Failure ] # These tests are timing out since their import. diff --git a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt index 82257d86fea6e..8ed6e389b06ef 100644 --- a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt +++ b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt @@ -1,5 +1,5 @@ PASS Test the Opus DTX flag works. -FAIL Test the Opus bitrateMode flag works. assert_less_than: expected a number less than 40330 but got 80660 +PASS Test the Opus bitrateMode flag works. PASS Test the AAC bitrateMode flag works. diff --git a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt index 0223bf8a5f7ba..7c21124715834 100644 --- a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt +++ b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt @@ -3,9 +3,9 @@ PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Empty codec PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing sampleRate PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero sampleRate assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero channels assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Bit rate too big assert_unreached: Should have rejected: undefined Reached unreachable code +PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero sampleRate +PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero channels +PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Bit rate too big PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus complexity too big PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus packetlossperc too big PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus frame duration too small @@ -15,15 +15,9 @@ PASS Test that AudioEncoder.configure() rejects invalid config: Missing codec PASS Test that AudioEncoder.configure() rejects invalid config: Empty codec PASS Test that AudioEncoder.configure() rejects invalid config: Missing sampleRate PASS Test that AudioEncoder.configure() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero sampleRate assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero channels assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Bit rate too big assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw +PASS Test that AudioEncoder.configure() rejects invalid config: Zero sampleRate +PASS Test that AudioEncoder.configure() rejects invalid config: Zero channels +PASS Test that AudioEncoder.configure() rejects invalid config: Bit rate too big PASS Test that AudioEncoder.configure() rejects invalid config: Opus complexity too big PASS Test that AudioEncoder.configure() rejects invalid config: Opus packetlossperc too big PASS Test that AudioEncoder.configure() rejects invalid config: Opus frame duration too small @@ -43,11 +37,11 @@ PASS Test that AudioEncoder.configure() doesn't support config: Sample rate is t PASS Test that AudioEncoder.configure() doesn't support config: Way too many channels PASS Test that AudioEncoder.configure() doesn't support config: Possible future opus codec string PASS Test that AudioEncoder.configure() doesn't support config: Possible future aac codec string -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":8000,"numberOfChannels":1} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"constant","bogus":123} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"variable","bogus":123} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"complexity":5,"frameDuration":20000,"packetlossperc":10,"useinbandfec":true}} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"format":"opus","complexity":10,"frameDuration":60000,"packetlossperc":20,"usedtx":true,"bogus":456}} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{}} assert_true: expected true got false +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":8000,"numberOfChannels":1} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"constant","bogus":123} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"variable","bogus":123} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"complexity":5,"frameDuration":20000,"packetlossperc":10,"useinbandfec":true}} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"format":"opus","complexity":10,"frameDuration":60000,"packetlossperc":20,"usedtx":true,"bogus":456}} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{}} diff --git a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt index 0223bf8a5f7ba..7c21124715834 100644 --- a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt +++ b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt @@ -3,9 +3,9 @@ PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Empty codec PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing sampleRate PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero sampleRate assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero channels assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Bit rate too big assert_unreached: Should have rejected: undefined Reached unreachable code +PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero sampleRate +PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero channels +PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Bit rate too big PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus complexity too big PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus packetlossperc too big PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus frame duration too small @@ -15,15 +15,9 @@ PASS Test that AudioEncoder.configure() rejects invalid config: Missing codec PASS Test that AudioEncoder.configure() rejects invalid config: Empty codec PASS Test that AudioEncoder.configure() rejects invalid config: Missing sampleRate PASS Test that AudioEncoder.configure() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero sampleRate assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero channels assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Bit rate too big assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw +PASS Test that AudioEncoder.configure() rejects invalid config: Zero sampleRate +PASS Test that AudioEncoder.configure() rejects invalid config: Zero channels +PASS Test that AudioEncoder.configure() rejects invalid config: Bit rate too big PASS Test that AudioEncoder.configure() rejects invalid config: Opus complexity too big PASS Test that AudioEncoder.configure() rejects invalid config: Opus packetlossperc too big PASS Test that AudioEncoder.configure() rejects invalid config: Opus frame duration too small @@ -43,11 +37,11 @@ PASS Test that AudioEncoder.configure() doesn't support config: Sample rate is t PASS Test that AudioEncoder.configure() doesn't support config: Way too many channels PASS Test that AudioEncoder.configure() doesn't support config: Possible future opus codec string PASS Test that AudioEncoder.configure() doesn't support config: Possible future aac codec string -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":8000,"numberOfChannels":1} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"constant","bogus":123} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"variable","bogus":123} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"complexity":5,"frameDuration":20000,"packetlossperc":10,"useinbandfec":true}} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"format":"opus","complexity":10,"frameDuration":60000,"packetlossperc":20,"usedtx":true,"bogus":456}} assert_true: expected true got false -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{}} assert_true: expected true got false +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":8000,"numberOfChannels":1} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"constant","bogus":123} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"variable","bogus":123} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"complexity":5,"frameDuration":20000,"packetlossperc":10,"useinbandfec":true}} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"format":"opus","complexity":10,"frameDuration":60000,"packetlossperc":20,"usedtx":true,"bogus":456}} +PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{}} diff --git a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt index 64fd6f4197d84..c08414ff5fc0c 100644 --- a/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt +++ b/LayoutTests/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt @@ -2,19 +2,19 @@ PASS Simple audio encoding PASS Test reset during flush PASS Encode audio with negative timestamp -FAIL Channel number variation: 1 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Channel number variation: 2 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 3000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 13000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 23000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 33000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 43000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 53000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 63000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 73000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 83000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Sample rate variation: 93000 assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Encoding and decoding assert_true: expected true got false +PASS Channel number variation: 1 +PASS Channel number variation: 2 +PASS Sample rate variation: 3000 +PASS Sample rate variation: 13000 +PASS Sample rate variation: 23000 +PASS Sample rate variation: 33000 +PASS Sample rate variation: 43000 +PASS Sample rate variation: 53000 +PASS Sample rate variation: 63000 +PASS Sample rate variation: 73000 +PASS Sample rate variation: 83000 +PASS Sample rate variation: 93000 +PASS Encoding and decoding PASS Emit decoder config and extra data. PASS encodeQueueSize test PASS Test encoding Opus with additional parameters: Empty Opus config diff --git a/LayoutTests/platform/glib/TestExpectations b/LayoutTests/platform/glib/TestExpectations index 1f1d2c4f658d3..209da4fda6630 100644 --- a/LayoutTests/platform/glib/TestExpectations +++ b/LayoutTests/platform/glib/TestExpectations @@ -1301,14 +1301,10 @@ imported/w3c/web-platform-tests/webcodecs/audioDecoder-codec-specific.https.any. # A bit flaky on Debug bot. imported/w3c/web-platform-tests/webcodecs/audioDecoder-codec-specific.https.any.html?mp4_aac [ Pass Failure ] -imported/w3c/web-platform-tests/webcodecs/encoded-audio-chunk.any.html [ Pass ] -imported/w3c/web-platform-tests/webcodecs/encoded-audio-chunk.any.worker.html [ Pass ] -imported/w3c/web-platform-tests/webcodecs/encoded-audio-chunk.crossOriginIsolated.https.any.html [ Pass ] -imported/w3c/web-platform-tests/webcodecs/encoded-audio-chunk.crossOriginIsolated.https.any.worker.html [ Pass ] -imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any.html [ Pass ] -imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.html [ Pass ] -imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker.html [ Pass ] -imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any.html [ Pass DumpJSConsoleLogInStdErr ] +webkit.org/b/284426 imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any.html [ Failure ] +webkit.org/b/284428 imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.html [ Failure ] +webkit.org/b/284428 imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker.html [ Failure ] +webkit.org/b/284429 imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any.html [ Failure ] http/wpt/webcodecs/encoder-task-failing.html [ Pass ] # H.264 high-4:2:2 encoding is supported in the GStreamer ports, so this test (checking that profile diff --git a/LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt b/LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt deleted file mode 100644 index 98cb25c7df71b..0000000000000 --- a/LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any-expected.txt +++ /dev/null @@ -1,53 +0,0 @@ - -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing codec -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Empty codec -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing sampleRate -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero sampleRate assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero channels assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Bit rate too big assert_unreached: Should have rejected: undefined Reached unreachable code -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus complexity too big -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus packetlossperc too big -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus frame duration too small -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus frame duration too big -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Invalid Opus frameDuration -PASS Test that AudioEncoder.configure() rejects invalid config: Missing codec -PASS Test that AudioEncoder.configure() rejects invalid config: Empty codec -PASS Test that AudioEncoder.configure() rejects invalid config: Missing sampleRate -PASS Test that AudioEncoder.configure() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero sampleRate assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero channels assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Bit rate too big assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -PASS Test that AudioEncoder.configure() rejects invalid config: Opus complexity too big -PASS Test that AudioEncoder.configure() rejects invalid config: Opus packetlossperc too big -PASS Test that AudioEncoder.configure() rejects invalid config: Opus frame duration too small -PASS Test that AudioEncoder.configure() rejects invalid config: Opus frame duration too big -PASS Test that AudioEncoder.configure() rejects invalid config: Invalid Opus frameDuration -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Bitrate is too low assert_false: expected false got true -PASS Test that AudioEncoder.isConfigSupported() doesn't support config: Unrecognized codec -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Sample rate is too small assert_false: expected false got true -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Sample rate is too large assert_false: expected false got true -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Way too many channels assert_false: expected false got true -PASS Test that AudioEncoder.isConfigSupported() doesn't support config: Possible future opus codec string -PASS Test that AudioEncoder.isConfigSupported() doesn't support config: Possible future aac codec string -FAIL Test that AudioEncoder.configure() doesn't support config: Bitrate is too low assert_unreached: flush succeeded unexpectedly Reached unreachable code -PASS Test that AudioEncoder.configure() doesn't support config: Unrecognized codec -FAIL Test that AudioEncoder.configure() doesn't support config: Sample rate is too small assert_unreached: flush succeeded unexpectedly Reached unreachable code -FAIL Test that AudioEncoder.configure() doesn't support config: Sample rate is too large assert_unreached: flush succeeded unexpectedly Reached unreachable code -FAIL Test that AudioEncoder.configure() doesn't support config: Way too many channels assert_unreached: flush succeeded unexpectedly Reached unreachable code -PASS Test that AudioEncoder.configure() doesn't support config: Possible future opus codec string -PASS Test that AudioEncoder.configure() doesn't support config: Possible future aac codec string -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":8000,"numberOfChannels":1} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"constant","bogus":123} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"variable","bogus":123} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"complexity":5,"frameDuration":20000,"packetlossperc":10,"useinbandfec":true}} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"format":"opus","complexity":10,"frameDuration":60000,"packetlossperc":20,"usedtx":true,"bogus":456}} -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{}} assert_true: expected true got false - diff --git a/LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt b/LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt deleted file mode 100644 index 98cb25c7df71b..0000000000000 --- a/LayoutTests/platform/glib/imported/w3c/web-platform-tests/webcodecs/audio-encoder-config.https.any.worker-expected.txt +++ /dev/null @@ -1,53 +0,0 @@ - -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing codec -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Empty codec -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing sampleRate -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero sampleRate assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Zero channels assert_unreached: Should have rejected: undefined Reached unreachable code -FAIL Test that AudioEncoder.isConfigSupported() rejects invalid config: Bit rate too big assert_unreached: Should have rejected: undefined Reached unreachable code -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus complexity too big -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus packetlossperc too big -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus frame duration too small -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Opus frame duration too big -PASS Test that AudioEncoder.isConfigSupported() rejects invalid config: Invalid Opus frameDuration -PASS Test that AudioEncoder.configure() rejects invalid config: Missing codec -PASS Test that AudioEncoder.configure() rejects invalid config: Empty codec -PASS Test that AudioEncoder.configure() rejects invalid config: Missing sampleRate -PASS Test that AudioEncoder.configure() rejects invalid config: Missing numberOfChannels -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero sampleRate assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Zero channels assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -FAIL Test that AudioEncoder.configure() rejects invalid config: Bit rate too big assert_throws_js: function "() => { - codec.configure(entry.config); - }" did not throw -PASS Test that AudioEncoder.configure() rejects invalid config: Opus complexity too big -PASS Test that AudioEncoder.configure() rejects invalid config: Opus packetlossperc too big -PASS Test that AudioEncoder.configure() rejects invalid config: Opus frame duration too small -PASS Test that AudioEncoder.configure() rejects invalid config: Opus frame duration too big -PASS Test that AudioEncoder.configure() rejects invalid config: Invalid Opus frameDuration -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Bitrate is too low assert_false: expected false got true -PASS Test that AudioEncoder.isConfigSupported() doesn't support config: Unrecognized codec -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Sample rate is too small assert_false: expected false got true -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Sample rate is too large assert_false: expected false got true -FAIL Test that AudioEncoder.isConfigSupported() doesn't support config: Way too many channels assert_false: expected false got true -PASS Test that AudioEncoder.isConfigSupported() doesn't support config: Possible future opus codec string -PASS Test that AudioEncoder.isConfigSupported() doesn't support config: Possible future aac codec string -FAIL Test that AudioEncoder.configure() doesn't support config: Bitrate is too low assert_unreached: flush succeeded unexpectedly Reached unreachable code -PASS Test that AudioEncoder.configure() doesn't support config: Unrecognized codec -FAIL Test that AudioEncoder.configure() doesn't support config: Sample rate is too small assert_unreached: flush succeeded unexpectedly Reached unreachable code -FAIL Test that AudioEncoder.configure() doesn't support config: Sample rate is too large assert_unreached: flush succeeded unexpectedly Reached unreachable code -FAIL Test that AudioEncoder.configure() doesn't support config: Way too many channels assert_unreached: flush succeeded unexpectedly Reached unreachable code -PASS Test that AudioEncoder.configure() doesn't support config: Possible future opus codec string -PASS Test that AudioEncoder.configure() doesn't support config: Possible future aac codec string -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":8000,"numberOfChannels":1} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"constant","bogus":123} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"bitrate":128000,"bitrateMode":"variable","bogus":123} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"complexity":5,"frameDuration":20000,"packetlossperc":10,"useinbandfec":true}} -PASS AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{"format":"opus","complexity":10,"frameDuration":60000,"packetlossperc":20,"usedtx":true,"bogus":456}} -FAIL AudioEncoder.isConfigSupported() supports: {"codec":"opus","sampleRate":48000,"numberOfChannels":2,"opus":{}} assert_true: expected true got false - diff --git a/LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt b/LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt new file mode 100644 index 0000000000000..05ef8ee2a9a7a --- /dev/null +++ b/LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt @@ -0,0 +1,5 @@ + +FAIL Test the Opus DTX flag works. assert_less_than: expected a number less than 250.5 but got 501 +PASS Test the Opus bitrateMode flag works. +PASS Test the AAC bitrateMode flag works. + diff --git a/LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt b/LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt new file mode 100644 index 0000000000000..d57ed864399cb --- /dev/null +++ b/LayoutTests/platform/ios/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt @@ -0,0 +1,26 @@ + +PASS Simple audio encoding +PASS Test reset during flush +PASS Encode audio with negative timestamp +PASS Channel number variation: 1 +PASS Channel number variation: 2 +PASS Sample rate variation: 3000 +PASS Sample rate variation: 13000 +PASS Sample rate variation: 23000 +PASS Sample rate variation: 33000 +PASS Sample rate variation: 43000 +PASS Sample rate variation: 53000 +PASS Sample rate variation: 63000 +PASS Sample rate variation: 73000 +PASS Sample rate variation: 83000 +PASS Sample rate variation: 93000 +FAIL Encoding and decoding assert_approx_equals: Difference between input and output is too large. index: 100 channel: 0 input: 0.9659258127212524 output: 0.4632495939731598 expected 0.4632495939731598 +/- 0.5 but got 0.9659258127212524 +PASS Emit decoder config and extra data. +PASS encodeQueueSize test +PASS Test encoding Opus with additional parameters: Empty Opus config +PASS Test encoding Opus with additional parameters: Opus with frameDuration +PASS Test encoding Opus with additional parameters: Opus with complexity +PASS Test encoding Opus with additional parameters: Opus with useinbandfec +PASS Test encoding Opus with additional parameters: Opus with usedtx +PASS Test encoding Opus with additional parameters: Opus mixed parameters + diff --git a/LayoutTests/platform/mac/TestExpectations b/LayoutTests/platform/mac/TestExpectations index e99304e393446..3066c914262c3 100644 --- a/LayoutTests/platform/mac/TestExpectations +++ b/LayoutTests/platform/mac/TestExpectations @@ -2422,6 +2422,8 @@ http/tests/media/fairplay/fps-mse-multi-key-renewal.html [ Pass Failure ] [ Ventura ] imported/w3c/web-platform-tests/webcodecs/audioDecoder-codec-specific.https.any.worker.html?mp3 [ Failure ] [ Ventura ] imported/w3c/web-platform-tests/webcodecs/audioDecoder-codec-specific.https.any.worker.html?opus [ Failure ] +[ Ventura ] imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any.html [ Failure ] # Ventura doesn't support resampling when encoding to Opus, causing the test to fail. + # webkit.org/b/280424 [ Sequoia ] fast/table/col-and-colgroup-offsets.html is a constant failure. [ Sequoia+ ] fast/table/col-and-colgroup-offsets.html [ Failure ] diff --git a/LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt b/LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt new file mode 100644 index 0000000000000..05ef8ee2a9a7a --- /dev/null +++ b/LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder-codec-specific.https.any-expected.txt @@ -0,0 +1,5 @@ + +FAIL Test the Opus DTX flag works. assert_less_than: expected a number less than 250.5 but got 501 +PASS Test the Opus bitrateMode flag works. +PASS Test the AAC bitrateMode flag works. + diff --git a/LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt b/LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt new file mode 100644 index 0000000000000..d57ed864399cb --- /dev/null +++ b/LayoutTests/platform/mac/imported/w3c/web-platform-tests/webcodecs/audio-encoder.https.any-expected.txt @@ -0,0 +1,26 @@ + +PASS Simple audio encoding +PASS Test reset during flush +PASS Encode audio with negative timestamp +PASS Channel number variation: 1 +PASS Channel number variation: 2 +PASS Sample rate variation: 3000 +PASS Sample rate variation: 13000 +PASS Sample rate variation: 23000 +PASS Sample rate variation: 33000 +PASS Sample rate variation: 43000 +PASS Sample rate variation: 53000 +PASS Sample rate variation: 63000 +PASS Sample rate variation: 73000 +PASS Sample rate variation: 83000 +PASS Sample rate variation: 93000 +FAIL Encoding and decoding assert_approx_equals: Difference between input and output is too large. index: 100 channel: 0 input: 0.9659258127212524 output: 0.4632495939731598 expected 0.4632495939731598 +/- 0.5 but got 0.9659258127212524 +PASS Emit decoder config and extra data. +PASS encodeQueueSize test +PASS Test encoding Opus with additional parameters: Empty Opus config +PASS Test encoding Opus with additional parameters: Opus with frameDuration +PASS Test encoding Opus with additional parameters: Opus with complexity +PASS Test encoding Opus with additional parameters: Opus with useinbandfec +PASS Test encoding Opus with additional parameters: Opus with usedtx +PASS Test encoding Opus with additional parameters: Opus mixed parameters + diff --git a/Source/WebCore/Modules/webcodecs/OpusEncoderConfig.h b/Source/WebCore/Modules/webcodecs/OpusEncoderConfig.h index 18bae11baa04f..a4f5b697259e2 100644 --- a/Source/WebCore/Modules/webcodecs/OpusEncoderConfig.h +++ b/Source/WebCore/Modules/webcodecs/OpusEncoderConfig.h @@ -39,10 +39,10 @@ struct OpusEncoderConfig { float frameDurationMs = frameDuration / 1000.0; if (!WTF::anyOf(Vector { 2.5, 5, 10, 20, 40, 60, 120 }, [frameDurationMs](auto value) -> bool { return WTF::areEssentiallyEqual(value, frameDurationMs); - })) + })) { return false; - - if (complexity && *complexity > 10) + } + if (complexity > 10) return false; if (packetlossperc > 100) @@ -54,7 +54,7 @@ struct OpusEncoderConfig { using BitstreamFormat = OpusBitstreamFormat; OpusBitstreamFormat format { OpusBitstreamFormat::Opus }; uint64_t frameDuration { 20000 }; - std::optional complexity; + size_t complexity { 9 }; size_t packetlossperc { 0 }; bool useinbandfec { false }; bool usedtx { false }; diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp index b6afafdffb0d2..68a16cb563a25 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp @@ -72,9 +72,10 @@ WebCodecsAudioEncoder::WebCodecsAudioEncoder(ScriptExecutionContext& context, In WebCodecsAudioEncoder::~WebCodecsAudioEncoder() = default; -static bool isSupportedEncoderCodec(const StringView& codec) +static bool isSupportedEncoderCodec(const WebCodecsAudioEncoderConfig& config) { // FIXME: Check codec more accurately. + const auto& codec = config.codec; bool isMPEG4AAC = codec == "mp4a.40.2"_s || codec == "mp4a.40.02"_s || codec == "mp4a.40.5"_s || codec == "mp4a.40.05"_s || codec == "mp4a.40.29"_s || codec == "mp4a.40.42"_s; bool isCodecAllowed = isMPEG4AAC || codec == "mp3"_s || codec == "opus"_s @@ -84,6 +85,16 @@ static bool isSupportedEncoderCodec(const StringView& codec) if (!isCodecAllowed) return false; + // FIXME: https://github.com/web-platform-tests/wpt/issues/49635 + // WPT audio-encoder-config.https.any.html checks for the samplingRate is between "supported" values. + if (config.sampleRate < 3000 || config.sampleRate > 384000) + return false; + + // FIXME: New WPT requires this to reject as non valid. For now we just state that it's not supported (webkit.org/b/283900) + // https://w3c.github.io/webcodecs/opus_codec_registration.html#opus-encoder-config + if (codec == "opus"_s && config.bitrate && (*config.bitrate < 6000 || *config.bitrate > 510000)) + return false; + return true; } @@ -92,6 +103,15 @@ static bool isValidEncoderConfig(const WebCodecsAudioEncoderConfig& config) if (StringView(config.codec).trim(isASCIIWhitespace).isEmpty()) return false; + if (!config.sampleRate || !config.numberOfChannels) + return false; + + // FIXME: This isn't per spec, but both Chrome and Firefox checks that the bitrate is now greater than INT_MAX + // Even though the spec made it a `long long` + // https://github.com/web-platform-tests/wpt/issues/49634 + if (config.bitrate && *config.bitrate > std::numeric_limits::max()) + return false; + // FIXME: The opus and flac checks will probably need to move so that they trigger NotSupported // errors in the future. if (auto opusConfig = config.opus) { @@ -110,8 +130,16 @@ static bool isValidEncoderConfig(const WebCodecsAudioEncoderConfig& config) static ExceptionOr createAudioEncoderConfig(const WebCodecsAudioEncoderConfig& config) { std::optional opusConfig = std::nullopt; - if (config.opus) - opusConfig = { config.opus->format == OpusBitstreamFormat::Ogg, config.opus->frameDuration, config.opus->complexity, config.opus->packetlossperc, config.opus->useinbandfec, config.opus->usedtx }; + if (config.opus) { + opusConfig = { + .isOggBitStream = config.opus->format == OpusBitstreamFormat::Ogg, + .frameDuration = config.opus->frameDuration, + .complexity = config.opus->complexity, + .packetlossperc = config.opus->packetlossperc, + .useinbandfec = config.opus->useinbandfec, + .usedtx = config.opus->usedtx + }; + } std::optional isAacADTS = std::nullopt; if (config.aac) @@ -121,7 +149,15 @@ static ExceptionOr createAudioEncoderConfig(const WebCodec if (config.flac) flacConfig = { config.flac->blockSize, config.flac->compressLevel }; - return AudioEncoder::Config { config.sampleRate, config.numberOfChannels, config.bitrate.value_or(0), WTFMove(opusConfig), WTFMove(isAacADTS), WTFMove(flacConfig) }; + return AudioEncoder::Config { + .sampleRate = config.sampleRate, + .numberOfChannels = config.numberOfChannels, + .bitRate = config.bitrate.value_or(0), + .bitRateMode = config.bitrateMode, + .opusConfig = WTFMove(opusConfig), + .isAacADTS = isAacADTS, + .flacConfig = WTFMove(flacConfig) + }; } ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebCodecsAudioEncoderConfig&& config) @@ -153,7 +189,7 @@ ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebC } }); } - bool isSupportedCodec = isSupportedEncoderCodec(config.codec); + bool isSupportedCodec = isSupportedEncoderCodec(config); queueControlMessageAndProcess({ *this, [this, config = WTFMove(config), isSupportedCodec, identifier = scriptExecutionContext()->identifier()]() mutable { RefPtr context = scriptExecutionContext(); @@ -255,27 +291,20 @@ ExceptionOr WebCodecsAudioEncoder::encode(Ref&& frame) if (m_state != WebCodecsCodecState::Configured) return Exception { ExceptionCode::InvalidStateError, "AudioEncoder is not configured"_s }; - // FIXME: These checks are not yet spec-compliant. See also https://github.com/w3c/webcodecs/issues/716 - if (m_activeConfiguration.numberOfChannels && *m_activeConfiguration.numberOfChannels != audioData->numberOfChannels()) { - frame->close(); - queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { - m_error->handleEvent(DOMException::create(Exception { ExceptionCode::EncodingError, "Input audio buffer is incompatible with codec parameters"_s })); - }); - return { }; - } - if (m_activeConfiguration.sampleRate && *m_activeConfiguration.sampleRate != audioData->sampleRate()) { - frame->close(); - queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { - m_error->handleEvent(DOMException::create(Exception { ExceptionCode::EncodingError, "Input audio buffer is incompatible with codec parameters"_s })); - }); - return { }; - } - ++m_encodeQueueSize; queueControlMessageAndProcess({ *this, [this, audioData = WTFMove(audioData), timestamp = frame->timestamp(), duration = frame->duration()]() mutable { --m_encodeQueueSize; scheduleDequeueEvent(); + // FIXME: These checks are not yet spec-compliant. See also https://github.com/w3c/webcodecs/issues/716 + if ((m_activeConfiguration.numberOfChannels && *m_activeConfiguration.numberOfChannels != audioData->numberOfChannels()) + || (m_activeConfiguration.sampleRate && *m_activeConfiguration.sampleRate != audioData->sampleRate())) { + queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { + closeEncoder(Exception { ExceptionCode::EncodingError, "Input audio buffer is incompatible with codec parameters"_s }); + }); + return; + } + protectedScriptExecutionContext()->enqueueTaskWhenSettled(Ref { *m_internalEncoder }->encode({ WTFMove(audioData), timestamp, duration }), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = makePendingActivity(*this)] (auto&& result) { RefPtr protectedThis = weakThis.get(); if (!protectedThis || !!result) @@ -323,7 +352,7 @@ void WebCodecsAudioEncoder::isConfigSupported(ScriptExecutionContext& context, W return; } - if (!isSupportedEncoderCodec(config.codec)) { + if (!isSupportedEncoderCodec(config)) { promise->template resolve>(WebCodecsAudioEncoderSupport { false, WTFMove(config) }); return; } diff --git a/Source/WebCore/PlatformMac.cmake b/Source/WebCore/PlatformMac.cmake index cd641b43b058e..e86dc706b139f 100644 --- a/Source/WebCore/PlatformMac.cmake +++ b/Source/WebCore/PlatformMac.cmake @@ -197,6 +197,7 @@ list(APPEND WebCore_SOURCES platform/audio/AudioSession.cpp platform/audio/cocoa/AudioDecoderCocoa.cpp + platform/audio/cocoa/AudioEncoderCocoa.cpp platform/audio/cocoa/WebAudioBufferList.cpp platform/audio/mac/AudioBusMac.mm @@ -583,6 +584,7 @@ list(APPEND WebCore_PRIVATE_FRAMEWORK_HEADERS platform/audio/cocoa/AudioDecoderCocoa.h platform/audio/cocoa/AudioDestinationCocoa.h + platform/audio/cocoa/AudioEncoderCocoa.h platform/audio/cocoa/AudioOutputUnitAdaptor.h platform/audio/cocoa/AudioSampleBufferList.h platform/audio/cocoa/AudioSampleDataConverter.h diff --git a/Source/WebCore/SourcesCocoa.txt b/Source/WebCore/SourcesCocoa.txt index 28cb05294bc31..ba2acd8786381 100644 --- a/Source/WebCore/SourcesCocoa.txt +++ b/Source/WebCore/SourcesCocoa.txt @@ -266,6 +266,7 @@ platform/animation/AcceleratedEffectStack.mm @no-unify platform/audio/AudioSession.cpp platform/audio/cocoa/AudioDecoderCocoa.cpp platform/audio/cocoa/AudioDestinationCocoa.cpp +platform/audio/cocoa/AudioEncoderCocoa.cpp platform/audio/cocoa/AudioFileReaderCocoa.cpp platform/audio/cocoa/AudioOutputUnitAdaptor.cpp platform/audio/cocoa/AudioSampleBufferList.cpp diff --git a/Source/WebCore/WebCore.xcodeproj/project.pbxproj b/Source/WebCore/WebCore.xcodeproj/project.pbxproj index 53864b7fa909d..92e89b1be2264 100644 --- a/Source/WebCore/WebCore.xcodeproj/project.pbxproj +++ b/Source/WebCore/WebCore.xcodeproj/project.pbxproj @@ -1932,6 +1932,7 @@ 510A91FE24D3C16700BFD89C /* GamepadConstantsMac.h in Headers */ = {isa = PBXBuildFile; fileRef = 510A91FB24D3C0FD00BFD89C /* GamepadConstantsMac.h */; }; 510A920824D4F49900BFD89C /* LogitechGamepad.h in Headers */ = {isa = PBXBuildFile; fileRef = 510A920524D4F07A00BFD89C /* LogitechGamepad.h */; }; 510A921224D5E60E00BFD89C /* StadiaHIDGamepad.h in Headers */ = {isa = PBXBuildFile; fileRef = 510A921124D5E21700BFD89C /* StadiaHIDGamepad.h */; }; + 510CF7572CFDB1B3005E1950 /* AudioEncoderCocoa.h in Headers */ = {isa = PBXBuildFile; fileRef = 510CF7552CFDB1B3005E1950 /* AudioEncoderCocoa.h */; settings = {ATTRIBUTES = (Private, ); }; }; 510D4A34103165EE0049EA54 /* SocketStreamError.h in Headers */ = {isa = PBXBuildFile; fileRef = 510D4A2E103165EE0049EA54 /* SocketStreamError.h */; settings = {ATTRIBUTES = (Private, ); }; }; 510E2F25276BB4EC00809333 /* NotificationEvent.h in Headers */ = {isa = PBXBuildFile; fileRef = 510E2F24276BB4EC00809333 /* NotificationEvent.h */; }; 510E2F27276BC19F00809333 /* NotificationOptions.h in Headers */ = {isa = PBXBuildFile; fileRef = 510E2F26276BC19F00809333 /* NotificationOptions.h */; settings = {ATTRIBUTES = (Private, ); }; }; @@ -11389,6 +11390,8 @@ 510A920724D4F07A00BFD89C /* LogitechGamepad.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LogitechGamepad.cpp; sourceTree = ""; }; 510A920F24D5E21700BFD89C /* StadiaHIDGamepad.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = StadiaHIDGamepad.cpp; sourceTree = ""; }; 510A921124D5E21700BFD89C /* StadiaHIDGamepad.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StadiaHIDGamepad.h; sourceTree = ""; }; + 510CF7552CFDB1B3005E1950 /* AudioEncoderCocoa.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AudioEncoderCocoa.h; sourceTree = ""; }; + 510CF7562CFDB1B3005E1950 /* AudioEncoderCocoa.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = AudioEncoderCocoa.cpp; sourceTree = ""; }; 510D4A2E103165EE0049EA54 /* SocketStreamError.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SocketStreamError.h; sourceTree = ""; }; 510E2F22276BA9E800809333 /* NotificationEvent.idl */ = {isa = PBXFileReference; lastKnownFileType = text; path = NotificationEvent.idl; sourceTree = ""; }; 510E2F24276BB4EC00809333 /* NotificationEvent.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NotificationEvent.h; sourceTree = ""; }; @@ -35769,6 +35772,8 @@ 51DCE1A92CF5800D00BE7E84 /* AudioDecoderCocoa.h */, 413151842357745E00115E6E /* AudioDestinationCocoa.cpp */, 413151862357745E00115E6E /* AudioDestinationCocoa.h */, + 510CF7562CFDB1B3005E1950 /* AudioEncoderCocoa.cpp */, + 510CF7552CFDB1B3005E1950 /* AudioEncoderCocoa.h */, 46AAAA3C25D3631400BAF42F /* AudioFileReaderCocoa.cpp */, 46AAAA3A25D3631400BAF42F /* AudioFileReaderCocoa.h */, 1DB66D38253678EA00B671B9 /* AudioOutputUnitAdaptor.cpp */, @@ -39555,6 +39560,7 @@ 7BC9642C291E76BE00377DF1 /* AudioDestinationResampler.h in Headers */, FD31608012B026F700C1A359 /* AudioDSPKernel.h in Headers */, FD31608212B026F700C1A359 /* AudioDSPKernelProcessor.h in Headers */, + 510CF7572CFDB1B3005E1950 /* AudioEncoderCocoa.h in Headers */, FD31608312B026F700C1A359 /* AudioFileReader.h in Headers */, 46AAAA3D25D3632000BAF42F /* AudioFileReaderCocoa.h in Headers */, CD2F4A2418D89F700063746D /* AudioHardwareListener.h in Headers */, diff --git a/Source/WebCore/platform/AudioEncoder.cpp b/Source/WebCore/platform/AudioEncoder.cpp index 0b87dbad7cb06..163efcb01ecf6 100644 --- a/Source/WebCore/platform/AudioEncoder.cpp +++ b/Source/WebCore/platform/AudioEncoder.cpp @@ -33,30 +33,32 @@ #if USE(GSTREAMER) #include "AudioEncoderGStreamer.h" +#elif USE(AVFOUNDATION) +#include "AudioEncoderCocoa.h" #endif namespace WebCore { Ref AudioEncoder::create(const String& codecName, const Config& config, DescriptionCallback&& descriptionCallback, OutputCallback&& outputCallback) { +#if USE(GSTREAMER) CreatePromise::Producer producer; Ref promise = producer.promise(); CreateCallback callback = [producer = WTFMove(producer)] (auto&& result) mutable { producer.settle(WTFMove(result)); }; - -#if USE(GSTREAMER) GStreamerAudioEncoder::create(codecName, config, WTFMove(callback), WTFMove(descriptionCallback), WTFMove(outputCallback)); + return promise; +#elif USE(AVFOUNDATION) + return AudioEncoderCocoa::create(codecName, config, WTFMove(descriptionCallback), WTFMove(outputCallback)); #else UNUSED_PARAM(codecName); UNUSED_PARAM(config); UNUSED_PARAM(descriptionCallback); UNUSED_PARAM(outputCallback); - callback(makeUnexpected("Not supported"_s)); + return CreatePromise::createAndReject("Not supported"_s)); #endif - - return promise; } } // namespace WebCore diff --git a/Source/WebCore/platform/AudioEncoder.h b/Source/WebCore/platform/AudioEncoder.h index c31c07070c008..cc373432db160 100644 --- a/Source/WebCore/platform/AudioEncoder.h +++ b/Source/WebCore/platform/AudioEncoder.h @@ -31,6 +31,7 @@ #if ENABLE(WEB_CODECS) #include "AudioEncoderActiveConfiguration.h" +#include "BitrateMode.h" #include "WebCodecsAudioInternalData.h" #include #include @@ -46,7 +47,7 @@ class AudioEncoder : public ThreadSafeRefCounted { struct OpusConfig { bool isOggBitStream { false }; uint64_t frameDuration { 20000 }; - std::optional complexity; + std::optional complexity { }; size_t packetlossperc { 0 }; bool useinbandfec { false }; bool usedtx { false }; @@ -61,21 +62,22 @@ class AudioEncoder : public ThreadSafeRefCounted { size_t sampleRate; size_t numberOfChannels; uint64_t bitRate { 0 }; - std::optional opusConfig; - std::optional isAacADTS; - std::optional flacConfig; + BitrateMode bitRateMode { BitrateMode::Variable }; + std::optional opusConfig { }; + std::optional isAacADTS { }; + std::optional flacConfig { }; }; using ActiveConfiguration = AudioEncoderActiveConfiguration; struct EncodedFrame { Vector data; bool isKeyFrame { false }; int64_t timestamp { 0 }; - std::optional duration; + std::optional duration { }; }; struct RawFrame { RefPtr frame; int64_t timestamp { 0 }; - std::optional duration; + std::optional duration { }; }; using CreateResult = Expected, String>; using CreatePromise = NativePromise, String>; diff --git a/Source/WebCore/platform/AudioEncoderActiveConfiguration.h b/Source/WebCore/platform/AudioEncoderActiveConfiguration.h index b27a9e03e06aa..50da9a9babd6f 100644 --- a/Source/WebCore/platform/AudioEncoderActiveConfiguration.h +++ b/Source/WebCore/platform/AudioEncoderActiveConfiguration.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Apple Inc. All rights reserved. + * Copyright (C) 2023-2024 Apple Inc. All rights reserved. * Copyright (C) 2023 Igalia S.L * * Redistribution and use in source and binary forms, with or without @@ -34,10 +34,10 @@ namespace WebCore { struct AudioEncoderActiveConfiguration { String codec; - std::optional sampleRate; - std::optional numberOfChannels; - std::optional bitrate; - std::optional> description; + std::optional sampleRate { }; + std::optional numberOfChannels { }; + std::optional bitrate { }; + std::optional> description { }; }; } diff --git a/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.cpp b/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.cpp index 8dbedd1d43f4e..970395d7279a0 100644 --- a/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.cpp +++ b/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.cpp @@ -50,12 +50,12 @@ namespace WebCore { WTF_MAKE_TZONE_ALLOCATED_IMPL(AudioDecoderCocoa); -static WorkQueue& queueSingleton() +WorkQueue& AudioDecoderCocoa::queueSingleton() { static std::once_flag onceKey; static LazyNeverDestroyed> workQueue; std::call_once(onceKey, [] { - workQueue.construct(WorkQueue::create("AudioDecoder queue"_s)); + workQueue.construct(WorkQueue::create("WebCodecCocoa queue"_s)); }); return workQueue.get(); } @@ -81,8 +81,9 @@ class InternalAudioDecoderCocoa : public ThreadSafeRefCountedAndCanMakeThreadSaf private: InternalAudioDecoderCocoa(AudioDecoder::OutputCallback&&); + static WorkQueue& queueSingleton() { return AudioDecoderCocoa::queueSingleton(); } static void decompressedAudioOutputBufferCallback(void*, CMBufferQueueTriggerToken); - Ref converter() + Ref converter() const { assertIsCurrent(queueSingleton()); ASSERT(!m_isClosed); @@ -287,7 +288,9 @@ String InternalAudioDecoderCocoa::initialize(const String& codecName, const Audi m_inputFormatDescription = createAudioFormatDescription(*m_inputDescription, m_codecDescription.span()); - m_outputDescription = CAAudioStreamDescription { double(config.sampleRate), uint32_t(config.numberOfChannels), AudioStreamDescription::Float32, CAAudioStreamDescription::IsInterleaved::No }; + // FIXME: we choose to create interleaved AudioData as tests incorrectly requires it (planar is more compatible with WebAudio) + // https://github.com/w3c/webcodecs/issues/859 + m_outputDescription = CAAudioStreamDescription { double(config.sampleRate), uint32_t(config.numberOfChannels), AudioStreamDescription::Float32, CAAudioStreamDescription::IsInterleaved::Yes }; AudioSampleBufferConverter::Options options = { .format = kAudioFormatLinearPCM, diff --git a/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.h b/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.h index 8ea2a040702a6..2e22202609564 100644 --- a/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.h +++ b/Source/WebCore/platform/audio/cocoa/AudioDecoderCocoa.h @@ -33,6 +33,10 @@ #include #include +namespace WTF { +class WorkQueue; +} + namespace WebCore { class InternalAudioDecoderCocoa; @@ -47,6 +51,8 @@ class AudioDecoderCocoa final : public AudioDecoder { static Expected>, String> isCodecSupported(const StringView&); + static WTF::WorkQueue& queueSingleton(); + private: explicit AudioDecoderCocoa(OutputCallback&&); Ref decode(EncodedData&&) final; diff --git a/Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.cpp b/Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.cpp new file mode 100644 index 0000000000000..8c092036f0a33 --- /dev/null +++ b/Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.cpp @@ -0,0 +1,386 @@ +/* + * Copyright (C) 2024 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "AudioEncoderCocoa.h" + +#if ENABLE(WEB_CODECS) && USE(AVFOUNDATION) + +#include "AudioDecoderCocoa.h" +#include "AudioSampleBufferConverter.h" +#include "AudioSampleFormat.h" +#include "CAAudioStreamDescription.h" +#include "MediaSampleAVFObjC.h" +#include "MediaUtilities.h" +#include "PlatformRawAudioDataCocoa.h" +#include "WebMAudioUtilitiesCocoa.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace WebCore { + +WTF_MAKE_TZONE_ALLOCATED_IMPL(AudioEncoderCocoa); + +class InternalAudioEncoderCocoa : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr { + WTF_MAKE_TZONE_ALLOCATED_INLINE(InternalAudioEncoderCocoa); + WTF_MAKE_NONCOPYABLE(InternalAudioEncoderCocoa); + +public: + using Config = AudioEncoder::Config; + using EncodePromise = AudioEncoder::EncodePromise; + + static Ref create(const Config& config, AudioEncoder::DescriptionCallback&& descriptionCallback, AudioEncoder::OutputCallback&& outputCallback) + { + return adoptRef(*new InternalAudioEncoderCocoa(config, WTFMove(descriptionCallback), WTFMove(outputCallback))); + } + ~InternalAudioEncoderCocoa() = default; + + String initialize(const String& codecName); + Ref encode(AudioEncoder::RawFrame&&); + Ref flush(); + void reset(); + void close(); + + static WorkQueue& queueSingleton() { return AudioDecoderCocoa::queueSingleton(); } + +private: + InternalAudioEncoderCocoa(const AudioEncoder::Config&, AudioEncoder::DescriptionCallback&&, AudioEncoder::OutputCallback&&); + static void compressedAudioOutputBufferCallback(void*, CMBufferQueueTriggerToken); + Ref converter() const { return *m_converter; } + void processEncodedOutputs(); + AudioEncoder::ActiveConfiguration activeConfiguration(CMSampleBufferRef) const; + Vector generateDecoderDescriptionFromSample(CMSampleBufferRef) const; + + AudioEncoder::DescriptionCallback m_descriptionCallback; + AudioEncoder::OutputCallback m_outputCallback; + RefPtr m_converter; + + bool m_isClosed WTF_GUARDED_BY_CAPABILITY(queueSingleton()) { false }; + const AudioEncoder::Config m_config; + String m_codecString; // set during initialisation, const after. + FourCharCode m_codec; // set during initialisation, const after. + std::optional m_pcmFormat; // set during initialisation, const after. + std::optional m_outputDescription; // set during initialisation, const after. + std::optional m_frameDuration; // set during initialisation, const after. + std::optional m_complexity; // set during initialisation, const after. + std::optional m_packetlossperc; // set during initialisation, const after. + std::optional m_useinbandfec; // set during initialisation, const after. + std::optional m_inputDescription WTF_GUARDED_BY_CAPABILITY(queueSingleton()); + bool m_isADTS { false }; + bool m_hasProvidedDecoderConfig WTF_GUARDED_BY_CAPABILITY(queueSingleton()) { false }; + bool m_needsFlushing WTF_GUARDED_BY_CAPABILITY(queueSingleton()) { false }; +}; + +Ref AudioEncoderCocoa::create(const String& codecName, const AudioEncoder::Config& config, DescriptionCallback&& descriptionCallback, OutputCallback&& outputCallback) +{ + Ref encoder = adoptRef(*new AudioEncoderCocoa(config, WTFMove(descriptionCallback), WTFMove(outputCallback))); + auto internalEncoder = encoder->m_internalEncoder; + auto error = internalEncoder->initialize(codecName); + if (!error.isEmpty()) + return CreatePromise::createAndReject(makeString("AudioEncoder initialization failed with error: "_s, error)); + return CreatePromise::createAndResolve(WTFMove(encoder)); +} + +AudioEncoderCocoa::AudioEncoderCocoa(const AudioEncoder::Config& config, DescriptionCallback&& descriptionCallback, OutputCallback&& outputCallback) + : m_internalEncoder(InternalAudioEncoderCocoa::create(config, WTFMove(descriptionCallback), WTFMove(outputCallback))) +{ +} + +AudioEncoderCocoa::~AudioEncoderCocoa() +{ + // We need to ensure the internal decoder is closed and the audio converter finished. + InternalAudioEncoderCocoa::queueSingleton().dispatch([encoder = m_internalEncoder] { + encoder->close(); + }); +} + +Ref AudioEncoderCocoa::encode(RawFrame&& frame) +{ + return invokeAsync(InternalAudioEncoderCocoa::queueSingleton(), [frame = WTFMove(frame), encoder = m_internalEncoder]() mutable { + return encoder->encode(WTFMove(frame)); + }); +} + +Ref AudioEncoderCocoa::flush() +{ + return invokeAsync(InternalAudioEncoderCocoa::queueSingleton(), [encoder = m_internalEncoder] { + return encoder->flush(); + }); +} + +void AudioEncoderCocoa::reset() +{ + InternalAudioEncoderCocoa::queueSingleton().dispatch([encoder = m_internalEncoder] { + encoder->close(); + }); +} + +void AudioEncoderCocoa::close() +{ + InternalAudioEncoderCocoa::queueSingleton().dispatch([encoder = m_internalEncoder] { + encoder->close(); + }); +} + +InternalAudioEncoderCocoa::InternalAudioEncoderCocoa(const Config& config, AudioEncoder::DescriptionCallback&& descriptionCallback, AudioEncoder::OutputCallback&& outputCallback) + : m_descriptionCallback(WTFMove(descriptionCallback)) + , m_outputCallback(WTFMove(outputCallback)) + , m_config(config) +{ +} + +String InternalAudioEncoderCocoa::initialize(const String& codecName) +{ + auto result = AudioDecoderCocoa::isCodecSupported(codecName); + if (!result) + return result.error(); + + std::tie(m_codec, m_pcmFormat) = *result; + if (m_codec == 'vorb') + return "Vorbis encoding is not supported"_s; + if (m_codec == kAudioFormatFLAC) + return "FLAC encoding is not supported"_s; + if (m_codec == kAudioFormatMPEGLayer3) + return "MP3 encoding is not supported"_s; + + if (m_codec == kAudioFormatOpus) { + if (m_config.numberOfChannels > 2) + return "Opus with more than two channels is not supported"_s; + + if (m_config.opusConfig && m_config.opusConfig->isOggBitStream) + return "Opus Ogg format is unsupported"_s; + + auto config = m_config.opusConfig.value_or(AudioEncoder::OpusConfig { }); + m_frameDuration = MediaTime(config.frameDuration, 1000000); + m_complexity = config.complexity; + m_packetlossperc = config.packetlossperc; + m_useinbandfec = config.useinbandfec; + } else if ((m_codec == kAudioFormatMPEG4AAC || m_codec == kAudioFormatMPEG4AAC_HE || m_codec == kAudioFormatMPEG4AAC_LD || m_codec == kAudioFormatMPEG4AAC_HE_V2 || m_codec == kAudioFormatMPEG4AAC_ELD)) { + m_isADTS = m_config.isAacADTS.value_or(false); + if (m_isADTS) + return "AAC ADTS format is unsupported"_s; + } + + if (m_pcmFormat) + m_outputDescription = CAAudioStreamDescription { double(m_config.sampleRate), uint32_t(m_config.numberOfChannels), *m_pcmFormat, CAAudioStreamDescription::IsInterleaved::Yes }; + else { + AudioStreamBasicDescription asbd { }; + asbd.mFormatID = m_codec; + asbd.mSampleRate = double(m_config.sampleRate); + asbd.mChannelsPerFrame = uint32_t(m_config.numberOfChannels); + if (m_frameDuration) + asbd.mFramesPerPacket = m_config.sampleRate * m_frameDuration->toDouble(); + m_outputDescription = asbd; + } + m_codecString = codecName.isolatedCopy(); + + return emptyString(); +} + +void InternalAudioEncoderCocoa::compressedAudioOutputBufferCallback(void* object, CMBufferQueueTriggerToken) +{ + // We can only be called from the CoreMedia callback if we are still alive. + RefPtr encoder = static_cast(object); + + InternalAudioEncoderCocoa::queueSingleton().dispatch([weakEncoder = ThreadSafeWeakPtr { *encoder }] { + if (auto strongEncoder = weakEncoder.get()) + strongEncoder->processEncodedOutputs(); + }); +} + +Vector InternalAudioEncoderCocoa::generateDecoderDescriptionFromSample(CMSampleBufferRef sample) const +{ + RetainPtr formatDescription = PAL::CMSampleBufferGetFormatDescription(sample); + ASSERT(formatDescription); + const AudioStreamBasicDescription* const asbd = PAL::CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription.get()); + if (!asbd) + return { }; + + if (asbd->mFormatID == kAudioFormatOpus) + return createOpusPrivateData(*asbd, m_converter->preSkip()); + + size_t cookieSize = 0; + auto* cookie = PAL::CMAudioFormatDescriptionGetMagicCookie(formatDescription.get(), &cookieSize); + if (!cookieSize) + return { }; + return unsafeMakeSpan(static_cast(cookie), cookieSize); +} + +AudioEncoder::ActiveConfiguration InternalAudioEncoderCocoa::activeConfiguration(CMSampleBufferRef sample) const +{ + assertIsCurrent(queueSingleton()); + ASSERT(!m_isClosed && m_converter); + + RetainPtr formatDescription = PAL::CMSampleBufferGetFormatDescription(sample); + ASSERT(formatDescription); + const AudioStreamBasicDescription* const asbd = PAL::CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription.get()); + if (!asbd) + return { }; + + return AudioEncoder::ActiveConfiguration { + .codec = m_codecString, + .sampleRate = asbd->mSampleRate, + .numberOfChannels = asbd->mChannelsPerFrame, + .bitrate = converter()->bitRate(), + .description = generateDecoderDescriptionFromSample(sample) + }; +} + +void InternalAudioEncoderCocoa::processEncodedOutputs() +{ + assertIsCurrent(queueSingleton()); + + if (m_isClosed) + return; + + while (RetainPtr cmSample = converter()->takeOutputSampleBuffer()) { + Ref sample = MediaSampleAVFObjC::create(cmSample.get(), 0); + CMBlockBufferRef rawBuffer = PAL::CMSampleBufferGetDataBuffer(cmSample.get()); + ASSERT(rawBuffer); + RetainPtr buffer = rawBuffer; + // Make sure block buffer is contiguous. + if (!PAL::CMBlockBufferIsRangeContiguous(rawBuffer, 0, 0)) { + CMBlockBufferRef contiguousBuffer; + if (auto error = PAL::CMBlockBufferCreateContiguous(nullptr, rawBuffer, nullptr, nullptr, 0, 0, 0, &contiguousBuffer)) { + RELEASE_LOG_ERROR(MediaStream, "Couldn't create buffer with error %d", error); + continue; + } + buffer = adoptCF(contiguousBuffer); + } + auto size = PAL::CMBlockBufferGetDataLength(buffer.get()); + char* data = nullptr; + if (auto error = PAL::CMBlockBufferGetDataPointer(buffer.get(), 0, nullptr, nullptr, &data)) { + RELEASE_LOG_ERROR(MediaStream, "Couldn't create buffer with error %d", error); + continue; + } + + std::optional duration; + if (auto sampleDuration = sample->duration(); sampleDuration.isValid()) + duration = sampleDuration.toMicroseconds(); + + AudioEncoder::EncodedFrame encodedFrame { + .data = unsafeMakeSpan(byteCast(data), size), + .isKeyFrame = sample->isSync(), + .timestamp = sample->presentationTime().toMicroseconds(), + .duration = duration + }; + + if (!m_hasProvidedDecoderConfig) { + m_hasProvidedDecoderConfig = true; + m_descriptionCallback(activeConfiguration(cmSample.get())); + } + m_outputCallback({ WTFMove(encodedFrame) }); + } +} + +Ref InternalAudioEncoderCocoa::encode(AudioEncoder::RawFrame&& rawFrame) +{ + assertIsCurrent(queueSingleton()); + + RetainPtr cmSample = downcast(rawFrame.frame)->sampleBuffer(); + ASSERT(cmSample); + if (auto error = PAL::CMSampleBufferSetOutputPresentationTimeStamp(cmSample.get(), PAL::CMTimeMake(rawFrame.timestamp, 1000000))) + RELEASE_LOG_ERROR(MediaStream, "AudioSampleBufferConverter CMSampleBufferSetOutputPresentationTimeStamp failed with %d", error); + + CMFormatDescriptionRef formatDescription = PAL::CMSampleBufferGetFormatDescription(cmSample.get()); + if (!formatDescription) + return EncodePromise::createAndReject("Couldn't retrieve AudioData's format description"_s); + const AudioStreamBasicDescription* const asbd = PAL::CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription); + if (!asbd) + return EncodePromise::createAndReject("Couldn't retrieve AudioData's basic description"_s); + + if (*asbd != m_inputDescription) { + if (m_converter) + converter()->finish(); + m_inputDescription = *asbd; + + AudioSampleBufferConverter::Options options = { + .format = m_codec, + .description = m_outputDescription->streamDescription(), + .outputBitRate = m_config.bitRate ? std::optional { m_config.bitRate } : std::nullopt, + .generateTimestamp = false, + .bitrateMode = m_config.bitRateMode, + .complexity = m_complexity, + .packetlossperc = m_packetlossperc, + .useinbandfec = m_useinbandfec + }; + m_converter = AudioSampleBufferConverter::create(compressedAudioOutputBufferCallback, this, options); + if (!m_converter) { + RELEASE_LOG_ERROR(MediaStream, "InternalAudioEncoderCocoa::encode: creation of converter failed"); + return EncodePromise::createAndReject("InternalAudioEncoderCocoa::encode: creation of converter failed"_s); + } + } + + m_needsFlushing = true; + + ASSERT(m_converter); + AudioEncoder::EncodePromise::Producer producer; + Ref promise = producer.promise(); + converter()->addSampleBuffer(cmSample.get())->whenSettled(queueSingleton(), [producer = WTFMove(producer)](auto result) mutable { + if (result) + producer.resolve(); + else + producer.reject("InternalAudioEncoderCocoa encoding failed"_s); + }); + return promise; +} + +Ref InternalAudioEncoderCocoa::flush() +{ + assertIsCurrent(queueSingleton()); + + if (!m_converter || !m_needsFlushing) + return GenericPromise::createAndResolve(); // No frame encoded yet, nothing to flush; + return converter()->drain()->whenSettled(queueSingleton(), [protectedThis = Ref { *this }] { + assertIsCurrent(queueSingleton()); + protectedThis->processEncodedOutputs(); + protectedThis->m_needsFlushing = false; + return GenericPromise::createAndResolve(); + }); +} + +void InternalAudioEncoderCocoa::close() +{ + assertIsCurrent(queueSingleton()); + + m_isClosed = true; + RefPtr converter = std::exchange(m_converter, { }); + if (!converter) + return; + // We keep a reference to ourselves until the converter has been marked as finished. This guarantees that no + // callback will occur after we are destructed. + converter->finish()->whenSettled(queueSingleton(), [protectedThis = Ref { *this }] { }); +} + +} // namespace WebCore + +#endif // ENABLE(WEB_CODECS) && USE(AVFOUNDATION) diff --git a/Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.h b/Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.h new file mode 100644 index 0000000000000..94d1077330df2 --- /dev/null +++ b/Source/WebCore/platform/audio/cocoa/AudioEncoderCocoa.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2024 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(WEB_CODECS) && USE(AVFOUNDATION) + +#include "AudioEncoder.h" + +#include +#include + +namespace WebCore { + +class InternalAudioEncoderCocoa; + +class AudioEncoderCocoa : public AudioEncoder { + WTF_MAKE_TZONE_ALLOCATED(AudioEncoderCocoa); +public: + static Ref create(const String& codecName, const Config&, DescriptionCallback&&, OutputCallback&&); + + ~AudioEncoderCocoa(); + +private: + AudioEncoderCocoa(const AudioEncoder::Config&, DescriptionCallback&&, OutputCallback&&); + + Ref encode(RawFrame&&) final; + Ref flush() final; + void reset() final; + void close() final; + + Ref m_internalEncoder; +}; + +} // namespace WebCore + +#endif // ENABLE(WEB_CODECS) && USE(GSTREAMER)