From 9502a0769941f2811a9737cd77a2d964d1035f5d Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Thu, 12 Dec 2024 13:15:28 +1100 Subject: [PATCH] [WebCodec|AudioDecoder] Limit the number of codec operations we can enqueue https://bugs.webkit.org/show_bug.cgi?id=284448 rdar://141272065 Reviewed by NOBODY (OOPS!). While the WebCodecs specs clearly allows for boundless queue for calculating "Codec Saturdation", a side effect is that both decodeQueueSize and encodeQueueSize can only ever be observed with a size of 0. The value of decodeQueueSize and encodeQueueSize is used by some site (including W3C's WebCodecs's own example) as a way to limit how much in advance it will attempt to encode or decode. Both Chrome and Firefox have a limit on how many codecs operations (1 for Firefox, between 1 and 8 for Chrome) they will enqueue concurrently. To minimise web compatibility issue, in addition to lodging a spec bug (https://github.com/w3c/webcodecs/issues/864) we also introduce a limit on how many frames with submit to the internal decoder/encoder before submitting more: - 1 for Audio - 4 for Video. In order of not having to replicate 4 times the code in {Audio|Video}{Encoder|Decoder} we create a new WebCodecsBase that handles all the control message queue operations as well as handle how many codec operations have been submitted, and of which all WebCodecs inherit from. Doing so allows to remove a lot of similar code across all webcodecs and get us closer to the verbiage of the specs. Another benefit is that WebCodecsControlMessage no longer needs to be a templated class, and we make it return a "Processed" or "Not Processed" value as the specs does. No change in observable behaviours with existing WPT. Tested that it allows the WebCodecs reference player to work. No new tests as none of the work above is per spec. * Source/WebCore/Headers.cmake: * Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp: (WebCore::WebCodecsAudioDecoder::WebCodecsAudioDecoder): (WebCore::WebCodecsAudioDecoder::configure): (WebCore::WebCodecsAudioDecoder::decode): (WebCore::WebCodecsAudioDecoder::flush): (WebCore::WebCodecsAudioDecoder::closeDecoder): (WebCore::WebCodecsAudioDecoder::resetDecoder): (WebCore::WebCodecsAudioDecoder::stop): (WebCore::WebCodecsAudioDecoder::scheduleDequeueEvent): Deleted. (WebCore::WebCodecsAudioDecoder::queueControlMessageAndProcess): Deleted. (WebCore::WebCodecsAudioDecoder::processControlMessageQueue): Deleted. (WebCore::WebCodecsAudioDecoder::virtualHasPendingActivity const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.h: (WebCore::WebCodecsAudioDecoder::decodeQueueSize const): (WebCore::WebCodecsAudioDecoder::state const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp: (WebCore::WebCodecsAudioEncoder::WebCodecsAudioEncoder): (WebCore::WebCodecsAudioEncoder::configure): (WebCore::WebCodecsAudioEncoder::encode): (WebCore::WebCodecsAudioEncoder::flush): (WebCore::WebCodecsAudioEncoder::closeEncoder): (WebCore::WebCodecsAudioEncoder::resetEncoder): (WebCore::WebCodecsAudioEncoder::stop): (WebCore::WebCodecsAudioEncoder::scheduleDequeueEvent): Deleted. (WebCore::WebCodecsAudioEncoder::queueControlMessageAndProcess): Deleted. (WebCore::WebCodecsAudioEncoder::processControlMessageQueue): Deleted. (WebCore::WebCodecsAudioEncoder::virtualHasPendingActivity const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.h: (WebCore::WebCodecsAudioEncoder::encodeQueueSize const): (WebCore::WebCodecsAudioEncoder::state const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsBase.cpp: Added. (WebCore::WebCodecsBase::WebCodecsBase): (WebCore::WebCodecsBase::queueControlMessageAndProcess): (WebCore::WebCodecsBase::queueControlMessageForCodecOperationAndProcess): (WebCore::WebCodecsBase::scheduleDequeueEvent): (WebCore::WebCodecsBase::processControlMessageQueue): (WebCore::WebCodecsBase::incrementOperationQueueSize): (WebCore::WebCodecsBase::decrementOperationQueueSizeAndScheduleDequeueEvent): (WebCore::WebCodecsBase::decreaseCodecOperationCountAndMaybeProcessControlMessageQueue): (WebCore::WebCodecsBase::clearControlMessageQueue): (WebCore::WebCodecsBase::clearControlMessageQueueAndMaybeScheduleDequeueEvent): (WebCore::WebCodecsBase::blockControlMessageQueue): (WebCore::WebCodecsBase::unblockControlMessageQueue): (WebCore::WebCodecsBase::virtualHasPendingActivity const): * Source/WebCore/Modules/webcodecs/WebCodecsBase.h: Copied from Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.h. (WebCore::WebCodecsBase::state const): (WebCore::WebCodecsBase::setState): (WebCore::WebCodecsBase::operationQueueSize const): (WebCore::WebCodecsBase::maximumCodecOperationsEnqueued const): (WebCore::WebCodecsBase::increaseCodecOperationCount): (WebCore::WebCodecsBase::isCodecSaturated const): * Source/WebCore/Modules/webcodecs/WebCodecsControlMessage.h: * Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp: (WebCore::WebCodecsVideoDecoder::WebCodecsVideoDecoder): (WebCore::WebCodecsVideoDecoder::configure): (WebCore::WebCodecsVideoDecoder::decode): (WebCore::WebCodecsVideoDecoder::flush): (WebCore::WebCodecsVideoDecoder::closeDecoder): (WebCore::WebCodecsVideoDecoder::resetDecoder): (WebCore::WebCodecsVideoDecoder::stop): (WebCore::WebCodecsVideoDecoder::scheduleDequeueEvent): Deleted. (WebCore::WebCodecsVideoDecoder::queueControlMessageAndProcess): Deleted. (WebCore::WebCodecsVideoDecoder::processControlMessageQueue): Deleted. (WebCore::WebCodecsVideoDecoder::virtualHasPendingActivity const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.h: (WebCore::WebCodecsVideoDecoder::decodeQueueSize const): (WebCore::WebCodecsVideoDecoder::state const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp: (WebCore::WebCodecsVideoEncoder::WebCodecsVideoEncoder): (WebCore::WebCodecsVideoEncoder::updateRates): (WebCore::WebCodecsVideoEncoder::configure): (WebCore::WebCodecsVideoEncoder::encode): (WebCore::WebCodecsVideoEncoder::flush): (WebCore::WebCodecsVideoEncoder::closeEncoder): (WebCore::WebCodecsVideoEncoder::resetEncoder): (WebCore::WebCodecsVideoEncoder::stop): (WebCore::WebCodecsVideoEncoder::scheduleDequeueEvent): Deleted. (WebCore::WebCodecsVideoEncoder::queueControlMessageAndProcess): Deleted. (WebCore::WebCodecsVideoEncoder::processControlMessageQueue): Deleted. (WebCore::WebCodecsVideoEncoder::virtualHasPendingActivity const): Deleted. * Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h: (WebCore::WebCodecsVideoEncoder::encodeQueueSize const): (WebCore::WebCodecsVideoEncoder::state const): Deleted. * Source/WebCore/Sources.txt: * Source/WebCore/WebCore.xcodeproj/project.pbxproj: --- Source/WebCore/Headers.cmake | 1 + .../webcodecs/WebCodecsAudioDecoder.cpp | 94 ++++-------- .../Modules/webcodecs/WebCodecsAudioDecoder.h | 31 +--- .../webcodecs/WebCodecsAudioEncoder.cpp | 110 +++++--------- .../Modules/webcodecs/WebCodecsAudioEncoder.h | 29 +--- .../Modules/webcodecs/WebCodecsBase.cpp | 140 ++++++++++++++++++ .../WebCore/Modules/webcodecs/WebCodecsBase.h | 102 +++++++++++++ .../webcodecs/WebCodecsControlMessage.h | 18 ++- .../webcodecs/WebCodecsVideoDecoder.cpp | 95 ++++-------- .../Modules/webcodecs/WebCodecsVideoDecoder.h | 32 +--- .../webcodecs/WebCodecsVideoEncoder.cpp | 119 +++++---------- .../Modules/webcodecs/WebCodecsVideoEncoder.h | 30 +--- Source/WebCore/Sources.txt | 1 + .../WebCore/WebCore.xcodeproj/project.pbxproj | 6 + 14 files changed, 407 insertions(+), 401 deletions(-) create mode 100644 Source/WebCore/Modules/webcodecs/WebCodecsBase.cpp create mode 100644 Source/WebCore/Modules/webcodecs/WebCodecsBase.h diff --git a/Source/WebCore/Headers.cmake b/Source/WebCore/Headers.cmake index 1fc435a1c233a..5be9e17020922 100644 --- a/Source/WebCore/Headers.cmake +++ b/Source/WebCore/Headers.cmake @@ -649,6 +649,7 @@ set(WebCore_PRIVATE_FRAMEWORK_HEADERS Modules/webcodecs/WebCodecsAlphaOption.h Modules/webcodecs/WebCodecsAudioData.h Modules/webcodecs/WebCodecsAudioInternalData.h + Modules/webcodecs/WebCodecsBase.h Modules/webcodecs/WebCodecsEncodedAudioChunk.h Modules/webcodecs/WebCodecsEncodedAudioChunkData.h Modules/webcodecs/WebCodecsEncodedAudioChunkType.h diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp index 56bdc0109fe92..37da0c63c27f9 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp @@ -32,15 +32,16 @@ #include "ContextDestructionObserverInlines.h" #include "DOMException.h" #include "Event.h" -#include "EventNames.h" #include "JSDOMPromiseDeferred.h" #include "JSWebCodecsAudioDecoderSupport.h" #include "ScriptExecutionContext.h" #include "WebCodecsAudioData.h" #include "WebCodecsAudioDataOutputCallback.h" +#include "WebCodecsControlMessage.h" #include "WebCodecsEncodedAudioChunk.h" #include "WebCodecsErrorCallback.h" #include "WebCodecsUtilities.h" + #include WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN @@ -57,7 +58,7 @@ Ref WebCodecsAudioDecoder::create(ScriptExecutionContext& } WebCodecsAudioDecoder::WebCodecsAudioDecoder(ScriptExecutionContext& context, Init&& init) - : ActiveDOMObject(&context) + : WebCodecsBase(context) , m_output(init.output.releaseNonNull()) , m_error(init.error.releaseNonNull()) { @@ -98,26 +99,26 @@ ExceptionOr WebCodecsAudioDecoder::configure(ScriptExecutionContext&, WebC if (!isValidDecoderConfig(config)) return Exception { ExceptionCode::TypeError, "Config is not valid"_s }; - if (m_state == WebCodecsCodecState::Closed || !scriptExecutionContext()) + if (state() == WebCodecsCodecState::Closed || !scriptExecutionContext()) return Exception { ExceptionCode::InvalidStateError, "AudioDecoder is closed"_s }; - m_state = WebCodecsCodecState::Configured; + setState(WebCodecsCodecState::Configured); m_isKeyChunkRequired = true; bool isSupportedCodec = AudioDecoder::isCodecSupported(config.codec); queueControlMessageAndProcess({ *this, [this, config = WTFMove(config), isSupportedCodec, identifier = scriptExecutionContext()->identifier()]() mutable { - m_isMessageQueueBlocked = true; + blockControlMessageQueue(); if (!isSupportedCodec) { postTaskToCodec(identifier, *this, [] (auto& decoder) { decoder.closeDecoder(Exception { ExceptionCode::NotSupportedError, "Codec is not supported"_s }); }); - return; + return WebCodecsControlMessageOutcome::Processed; } Ref createDecoderPromise = AudioDecoder::create(config.codec, createAudioDecoderConfig(config), [identifier, weakThis = ThreadSafeWeakPtr { *this }, decoderCount = ++m_decoderCount] (auto&& result) { postTaskToCodec(identifier, weakThis, [result = WTFMove(result), decoderCount] (auto& decoder) mutable { - if (decoder.m_state != WebCodecsCodecState::Configured || decoder.m_decoderCount != decoderCount) + if (decoder.state() != WebCodecsCodecState::Configured || decoder.m_decoderCount != decoderCount) return; if (!result.has_value()) { @@ -142,16 +143,16 @@ ExceptionOr WebCodecsAudioDecoder::configure(ScriptExecutionContext&, WebC } protectedThis->setInternalDecoder(WTFMove(*result)); - protectedThis->m_isMessageQueueBlocked = false; - protectedThis->processControlMessageQueue(); + protectedThis->unblockControlMessageQueue(); }); + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } ExceptionOr WebCodecsAudioDecoder::decode(Ref&& chunk) { - if (m_state != WebCodecsCodecState::Configured) + if (state() != WebCodecsCodecState::Configured) return Exception { ExceptionCode::InvalidStateError, "AudioDecoder is not configured"_s }; if (m_isKeyChunkRequired) { @@ -160,25 +161,27 @@ ExceptionOr WebCodecsAudioDecoder::decode(Ref& m_isKeyChunkRequired = false; } - ++m_decodeQueueSize; - queueControlMessageAndProcess({ *this, [this, chunk = WTFMove(chunk)]() mutable { - --m_decodeQueueSize; - scheduleDequeueEvent(); - + queueControlMessageForCodecOperationAndProcess({ *this, [this, chunk = WTFMove(chunk)]() mutable { + increaseCodecOperationCount(); protectedScriptExecutionContext()->enqueueTaskWhenSettled(Ref { *m_internalDecoder }->decode({ chunk->span(), chunk->type() == WebCodecsEncodedAudioChunkType::Key, chunk->timestamp(), chunk->duration() }), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = makePendingActivity(*this)] (auto&& result) { RefPtr protectedThis = weakThis.get(); - if (!protectedThis || !!result) + if (!protectedThis) return; - protectedThis->closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + if (!result) + protectedThis->closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + else + protectedThis->decreaseCodecOperationCountAndMaybeProcessControlMessageQueue(); }); + + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } ExceptionOr WebCodecsAudioDecoder::flush(Ref&& promise) { - if (m_state != WebCodecsCodecState::Configured) + if (state() != WebCodecsCodecState::Configured) return Exception { ExceptionCode::InvalidStateError, "AudioDecoder is not configured"_s }; m_isKeyChunkRequired = true; @@ -189,6 +192,7 @@ ExceptionOr WebCodecsAudioDecoder::flush(Ref&& promise) if (RefPtr protectedThis = weakThis.get()) protectedThis->m_pendingFlushPromises.removeFirstMatching([&](auto& flushPromise) { return promise.ptr() == flushPromise.ptr(); }); }); + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } @@ -226,7 +230,7 @@ ExceptionOr WebCodecsAudioDecoder::closeDecoder(Exception&& exception) auto result = resetDecoder(exception); if (result.hasException()) return result; - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalDecoder = nullptr; if (exception.code() != ExceptionCode::AbortError) m_error->handleEvent(DOMException::create(WTFMove(exception))); @@ -236,17 +240,13 @@ ExceptionOr WebCodecsAudioDecoder::closeDecoder(Exception&& exception) ExceptionOr WebCodecsAudioDecoder::resetDecoder(const Exception& exception) { - if (m_state == WebCodecsCodecState::Closed) + if (state() == WebCodecsCodecState::Closed) return Exception { ExceptionCode::InvalidStateError, "AudioDecoder is closed"_s }; - m_state = WebCodecsCodecState::Unconfigured; + setState(WebCodecsCodecState::Unconfigured); if (RefPtr internalDecoder = std::exchange(m_internalDecoder, { })) internalDecoder->reset(); - m_controlMessageQueue.clear(); - if (m_decodeQueueSize) { - m_decodeQueueSize = 0; - scheduleDequeueEvent(); - } + clearControlMessageQueueAndMaybeScheduleDequeueEvent(); auto promises = std::exchange(m_pendingFlushPromises, { }); for (auto& promise : promises) @@ -255,61 +255,23 @@ ExceptionOr WebCodecsAudioDecoder::resetDecoder(const Exception& exception return { }; } -void WebCodecsAudioDecoder::scheduleDequeueEvent() -{ - if (m_dequeueEventScheduled) - return; - - m_dequeueEventScheduled = true; - queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { - dispatchEvent(Event::create(eventNames().dequeueEvent, Event::CanBubble::No, Event::IsCancelable::No)); - m_dequeueEventScheduled = false; - }); -} - void WebCodecsAudioDecoder::setInternalDecoder(Ref&& internalDecoder) { m_internalDecoder = WTFMove(internalDecoder); } -void WebCodecsAudioDecoder::queueControlMessageAndProcess(WebCodecsControlMessage&& message) -{ - if (m_isMessageQueueBlocked) { - m_controlMessageQueue.append(WTFMove(message)); - return; - } - if (m_controlMessageQueue.isEmpty()) { - message(); - return; - } - - m_controlMessageQueue.append(WTFMove(message)); - processControlMessageQueue(); -} - -void WebCodecsAudioDecoder::processControlMessageQueue() -{ - while (!m_isMessageQueueBlocked && !m_controlMessageQueue.isEmpty()) - m_controlMessageQueue.takeFirst()(); -} - void WebCore::WebCodecsAudioDecoder::suspend(ReasonForSuspension) { } void WebCodecsAudioDecoder::stop() { - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalDecoder = nullptr; - m_controlMessageQueue.clear(); + clearControlMessageQueue(); m_pendingFlushPromises.clear(); } -bool WebCodecsAudioDecoder::virtualHasPendingActivity() const -{ - return m_state == WebCodecsCodecState::Configured && (m_decodeQueueSize || m_isMessageQueueBlocked); -} - } // namespace WebCore WTF_ALLOW_UNSAFE_BUFFER_USAGE_END diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.h b/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.h index 910ae946dcddd..1ab9bd187390c 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.h +++ b/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.h @@ -28,16 +28,12 @@ #if ENABLE(WEB_CODECS) -#include "ActiveDOMObject.h" #include "AudioDecoder.h" -#include "EventTarget.h" #include "JSDOMPromiseDeferredForward.h" #include "WebCodecsAudioDecoderConfig.h" #include "WebCodecsAudioDecoderSupport.h" -#include "WebCodecsCodecState.h" -#include "WebCodecsControlMessage.h" +#include "WebCodecsBase.h" #include "WebCodecsEncodedAudioChunkType.h" -#include #include namespace WebCore { @@ -46,10 +42,7 @@ class WebCodecsEncodedAudioChunk; class WebCodecsErrorCallback; class WebCodecsAudioDataOutputCallback; -class WebCodecsAudioDecoder - : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr - , public ActiveDOMObject - , public EventTarget { +class WebCodecsAudioDecoder : public WebCodecsBase { WTF_MAKE_TZONE_OR_ISO_ALLOCATED(WebCodecsAudioDecoder); public: ~WebCodecsAudioDecoder(); @@ -61,8 +54,7 @@ class WebCodecsAudioDecoder static Ref create(ScriptExecutionContext&, Init&&); - WebCodecsCodecState state() const { return m_state; } - size_t decodeQueueSize() const { return m_decodeQueueSize; } + size_t decodeQueueSize() const { return operationQueueSize(); } ExceptionOr configure(ScriptExecutionContext&, WebCodecsAudioDecoderConfig&&); ExceptionOr decode(Ref&&); @@ -72,10 +64,6 @@ class WebCodecsAudioDecoder static void isConfigSupported(ScriptExecutionContext&, WebCodecsAudioDecoderConfig&&, Ref&&); - // ActiveDOMObject. - void ref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::ref(); } - void deref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::deref(); } - WebCodecsAudioDataOutputCallback& outputCallbackConcurrently() { return m_output.get(); } WebCodecsErrorCallback& errorCallbackConcurrently() { return m_error.get(); } @@ -85,32 +73,19 @@ class WebCodecsAudioDecoder // ActiveDOMObject. void stop() final; void suspend(ReasonForSuspension) final; - bool virtualHasPendingActivity() const final; // EventTarget - void refEventTarget() final { ref(); } - void derefEventTarget() final { deref(); } enum EventTargetInterfaceType eventTargetInterface() const final { return EventTargetInterfaceType::WebCodecsAudioDecoder; } - ScriptExecutionContext* scriptExecutionContext() const final { return ActiveDOMObject::scriptExecutionContext(); } ExceptionOr closeDecoder(Exception&&); ExceptionOr resetDecoder(const Exception&); void setInternalDecoder(Ref&&); - void scheduleDequeueEvent(); - - void queueControlMessageAndProcess(WebCodecsControlMessage&&); - void processControlMessageQueue(); - WebCodecsCodecState m_state { WebCodecsCodecState::Unconfigured }; - size_t m_decodeQueueSize { 0 }; Ref m_output; Ref m_error; RefPtr m_internalDecoder; - bool m_dequeueEventScheduled { false }; Vector> m_pendingFlushPromises; bool m_isKeyChunkRequired { false }; - Deque> m_controlMessageQueue; - bool m_isMessageQueueBlocked { false }; size_t m_decoderCount { 0 }; }; diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp index 68a16cb563a25..84a3b85da46ea 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp @@ -33,7 +33,6 @@ #include "ContextDestructionObserverInlines.h" #include "DOMException.h" #include "Event.h" -#include "EventNames.h" #include "FlacEncoderConfig.h" #include "JSDOMPromiseDeferred.h" #include "JSWebCodecsAudioEncoderSupport.h" @@ -42,6 +41,7 @@ #include "SecurityOrigin.h" #include "WebCodecsAudioData.h" #include "WebCodecsAudioEncoderConfig.h" +#include "WebCodecsControlMessage.h" #include "WebCodecsEncodedAudioChunk.h" #include "WebCodecsEncodedAudioChunkMetadata.h" #include "WebCodecsEncodedAudioChunkOutputCallback.h" @@ -64,7 +64,7 @@ Ref WebCodecsAudioEncoder::create(ScriptExecutionContext& WebCodecsAudioEncoder::WebCodecsAudioEncoder(ScriptExecutionContext& context, Init&& init) - : ActiveDOMObject(&context) + : WebCodecsBase(context) , m_output(init.output.releaseNonNull()) , m_error(init.error.releaseNonNull()) { @@ -165,27 +165,27 @@ ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebC if (!isValidEncoderConfig(config)) return Exception { ExceptionCode::TypeError, "Config is invalid"_s }; - if (m_state == WebCodecsCodecState::Closed || !scriptExecutionContext()) + if (state() == WebCodecsCodecState::Closed || !scriptExecutionContext()) return Exception { ExceptionCode::InvalidStateError, "AudioEncoder is closed"_s }; - m_state = WebCodecsCodecState::Configured; + setState(WebCodecsCodecState::Configured); m_isKeyChunkRequired = true; if (m_internalEncoder) { queueControlMessageAndProcess({ *this, [this, config]() mutable { - m_isMessageQueueBlocked = true; + blockControlMessageQueue(); protectedScriptExecutionContext()->enqueueTaskWhenSettled(Ref { *m_internalEncoder }->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, config = WTFMove(config)] (auto&&) mutable { RefPtr protectedThis = weakThis.get(); if (!protectedThis) return; - if (protectedThis->m_state == WebCodecsCodecState::Closed || !protectedThis->scriptExecutionContext()) + if (protectedThis->state() == WebCodecsCodecState::Closed || !protectedThis->scriptExecutionContext()) return; - protectedThis->m_isMessageQueueBlocked = false; - protectedThis->processControlMessageQueue(); + protectedThis->unblockControlMessageQueue(); }); + return WebCodecsControlMessageOutcome::Processed; } }); } @@ -193,12 +193,12 @@ ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebC queueControlMessageAndProcess({ *this, [this, config = WTFMove(config), isSupportedCodec, identifier = scriptExecutionContext()->identifier()]() mutable { RefPtr context = scriptExecutionContext(); - m_isMessageQueueBlocked = true; + blockControlMessageQueue(); if (!isSupportedCodec) { postTaskToCodec(identifier, *this, [] (auto& encoder) { encoder.closeEncoder(Exception { ExceptionCode::NotSupportedError, "Codec is not supported"_s }); }); - return; + return WebCodecsControlMessageOutcome::Processed; } auto encoderConfig = createAudioEncoderConfig(config); @@ -206,7 +206,7 @@ ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebC postTaskToCodec(identifier, *this, [message = encoderConfig.releaseException().message()] (auto& encoder) mutable { encoder.closeEncoder(Exception { ExceptionCode::NotSupportedError, WTFMove(message) }); }); - return; + return WebCodecsControlMessageOutcome::Processed; } m_baseConfiguration = config; @@ -218,7 +218,7 @@ ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebC }); }, [identifier, weakThis = ThreadSafeWeakPtr { *this }, encoderCount = ++m_encoderCount] (auto&& result) { postTaskToCodec(identifier, weakThis, [result = WTFMove(result), encoderCount] (auto& encoder) mutable { - if (encoder.m_state != WebCodecsCodecState::Configured || encoder.m_encoderCount != encoderCount) + if (encoder.state() != WebCodecsCodecState::Configured || encoder.m_encoderCount != encoderCount) return; RefPtr buffer = JSC::ArrayBuffer::create(result.data); @@ -242,10 +242,11 @@ ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebC return; } protectedThis->setInternalEncoder(WTFMove(*result)); - protectedThis->m_isMessageQueueBlocked = false; protectedThis->m_hasNewActiveConfiguration = true; - protectedThis->processControlMessageQueue(); + protectedThis->unblockControlMessageQueue(); }); + + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } @@ -288,39 +289,41 @@ ExceptionOr WebCodecsAudioEncoder::encode(Ref&& frame) } ASSERT(!frame->isDetached()); - if (m_state != WebCodecsCodecState::Configured) + if (state() != WebCodecsCodecState::Configured) return Exception { ExceptionCode::InvalidStateError, "AudioEncoder is not configured"_s }; - ++m_encodeQueueSize; - queueControlMessageAndProcess({ *this, [this, audioData = WTFMove(audioData), timestamp = frame->timestamp(), duration = frame->duration()]() mutable { - --m_encodeQueueSize; - scheduleDequeueEvent(); - + queueControlMessageForCodecOperationAndProcess({ *this, [this, audioData = WTFMove(audioData), timestamp = frame->timestamp(), duration = frame->duration()]() mutable { // FIXME: These checks are not yet spec-compliant. See also https://github.com/w3c/webcodecs/issues/716 if ((m_activeConfiguration.numberOfChannels && *m_activeConfiguration.numberOfChannels != audioData->numberOfChannels()) || (m_activeConfiguration.sampleRate && *m_activeConfiguration.sampleRate != audioData->sampleRate())) { queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { closeEncoder(Exception { ExceptionCode::EncodingError, "Input audio buffer is incompatible with codec parameters"_s }); }); - return; + return WebCodecsControlMessageOutcome::Processed; } + increaseCodecOperationCount(); protectedScriptExecutionContext()->enqueueTaskWhenSettled(Ref { *m_internalEncoder }->encode({ WTFMove(audioData), timestamp, duration }), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = makePendingActivity(*this)] (auto&& result) { RefPtr protectedThis = weakThis.get(); - if (!protectedThis || !!result) + if (!protectedThis) return; - if (auto context = protectedThis->protectedScriptExecutionContext()) - context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("AudioEncoder encode failed: "_s, result.error())); - protectedThis->closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + if (!result) { + if (auto context = protectedThis->protectedScriptExecutionContext()) + context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("AudioEncoder encode failed: "_s, result.error())); + protectedThis->closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + return; + } + protectedThis->decreaseCodecOperationCountAndMaybeProcessControlMessageQueue(); }); + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } void WebCodecsAudioEncoder::flush(Ref&& promise) { - if (m_state != WebCodecsCodecState::Configured) { + if (state() != WebCodecsCodecState::Configured) { promise->reject(Exception { ExceptionCode::InvalidStateError, "AudioEncoder is not configured"_s }); return; } @@ -332,6 +335,7 @@ void WebCodecsAudioEncoder::flush(Ref&& promise) if (RefPtr protectedThis = weakThis.get()) protectedThis->m_pendingFlushPromises.removeFirstMatching([&](auto& flushPromise) { return promise.ptr() == flushPromise.ptr(); }); }); + return WebCodecsControlMessageOutcome::Processed; } }); } @@ -374,7 +378,7 @@ ExceptionOr WebCodecsAudioEncoder::closeEncoder(Exception&& exception) auto result = resetEncoder(exception); if (result.hasException()) return result; - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalEncoder = nullptr; if (exception.code() != ExceptionCode::AbortError) m_error->handleEvent(DOMException::create(WTFMove(exception))); @@ -384,17 +388,13 @@ ExceptionOr WebCodecsAudioEncoder::closeEncoder(Exception&& exception) ExceptionOr WebCodecsAudioEncoder::resetEncoder(const Exception& exception) { - if (m_state == WebCodecsCodecState::Closed) + if (state() == WebCodecsCodecState::Closed) return Exception { ExceptionCode::InvalidStateError, "AudioEncoder is closed"_s }; - m_state = WebCodecsCodecState::Unconfigured; + setState(WebCodecsCodecState::Unconfigured); if (RefPtr internalEncoder = std::exchange(m_internalEncoder, { })) internalEncoder->reset(); - m_controlMessageQueue.clear(); - if (m_encodeQueueSize) { - m_encodeQueueSize = 0; - scheduleDequeueEvent(); - } + clearControlMessageQueueAndMaybeScheduleDequeueEvent(); auto promises = std::exchange(m_pendingFlushPromises, { }); for (auto& promise : promises) @@ -403,61 +403,23 @@ ExceptionOr WebCodecsAudioEncoder::resetEncoder(const Exception& exception return { }; } -void WebCodecsAudioEncoder::scheduleDequeueEvent() -{ - if (m_dequeueEventScheduled) - return; - - m_dequeueEventScheduled = true; - queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { - dispatchEvent(Event::create(eventNames().dequeueEvent, Event::CanBubble::No, Event::IsCancelable::No)); - m_dequeueEventScheduled = false; - }); -} - void WebCodecsAudioEncoder::setInternalEncoder(Ref&& internalEncoder) { m_internalEncoder = WTFMove(internalEncoder); } -void WebCodecsAudioEncoder::queueControlMessageAndProcess(WebCodecsControlMessage&& message) -{ - if (m_isMessageQueueBlocked) { - m_controlMessageQueue.append(WTFMove(message)); - return; - } - if (m_controlMessageQueue.isEmpty()) { - message(); - return; - } - - m_controlMessageQueue.append(WTFMove(message)); - processControlMessageQueue(); -} - -void WebCodecsAudioEncoder::processControlMessageQueue() -{ - while (!m_isMessageQueueBlocked && !m_controlMessageQueue.isEmpty()) - m_controlMessageQueue.takeFirst()(); -} - void WebCore::WebCodecsAudioEncoder::suspend(ReasonForSuspension) { } void WebCodecsAudioEncoder::stop() { - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalEncoder = nullptr; - m_controlMessageQueue.clear(); + clearControlMessageQueue(); m_pendingFlushPromises.clear(); } -bool WebCodecsAudioEncoder::virtualHasPendingActivity() const -{ - return m_state == WebCodecsCodecState::Configured && (m_encodeQueueSize || m_isMessageQueueBlocked); -} - } // namespace WebCore #endif // ENABLE(WEB_CODECS) diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.h b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.h index 1415504f899dd..a72e62b53f6a6 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.h +++ b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.h @@ -28,14 +28,10 @@ #if ENABLE(WEB_CODECS) -#include "ActiveDOMObject.h" #include "AudioEncoder.h" -#include "EventTarget.h" #include "JSDOMPromiseDeferredForward.h" #include "WebCodecsAudioEncoderConfig.h" -#include "WebCodecsCodecState.h" -#include "WebCodecsControlMessage.h" -#include +#include "WebCodecsBase.h" #include namespace WebCore { @@ -46,10 +42,7 @@ class WebCodecsEncodedAudioChunkOutputCallback; class WebCodecsAudioData; struct WebCodecsEncodedAudioChunkMetadata; -class WebCodecsAudioEncoder - : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr - , public ActiveDOMObject - , public EventTarget { +class WebCodecsAudioEncoder : public WebCodecsBase { WTF_MAKE_TZONE_OR_ISO_ALLOCATED(WebCodecsAudioEncoder); public: ~WebCodecsAudioEncoder(); @@ -61,8 +54,7 @@ class WebCodecsAudioEncoder static Ref create(ScriptExecutionContext&, Init&&); - WebCodecsCodecState state() const { return m_state; } - size_t encodeQueueSize() const { return m_encodeQueueSize; } + size_t encodeQueueSize() const { return operationQueueSize(); } ExceptionOr configure(ScriptExecutionContext&, WebCodecsAudioEncoderConfig&&); ExceptionOr encode(Ref&&); @@ -72,10 +64,6 @@ class WebCodecsAudioEncoder static void isConfigSupported(ScriptExecutionContext&, WebCodecsAudioEncoderConfig&&, Ref&&); - // ActiveDOMObject. - void ref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::ref(); } - void deref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::deref(); } - WebCodecsEncodedAudioChunkOutputCallback& outputCallbackConcurrently() { return m_output.get(); } WebCodecsErrorCallback& errorCallbackConcurrently() { return m_error.get(); } private: @@ -84,33 +72,22 @@ class WebCodecsAudioEncoder // ActiveDOMObject. void stop() final; void suspend(ReasonForSuspension) final; - bool virtualHasPendingActivity() const final; // EventTarget. - void refEventTarget() final { ref(); } - void derefEventTarget() final { deref(); } enum EventTargetInterfaceType eventTargetInterface() const final { return EventTargetInterfaceType::WebCodecsAudioEncoder; } - ScriptExecutionContext* scriptExecutionContext() const final { return ActiveDOMObject::scriptExecutionContext(); } ExceptionOr closeEncoder(Exception&&); ExceptionOr resetEncoder(const Exception&); void setInternalEncoder(Ref&&); - void scheduleDequeueEvent(); - void queueControlMessageAndProcess(WebCodecsControlMessage&&); - void processControlMessageQueue(); WebCodecsEncodedAudioChunkMetadata createEncodedChunkMetadata(); - WebCodecsCodecState m_state { WebCodecsCodecState::Unconfigured }; size_t m_encodeQueueSize { 0 }; Ref m_output; Ref m_error; RefPtr m_internalEncoder; - bool m_dequeueEventScheduled { false }; Vector> m_pendingFlushPromises; bool m_isKeyChunkRequired { false }; - Deque> m_controlMessageQueue; - bool m_isMessageQueueBlocked { false }; WebCodecsAudioEncoderConfig m_baseConfiguration; AudioEncoder::ActiveConfiguration m_activeConfiguration; bool m_hasNewActiveConfiguration { false }; diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsBase.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsBase.cpp new file mode 100644 index 0000000000000..78012745f9d90 --- /dev/null +++ b/Source/WebCore/Modules/webcodecs/WebCodecsBase.cpp @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2024 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "WebCodecsBase.h" + +#include "EventNames.h" +#include "WebCodecsControlMessage.h" + +#include + +namespace WebCore { + +WTF_MAKE_TZONE_OR_ISO_ALLOCATED_IMPL(WebCodecsBase); + +WebCodecsBase::WebCodecsBase(ScriptExecutionContext& context) + : ActiveDOMObject(&context) +{ +} + +WebCodecsBase::~WebCodecsBase() = default; + +void WebCodecsBase::queueControlMessageAndProcess(WebCodecsControlMessage&& message) +{ + if (m_isMessageQueueBlocked) { + m_controlMessageQueue.append(WTFMove(message)); + return; + } + m_controlMessageQueue.append(WTFMove(message)); + processControlMessageQueue(); +} + +void WebCodecsBase::queueControlMessageForCodecOperationAndProcess(WebCodecsControlMessage&& message) +{ + incrementOperationQueueSize(); + // message holds a strong ref to ourselves already. + queueControlMessageAndProcess({ *this, [this, message = WTFMove(message)]() mutable { + if (isCodecSaturated()) + return WebCodecsControlMessageOutcome::NotProcessed; + decrementOperationQueueSizeAndScheduleDequeueEvent(); + return message(); + } }); +} + +void WebCodecsBase::scheduleDequeueEvent() +{ + if (m_dequeueEventScheduled) + return; + + m_dequeueEventScheduled = true; + queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { + dispatchEvent(Event::create(eventNames().dequeueEvent, Event::CanBubble::No, Event::IsCancelable::No)); + m_dequeueEventScheduled = false; + }); +} + +void WebCodecsBase::processControlMessageQueue() +{ + while (!m_isMessageQueueBlocked && !m_controlMessageQueue.isEmpty()) { + auto& frontMessage = m_controlMessageQueue.first(); + auto outcome = frontMessage(); + if (outcome == WebCodecsControlMessageOutcome::NotProcessed) + break; + m_controlMessageQueue.removeFirst(); + } +} + +void WebCodecsBase::incrementOperationQueueSize() +{ + m_operationQueueSize++; +} + +// Equivalent to spec's "Decrement [[encodeQueueSize]] or "Decrement [[decodeQueueSize]]" and run the Schedule Dequeue Event algorithm" +void WebCodecsBase::decrementOperationQueueSizeAndScheduleDequeueEvent() +{ + m_operationQueueSize--; + scheduleDequeueEvent(); +} + +void WebCodecsBase::decreaseCodecOperationCountAndMaybeProcessControlMessageQueue() +{ + ASSERT(m_codecOperationsEnqueued > 0); + m_codecOperationsEnqueued--; + if (!isCodecSaturated()) + processControlMessageQueue(); +} + +void WebCodecsBase::clearControlMessageQueue() +{ + m_controlMessageQueue.clear(); +} + +void WebCodecsBase::clearControlMessageQueueAndMaybeScheduleDequeueEvent() +{ + clearControlMessageQueue(); + if (m_operationQueueSize) { + m_operationQueueSize = 0; + scheduleDequeueEvent(); + } +} + +void WebCodecsBase::blockControlMessageQueue() +{ + m_isMessageQueueBlocked = true; +} + +void WebCodecsBase::unblockControlMessageQueue() +{ + m_isMessageQueueBlocked = false; + processControlMessageQueue(); +} + +bool WebCodecsBase::virtualHasPendingActivity() const +{ + return m_state == WebCodecsCodecState::Configured && (m_operationQueueSize || m_isMessageQueueBlocked); +} + +} // namespace WebCore diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsBase.h b/Source/WebCore/Modules/webcodecs/WebCodecsBase.h new file mode 100644 index 0000000000000..c4aa6f73220a9 --- /dev/null +++ b/Source/WebCore/Modules/webcodecs/WebCodecsBase.h @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2024 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if ENABLE(WEB_CODECS) + +#include "ActiveDOMObject.h" +#include "EventTarget.h" +#include "WebCodecsCodecState.h" +#include +#include +#include + +namespace WebCore { + +class WebCodecsControlMessage; + +// WebCodecsBase implements the "Control Message Queue" +// as per https://w3c.github.io/webcodecs/#control-message-queue-slot +// And handle "Codec Saturation" +// as per https://w3c.github.io/webcodecs/#saturated +class WebCodecsBase + : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr + , public ActiveDOMObject + , public EventTarget { + WTF_MAKE_TZONE_OR_ISO_ALLOCATED(WebCodecsBase); +public: + virtual ~WebCodecsBase(); + + WebCodecsCodecState state() const { return m_state; } + + // ActiveDOMObject. + void ref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::ref(); } + void deref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::deref(); } + bool virtualHasPendingActivity() const final; + +protected: + WebCodecsBase(ScriptExecutionContext&); + ScriptExecutionContext* scriptExecutionContext() const final { return ActiveDOMObject::scriptExecutionContext(); } + + void setState(WebCodecsCodecState state) { m_state = state; } + + size_t operationQueueSize() const { return m_operationQueueSize; } + // Equivalent to spec's "Increment [[encodeQueueSize]]." or "Increment [[decodeQueueSize]]" + void incrementOperationQueueSize(); + // Equivalent to spec's "Decrement [[encodeQueueSize]] or "Decrement [[decodeQueueSize]]" and run the Schedule Dequeue Event algorithm" + void decrementOperationQueueSizeAndScheduleDequeueEvent(); + + void queueControlMessageAndProcess(WebCodecsControlMessage&&); + void queueControlMessageForCodecOperationAndProcess(WebCodecsControlMessage&&); + void processControlMessageQueue(); + void clearControlMessageQueue(); + void clearControlMessageQueueAndMaybeScheduleDequeueEvent(); + void blockControlMessageQueue(); + void unblockControlMessageQueue(); + + virtual size_t maximumCodecOperationsEnqueued() const { return 1; } + void increaseCodecOperationCount() { m_codecOperationsEnqueued++; }; + void decreaseCodecOperationCountAndMaybeProcessControlMessageQueue(); + +private: + // EventTarget + void refEventTarget() final { ref(); } + void derefEventTarget() final { deref(); } + + bool isCodecSaturated() const { return m_codecOperationsEnqueued >= maximumCodecOperationsEnqueued(); } + void scheduleDequeueEvent(); + + bool m_isMessageQueueBlocked = false; + size_t m_operationQueueSize { 0 }; + size_t m_codecOperationsEnqueued { 0 }; + bool m_dequeueEventScheduled { false }; + Deque m_controlMessageQueue; + WebCodecsCodecState m_state { WebCodecsCodecState::Unconfigured }; +}; + +} // namespace WebCore + +#endif // ENABLE(WEB_CODECS) diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsControlMessage.h b/Source/WebCore/Modules/webcodecs/WebCodecsControlMessage.h index 4fbc757848895..7054432e0ae3d 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsControlMessage.h +++ b/Source/WebCore/Modules/webcodecs/WebCodecsControlMessage.h @@ -27,15 +27,19 @@ #if ENABLE(WEB_CODECS) -#include "ActiveDOMObject.h" +#include "WebCodecsBase.h" #include namespace WebCore { -template +enum class WebCodecsControlMessageOutcome : bool { + NotProcessed, + Processed +}; + class WebCodecsControlMessage final { public: - WebCodecsControlMessage(CodecType& codec, Function&& message) + WebCodecsControlMessage(WebCodecsBase& codec, Function&& message) : m_pendingActivity(codec.makePendingActivity(codec)) , m_message(WTFMove(message)) { @@ -47,14 +51,14 @@ class WebCodecsControlMessage final { { } - void operator()() + WebCodecsControlMessageOutcome operator()() { - m_message(); + return m_message(); } private: - Ref> m_pendingActivity; - Function m_message; + Ref> m_pendingActivity; + Function m_message; }; } diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp index d6139cdfa0660..b7d29fbd824b0 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp @@ -32,7 +32,6 @@ #include "ContextDestructionObserverInlines.h" #include "DOMException.h" #include "Event.h" -#include "EventNames.h" #include "HTMLCanvasElement.h" #include "HTMLImageElement.h" #include "HTMLVideoElement.h" @@ -42,12 +41,13 @@ #include "OffscreenCanvas.h" #include "SVGImageElement.h" #include "ScriptExecutionContext.h" -#include +#include "WebCodecsControlMessage.h" #include "WebCodecsEncodedVideoChunk.h" #include "WebCodecsErrorCallback.h" #include "WebCodecsUtilities.h" #include "WebCodecsVideoFrame.h" #include "WebCodecsVideoFrameOutputCallback.h" +#include #include #include @@ -65,7 +65,7 @@ Ref WebCodecsVideoDecoder::create(ScriptExecutionContext& } WebCodecsVideoDecoder::WebCodecsVideoDecoder(ScriptExecutionContext& context, Init&& init) - : ActiveDOMObject(&context) + : WebCodecsBase(context) , m_output(init.output.releaseNonNull()) , m_error(init.error.releaseNonNull()) { @@ -137,10 +137,10 @@ ExceptionOr WebCodecsVideoDecoder::configure(ScriptExecutionContext& conte if (!isValidDecoderConfig(config)) return Exception { ExceptionCode::TypeError, "Config is not valid"_s }; - if (m_state == WebCodecsCodecState::Closed || !scriptExecutionContext()) + if (state() == WebCodecsCodecState::Closed || !scriptExecutionContext()) return Exception { ExceptionCode::InvalidStateError, "VideoDecoder is closed"_s }; - m_state = WebCodecsCodecState::Configured; + setState(WebCodecsCodecState::Configured); m_isKeyChunkRequired = true; bool isSupportedCodec = isSupportedDecoderCodec(config.codec, context.settingsValues()); @@ -149,17 +149,17 @@ ExceptionOr WebCodecsVideoDecoder::configure(ScriptExecutionContext& conte auto identifier = context->identifier(); - m_isMessageQueueBlocked = true; + blockControlMessageQueue(); if (!isSupportedCodec) { postTaskToCodec(identifier, *this, [] (auto& decoder) { decoder.closeDecoder(Exception { ExceptionCode::NotSupportedError, "Codec is not supported"_s }); }); - return; + return WebCodecsControlMessageOutcome::Processed; } Ref createDecoderPromise = VideoDecoder::create(config.codec, createVideoDecoderConfig(config), [identifier, weakThis = ThreadSafeWeakPtr { *this }, decoderCount = ++m_decoderCount] (auto&& result) { postTaskToCodec(identifier, weakThis, [result = WTFMove(result), decoderCount] (auto& decoder) mutable { - if (decoder.m_state != WebCodecsCodecState::Configured || decoder.m_decoderCount != decoderCount) + if (decoder.state() != WebCodecsCodecState::Configured || decoder.m_decoderCount != decoderCount) return; if (!result) { @@ -189,16 +189,17 @@ ExceptionOr WebCodecsVideoDecoder::configure(ScriptExecutionContext& conte return; } protectedThis->setInternalDecoder(WTFMove(*result)); - protectedThis->m_isMessageQueueBlocked = false; - protectedThis->processControlMessageQueue(); + protectedThis->unblockControlMessageQueue(); }); + + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } ExceptionOr WebCodecsVideoDecoder::decode(Ref&& chunk) { - if (m_state != WebCodecsCodecState::Configured) + if (state() != WebCodecsCodecState::Configured) return Exception { ExceptionCode::InvalidStateError, "VideoDecoder is not configured"_s }; if (m_isKeyChunkRequired) { @@ -207,26 +208,27 @@ ExceptionOr WebCodecsVideoDecoder::decode(Ref& m_isKeyChunkRequired = false; } - ++m_decodeQueueSize; - queueControlMessageAndProcess({ *this, [this, chunk = WTFMove(chunk)]() mutable { - --m_decodeQueueSize; - scheduleDequeueEvent(); - + queueControlMessageForCodecOperationAndProcess({ *this, [this, chunk = WTFMove(chunk)]() mutable { + increaseCodecOperationCount(); Ref internalDecoder = *m_internalDecoder; protectedScriptExecutionContext()->enqueueTaskWhenSettled(internalDecoder->decode({ chunk->span(), chunk->type() == WebCodecsEncodedVideoChunkType::Key, chunk->timestamp(), chunk->duration() }), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { * this }, pendingActivity = makePendingActivity(*this)] (auto&& result) { RefPtr protectedThis = weakThis.get(); - if (!protectedThis || !!result) + if (!protectedThis) return; - protectedThis->closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + if (!result) + protectedThis->closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + else + protectedThis->decreaseCodecOperationCountAndMaybeProcessControlMessageQueue(); }); + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } ExceptionOr WebCodecsVideoDecoder::flush(Ref&& promise) { - if (m_state != WebCodecsCodecState::Configured) + if (state() != WebCodecsCodecState::Configured) return Exception { ExceptionCode::InvalidStateError, "VideoDecoder is not configured"_s }; m_isKeyChunkRequired = true; @@ -238,6 +240,7 @@ ExceptionOr WebCodecsVideoDecoder::flush(Ref&& promise) if (RefPtr protectedThis = weakThis.get()) protectedThis->m_pendingFlushPromises.removeFirstMatching([&](auto& flushPromise) { return promise.ptr() == flushPromise.ptr(); }); }); + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } @@ -275,7 +278,7 @@ ExceptionOr WebCodecsVideoDecoder::closeDecoder(Exception&& exception) auto result = resetDecoder(exception); if (result.hasException()) return result; - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalDecoder = nullptr; if (exception.code() != ExceptionCode::AbortError) m_error->handleEvent(DOMException::create(WTFMove(exception))); @@ -285,17 +288,13 @@ ExceptionOr WebCodecsVideoDecoder::closeDecoder(Exception&& exception) ExceptionOr WebCodecsVideoDecoder::resetDecoder(const Exception& exception) { - if (m_state == WebCodecsCodecState::Closed) + if (state() == WebCodecsCodecState::Closed) return Exception { ExceptionCode::InvalidStateError, "VideoDecoder is closed"_s }; - m_state = WebCodecsCodecState::Unconfigured; + setState(WebCodecsCodecState::Unconfigured); if (RefPtr internalDecoder = std::exchange(m_internalDecoder, { })) internalDecoder->reset(); - m_controlMessageQueue.clear(); - if (m_decodeQueueSize) { - m_decodeQueueSize = 0; - scheduleDequeueEvent(); - } + clearControlMessageQueueAndMaybeScheduleDequeueEvent(); auto promises = std::exchange(m_pendingFlushPromises, { }); for (auto& promise : promises) @@ -304,61 +303,23 @@ ExceptionOr WebCodecsVideoDecoder::resetDecoder(const Exception& exception return { }; } -void WebCodecsVideoDecoder::scheduleDequeueEvent() -{ - if (m_dequeueEventScheduled) - return; - - m_dequeueEventScheduled = true; - queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { - dispatchEvent(Event::create(eventNames().dequeueEvent, Event::CanBubble::No, Event::IsCancelable::No)); - m_dequeueEventScheduled = false; - }); -} - void WebCodecsVideoDecoder::setInternalDecoder(Ref&& internalDecoder) { m_internalDecoder = WTFMove(internalDecoder); } -void WebCodecsVideoDecoder::queueControlMessageAndProcess(WebCodecsControlMessage&& message) -{ - if (m_isMessageQueueBlocked) { - m_controlMessageQueue.append(WTFMove(message)); - return; - } - if (m_controlMessageQueue.isEmpty()) { - message(); - return; - } - - m_controlMessageQueue.append(WTFMove(message)); - processControlMessageQueue(); -} - -void WebCodecsVideoDecoder::processControlMessageQueue() -{ - while (!m_isMessageQueueBlocked && !m_controlMessageQueue.isEmpty()) - m_controlMessageQueue.takeFirst()(); -} - void WebCore::WebCodecsVideoDecoder::suspend(ReasonForSuspension) { } void WebCodecsVideoDecoder::stop() { - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalDecoder = nullptr; - m_controlMessageQueue.clear(); + clearControlMessageQueue(); m_pendingFlushPromises.clear(); } -bool WebCodecsVideoDecoder::virtualHasPendingActivity() const -{ - return m_state == WebCodecsCodecState::Configured && (m_decodeQueueSize || m_isMessageQueueBlocked); -} - } // namespace WebCore WTF_ALLOW_UNSAFE_BUFFER_USAGE_END diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.h b/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.h index 5a33379bc95e4..4ae444b614187 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.h +++ b/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.h @@ -27,15 +27,11 @@ #if ENABLE(WEB_CODECS) -#include "ActiveDOMObject.h" -#include "EventTarget.h" #include "JSDOMPromiseDeferredForward.h" #include "VideoDecoder.h" -#include "WebCodecsCodecState.h" -#include "WebCodecsControlMessage.h" +#include "WebCodecsBase.h" #include "WebCodecsEncodedVideoChunkType.h" #include "WebCodecsVideoDecoderSupport.h" -#include #include namespace WebCore { @@ -44,10 +40,7 @@ class WebCodecsEncodedVideoChunk; class WebCodecsErrorCallback; class WebCodecsVideoFrameOutputCallback; -class WebCodecsVideoDecoder - : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr - , public ActiveDOMObject - , public EventTarget { +class WebCodecsVideoDecoder : public WebCodecsBase { WTF_MAKE_TZONE_OR_ISO_ALLOCATED(WebCodecsVideoDecoder); public: ~WebCodecsVideoDecoder(); @@ -59,8 +52,7 @@ class WebCodecsVideoDecoder static Ref create(ScriptExecutionContext&, Init&&); - WebCodecsCodecState state() const { return m_state; } - size_t decodeQueueSize() const { return m_decodeQueueSize; } + size_t decodeQueueSize() const { return operationQueueSize(); } WebCodecsVideoFrameOutputCallback& outputCallbackConcurrently() { return m_output.get(); } WebCodecsErrorCallback& errorCallbackConcurrently() { return m_error.get(); } @@ -73,42 +65,26 @@ class WebCodecsVideoDecoder static void isConfigSupported(ScriptExecutionContext&, WebCodecsVideoDecoderConfig&&, Ref&&); - // ActiveDOMObject. - void ref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::ref(); } - void deref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::deref(); } - private: WebCodecsVideoDecoder(ScriptExecutionContext&, Init&&); + size_t maximumCodecOperationsEnqueued() const final { return 4; } // ActiveDOMObject. void stop() final; void suspend(ReasonForSuspension) final; - bool virtualHasPendingActivity() const final; // EventTarget. - void refEventTarget() final { ref(); } - void derefEventTarget() final { deref(); } enum EventTargetInterfaceType eventTargetInterface() const final { return EventTargetInterfaceType::WebCodecsVideoDecoder; } - ScriptExecutionContext* scriptExecutionContext() const final { return ActiveDOMObject::scriptExecutionContext(); } ExceptionOr closeDecoder(Exception&&); ExceptionOr resetDecoder(const Exception&); void setInternalDecoder(Ref&&); - void scheduleDequeueEvent(); - - void queueControlMessageAndProcess(WebCodecsControlMessage&&); - void processControlMessageQueue(); - WebCodecsCodecState m_state { WebCodecsCodecState::Unconfigured }; - size_t m_decodeQueueSize { 0 }; Ref m_output; Ref m_error; RefPtr m_internalDecoder; - bool m_dequeueEventScheduled { false }; Vector> m_pendingFlushPromises; bool m_isKeyChunkRequired { false }; - Deque> m_controlMessageQueue; - bool m_isMessageQueueBlocked { false }; size_t m_decoderCount { 0 }; }; diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp index 238115264f856..1e488c006d23e 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp @@ -31,10 +31,10 @@ #include "ContextDestructionObserverInlines.h" #include "DOMException.h" #include "Event.h" -#include "EventNames.h" #include "JSDOMPromiseDeferred.h" #include "JSWebCodecsVideoEncoderSupport.h" #include "Logging.h" +#include "WebCodecsControlMessage.h" #include "WebCodecsEncodedVideoChunk.h" #include "WebCodecsEncodedVideoChunkMetadata.h" #include "WebCodecsEncodedVideoChunkOutputCallback.h" @@ -60,7 +60,7 @@ Ref WebCodecsVideoEncoder::create(ScriptExecutionContext& WebCodecsVideoEncoder::WebCodecsVideoEncoder(ScriptExecutionContext& context, Init&& init) - : ActiveDOMObject(&context) + : WebCodecsBase(context) , m_output(init.output.releaseNonNull()) , m_error(init.error.releaseNonNull()) { @@ -119,21 +119,20 @@ void WebCodecsVideoEncoder::updateRates(const WebCodecsVideoEncoderConfig& confi auto bitrate = config.bitrate.value_or(0); auto framerate = config.framerate.value_or(0); - m_isMessageQueueBlocked = true; + blockControlMessageQueue(); protectedScriptExecutionContext()->enqueueTaskWhenSettled(Ref { *m_internalEncoder }->setRates(bitrate, framerate), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, bitrate, framerate] (auto&&) mutable { auto protectedThis = weakThis.get(); if (!protectedThis) return; - if (protectedThis->m_state == WebCodecsCodecState::Closed || !protectedThis->scriptExecutionContext()) + if (protectedThis->state() == WebCodecsCodecState::Closed || !protectedThis->scriptExecutionContext()) return; if (bitrate) protectedThis->m_baseConfiguration.bitrate = bitrate; if (framerate) protectedThis->m_baseConfiguration.framerate = framerate; - protectedThis->m_isMessageQueueBlocked = false; - protectedThis->processControlMessageQueue(); + protectedThis->unblockControlMessageQueue(); }); } @@ -142,31 +141,31 @@ ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& conte if (!isValidEncoderConfig(config)) return Exception { ExceptionCode::TypeError, "Config is invalid"_s }; - if (m_state == WebCodecsCodecState::Closed || !scriptExecutionContext()) + if (state() == WebCodecsCodecState::Closed || !scriptExecutionContext()) return Exception { ExceptionCode::InvalidStateError, "VideoEncoder is closed"_s }; - m_state = WebCodecsCodecState::Configured; + setState(WebCodecsCodecState::Configured); m_isKeyChunkRequired = true; if (m_internalEncoder) { queueControlMessageAndProcess({ *this, [this, config]() mutable { if (isSameConfigurationExceptBitrateAndFramerate(m_baseConfiguration, config)) { updateRates(config); - return; + return WebCodecsControlMessageOutcome::Processed; } - m_isMessageQueueBlocked = true; + blockControlMessageQueue(); protectedScriptExecutionContext()->enqueueTaskWhenSettled(Ref { *m_internalEncoder }->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, config = WTFMove(config), pendingActivity = makePendingActivity(*this)] (auto&&) mutable { RefPtr protectedThis = weakThis.get(); if (!protectedThis) return; - if (protectedThis->m_state == WebCodecsCodecState::Closed || !protectedThis->scriptExecutionContext()) + if (protectedThis->state() == WebCodecsCodecState::Closed || !protectedThis->scriptExecutionContext()) return; - protectedThis->m_isMessageQueueBlocked = false; - protectedThis->processControlMessageQueue(); + protectedThis->unblockControlMessageQueue(); }); + return WebCodecsControlMessageOutcome::Processed; } }); } @@ -174,18 +173,18 @@ ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& conte queueControlMessageAndProcess({ *this, [this, config = WTFMove(config), isSupportedCodec]() mutable { if (isSupportedCodec && isSameConfigurationExceptBitrateAndFramerate(m_baseConfiguration, config)) { updateRates(config); - return; + return WebCodecsControlMessageOutcome::Processed; } auto identifier = scriptExecutionContext()->identifier(); - m_isMessageQueueBlocked = true; + blockControlMessageQueue(); if (!isSupportedCodec) { postTaskToCodec(identifier, *this, [] (auto& encoder) { encoder.closeEncoder(Exception { ExceptionCode::NotSupportedError, "Codec is not supported"_s }); }); - return; + return WebCodecsControlMessageOutcome::Processed; } auto encoderConfig = createVideoEncoderConfig(config); @@ -193,7 +192,7 @@ ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& conte postTaskToCodec(identifier, *this, [message = encoderConfig.releaseException().message()] (auto& encoder) mutable { encoder.closeEncoder(Exception { ExceptionCode::NotSupportedError, WTFMove(message) }); }); - return; + return WebCodecsControlMessageOutcome::Processed; } m_baseConfiguration = config; @@ -205,7 +204,7 @@ ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& conte }); }, [identifier, weakThis = ThreadSafeWeakPtr { *this }, encoderCount = ++m_encoderCount](auto&& result) { postTaskToCodec(identifier, weakThis, [result = WTFMove(result), encoderCount] (auto& encoder) mutable { - if (encoder.m_state != WebCodecsCodecState::Configured || encoder.m_encoderCount != encoderCount) + if (encoder.state() != WebCodecsCodecState::Configured || encoder.m_encoderCount != encoderCount) return; RefPtr buffer = JSC::ArrayBuffer::create(result.data); @@ -229,9 +228,10 @@ ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& conte } protectedThis->setInternalEncoder(WTFMove(*result)); protectedThis->m_hasNewActiveConfiguration = true; - protectedThis->m_isMessageQueueBlocked = false; - protectedThis->processControlMessageQueue(); + protectedThis->unblockControlMessageQueue(); }); + + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } @@ -281,30 +281,32 @@ ExceptionOr WebCodecsVideoEncoder::encode(Ref&& frame } ASSERT(!frame->isDetached()); - if (m_state != WebCodecsCodecState::Configured) + if (state() != WebCodecsCodecState::Configured) return Exception { ExceptionCode::InvalidStateError, "VideoEncoder is not configured"_s }; - ++m_encodeQueueSize; - queueControlMessageAndProcess({ *this, [this, internalFrame = internalFrame.releaseNonNull(), timestamp = frame->timestamp(), duration = frame->duration(), options = WTFMove(options)]() mutable { - --m_encodeQueueSize; - scheduleDequeueEvent(); - + queueControlMessageForCodecOperationAndProcess({ *this, [this, internalFrame = internalFrame.releaseNonNull(), timestamp = frame->timestamp(), duration = frame->duration(), options = WTFMove(options)]() mutable { + increaseCodecOperationCount(); protectedScriptExecutionContext()->enqueueTaskWhenSettled(Ref { *m_internalEncoder }->encode({ WTFMove(internalFrame), timestamp, duration }, options.keyFrame), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = makePendingActivity(*this)] (auto&& result) { RefPtr protectedThis = weakThis.get(); - if (!protectedThis || !!result) + if (!protectedThis) return; - if (RefPtr context = protectedThis->scriptExecutionContext()) - context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("VideoEncoder encode failed: "_s, result.error())); - protectedThis->closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + if (!protectedThis) { + if (RefPtr context = protectedThis->scriptExecutionContext()) + context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("VideoEncoder encode failed: "_s, result.error())); + protectedThis->closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); + return; + } + protectedThis->decreaseCodecOperationCountAndMaybeProcessControlMessageQueue(); }); + return WebCodecsControlMessageOutcome::Processed; } }); return { }; } void WebCodecsVideoEncoder::flush(Ref&& promise) { - if (m_state != WebCodecsCodecState::Configured) { + if (state() != WebCodecsCodecState::Configured) { promise->reject(Exception { ExceptionCode::InvalidStateError, "VideoEncoder is not configured"_s }); return; } @@ -316,6 +318,7 @@ void WebCodecsVideoEncoder::flush(Ref&& promise) if (RefPtr protectedThis = weakThis.get()) protectedThis->m_pendingFlushPromises.removeFirstMatching([&](auto& flushPromise) { return promise.ptr() == flushPromise.ptr(); }); }); + return WebCodecsControlMessageOutcome::Processed; } }); } @@ -358,7 +361,7 @@ ExceptionOr WebCodecsVideoEncoder::closeEncoder(Exception&& exception) auto result = resetEncoder(exception); if (result.hasException()) return result; - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalEncoder = nullptr; if (exception.code() != ExceptionCode::AbortError) m_error->handleEvent(DOMException::create(WTFMove(exception))); @@ -368,17 +371,13 @@ ExceptionOr WebCodecsVideoEncoder::closeEncoder(Exception&& exception) ExceptionOr WebCodecsVideoEncoder::resetEncoder(const Exception& exception) { - if (m_state == WebCodecsCodecState::Closed) + if (state() == WebCodecsCodecState::Closed) return Exception { ExceptionCode::InvalidStateError, "VideoEncoder is closed"_s }; - m_state = WebCodecsCodecState::Unconfigured; + setState(WebCodecsCodecState::Unconfigured); if (RefPtr internalEncoder = std::exchange(m_internalEncoder, { })) internalEncoder->reset(); - m_controlMessageQueue.clear(); - if (m_encodeQueueSize) { - m_encodeQueueSize = 0; - scheduleDequeueEvent(); - } + clearControlMessageQueueAndMaybeScheduleDequeueEvent(); auto promises = std::exchange(m_pendingFlushPromises, { }); for (auto& promise : promises) @@ -387,61 +386,23 @@ ExceptionOr WebCodecsVideoEncoder::resetEncoder(const Exception& exception return { }; } -void WebCodecsVideoEncoder::scheduleDequeueEvent() -{ - if (m_dequeueEventScheduled) - return; - - m_dequeueEventScheduled = true; - queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this]() mutable { - dispatchEvent(Event::create(eventNames().dequeueEvent, Event::CanBubble::No, Event::IsCancelable::No)); - m_dequeueEventScheduled = false; - }); -} - void WebCodecsVideoEncoder::setInternalEncoder(Ref&& internalEncoder) { m_internalEncoder = WTFMove(internalEncoder); } -void WebCodecsVideoEncoder::queueControlMessageAndProcess(WebCodecsControlMessage&& message) -{ - if (m_isMessageQueueBlocked) { - m_controlMessageQueue.append(WTFMove(message)); - return; - } - if (m_controlMessageQueue.isEmpty()) { - message(); - return; - } - - m_controlMessageQueue.append(WTFMove(message)); - processControlMessageQueue(); -} - -void WebCodecsVideoEncoder::processControlMessageQueue() -{ - while (!m_isMessageQueueBlocked && !m_controlMessageQueue.isEmpty()) - m_controlMessageQueue.takeFirst()(); -} - void WebCore::WebCodecsVideoEncoder::suspend(ReasonForSuspension) { } void WebCodecsVideoEncoder::stop() { - m_state = WebCodecsCodecState::Closed; + setState(WebCodecsCodecState::Closed); m_internalEncoder = nullptr; - m_controlMessageQueue.clear(); + clearControlMessageQueue(); m_pendingFlushPromises.clear(); } -bool WebCodecsVideoEncoder::virtualHasPendingActivity() const -{ - return m_state == WebCodecsCodecState::Configured && (m_encodeQueueSize || m_isMessageQueueBlocked); -} - } // namespace WebCore #endif // ENABLE(WEB_CODECS) diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h index 2982c00024fe3..6fbc1f6fe54fc 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h +++ b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h @@ -27,12 +27,9 @@ #if ENABLE(WEB_CODECS) -#include "ActiveDOMObject.h" -#include "EventTarget.h" #include "JSDOMPromiseDeferredForward.h" #include "VideoEncoder.h" -#include "WebCodecsCodecState.h" -#include "WebCodecsControlMessage.h" +#include "WebCodecsBase.h" #include "WebCodecsVideoEncoderConfig.h" #include @@ -45,10 +42,7 @@ class WebCodecsVideoFrame; struct WebCodecsEncodedVideoChunkMetadata; struct WebCodecsVideoEncoderEncodeOptions; -class WebCodecsVideoEncoder - : public ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr - , public ActiveDOMObject - , public EventTarget { +class WebCodecsVideoEncoder : public WebCodecsBase { WTF_MAKE_TZONE_OR_ISO_ALLOCATED(WebCodecsVideoEncoder); public: ~WebCodecsVideoEncoder(); @@ -60,8 +54,7 @@ class WebCodecsVideoEncoder static Ref create(ScriptExecutionContext&, Init&&); - WebCodecsCodecState state() const { return m_state; } - size_t encodeQueueSize() const { return m_encodeQueueSize; } + size_t encodeQueueSize() const { return operationQueueSize(); } ExceptionOr configure(ScriptExecutionContext&, WebCodecsVideoEncoderConfig&&); ExceptionOr encode(Ref&&, WebCodecsVideoEncoderEncodeOptions&&); @@ -71,47 +64,32 @@ class WebCodecsVideoEncoder static void isConfigSupported(ScriptExecutionContext&, WebCodecsVideoEncoderConfig&&, Ref&&); - // ActiveDOMObject. - void ref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::ref(); } - void deref() const final { ThreadSafeRefCountedAndCanMakeThreadSafeWeakPtr::deref(); } - WebCodecsEncodedVideoChunkOutputCallback& outputCallbackConcurrently() { return m_output.get(); } WebCodecsErrorCallback& errorCallbackConcurrently() { return m_error.get(); } private: WebCodecsVideoEncoder(ScriptExecutionContext&, Init&&); + size_t maximumCodecOperationsEnqueued() const final { return 4; } // ActiveDOMObject. void stop() final; void suspend(ReasonForSuspension) final; - bool virtualHasPendingActivity() const final; // EventTarget - void refEventTarget() final { ref(); } - void derefEventTarget() final { deref(); } enum EventTargetInterfaceType eventTargetInterface() const final { return EventTargetInterfaceType::WebCodecsVideoEncoder; } - ScriptExecutionContext* scriptExecutionContext() const final { return ActiveDOMObject::scriptExecutionContext(); } ExceptionOr closeEncoder(Exception&&); ExceptionOr resetEncoder(const Exception&); void setInternalEncoder(Ref&&); - void scheduleDequeueEvent(); - void queueControlMessageAndProcess(WebCodecsControlMessage&&); - void processControlMessageQueue(); WebCodecsEncodedVideoChunkMetadata createEncodedChunkMetadata(std::optional); void updateRates(const WebCodecsVideoEncoderConfig&); - WebCodecsCodecState m_state { WebCodecsCodecState::Unconfigured }; - size_t m_encodeQueueSize { 0 }; Ref m_output; Ref m_error; RefPtr m_internalEncoder; - bool m_dequeueEventScheduled { false }; Vector> m_pendingFlushPromises; bool m_isKeyChunkRequired { false }; - Deque> m_controlMessageQueue; - bool m_isMessageQueueBlocked { false }; WebCodecsVideoEncoderConfig m_baseConfiguration; VideoEncoder::ActiveConfiguration m_activeConfiguration; bool m_hasNewActiveConfiguration { false }; diff --git a/Source/WebCore/Sources.txt b/Source/WebCore/Sources.txt index 0520888db2758..1bc6ed344854f 100644 --- a/Source/WebCore/Sources.txt +++ b/Source/WebCore/Sources.txt @@ -463,6 +463,7 @@ Modules/webcodecs/WebCodecsAudioDecoder.cpp Modules/webcodecs/WebCodecsAudioData.cpp Modules/webcodecs/WebCodecsAudioDataAlgorithms.cpp Modules/webcodecs/WebCodecsAudioEncoder.cpp +Modules/webcodecs/WebCodecsBase.cpp Modules/webcodecs/WebCodecsEncodedAudioChunk.cpp Modules/webcodecs/WebCodecsEncodedVideoChunk.cpp Modules/webcodecs/WebCodecsVideoDecoder.cpp diff --git a/Source/WebCore/WebCore.xcodeproj/project.pbxproj b/Source/WebCore/WebCore.xcodeproj/project.pbxproj index 33502400aa892..8d92c0f2fbc7d 100644 --- a/Source/WebCore/WebCore.xcodeproj/project.pbxproj +++ b/Source/WebCore/WebCore.xcodeproj/project.pbxproj @@ -2102,6 +2102,7 @@ 51C81B8A0C4422F70019ECE3 /* FTPDirectoryParser.h in Headers */ = {isa = PBXBuildFile; fileRef = 51C81B880C4422F70019ECE3 /* FTPDirectoryParser.h */; }; 51CA7EE91F883390003D3131 /* ServiceWorkerContextData.h in Headers */ = {isa = PBXBuildFile; fileRef = 51CA7EE71F8832E0003D3131 /* ServiceWorkerContextData.h */; settings = {ATTRIBUTES = (Private, ); }; }; 51CBFC990D10E483002DBF51 /* CachedFramePlatformData.h in Headers */ = {isa = PBXBuildFile; fileRef = 51CBFC980D10E483002DBF51 /* CachedFramePlatformData.h */; settings = {ATTRIBUTES = (Private, ); }; }; + 51D444702D09B20A00F26553 /* WebCodecsBase.h in Headers */ = {isa = PBXBuildFile; fileRef = 51D4446F2D09B1F800F26553 /* WebCodecsBase.h */; settings = {ATTRIBUTES = (Private, ); }; }; 51D596CF2CECF71000AA8531 /* ProcessSyncData.h in Headers */ = {isa = PBXBuildFile; fileRef = 51D596CD2CECF71000AA8531 /* ProcessSyncData.h */; settings = {ATTRIBUTES = (Private, ); }; }; 51D596D02CECF71000AA8531 /* ProcessSyncClient.h in Headers */ = {isa = PBXBuildFile; fileRef = 51D596CB2CECF71000AA8531 /* ProcessSyncClient.h */; settings = {ATTRIBUTES = (Private, ); }; }; 51D596D22CECF71000AA8531 /* ProcessSyncData.serialization.in in Headers */ = {isa = PBXBuildFile; fileRef = 51D596CE2CECF71000AA8531 /* ProcessSyncData.serialization.in */; settings = {ATTRIBUTES = (Private, ); }; }; @@ -11719,6 +11720,8 @@ 51CBFC980D10E483002DBF51 /* CachedFramePlatformData.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CachedFramePlatformData.h; sourceTree = ""; }; 51D1248A1E73625C002B2820 /* NetworkStorageSessionCocoa.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = NetworkStorageSessionCocoa.mm; sourceTree = ""; }; 51D1248C1E736456002B2820 /* CookieCocoa.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = CookieCocoa.mm; sourceTree = ""; }; + 51D4446F2D09B1F800F26553 /* WebCodecsBase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WebCodecsBase.h; sourceTree = ""; }; + 51D444712D09B9EA00F26553 /* WebCodecsBase.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = WebCodecsBase.cpp; sourceTree = ""; }; 51D596CB2CECF71000AA8531 /* ProcessSyncClient.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ProcessSyncClient.h; sourceTree = ""; }; 51D596CC2CECF71000AA8531 /* ProcessSyncClient.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = ProcessSyncClient.cpp; sourceTree = ""; }; 51D596CD2CECF71000AA8531 /* ProcessSyncData.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ProcessSyncData.h; sourceTree = ""; }; @@ -35624,6 +35627,8 @@ 9ACB2F0F2AA5E982006E9556 /* WebCodecsAudioEncoderSupport.h */, 9ACB2F0D2AA5E981006E9556 /* WebCodecsAudioEncoderSupport.idl */, 9A87D9EB2A8295F100EE7A8A /* WebCodecsAudioInternalData.h */, + 51D444712D09B9EA00F26553 /* WebCodecsBase.cpp */, + 51D4446F2D09B1F800F26553 /* WebCodecsBase.h */, 4110A21128E3440C00321D09 /* WebCodecsCodecState.h */, 4110A20F28E3440C00321D09 /* WebCodecsCodecState.idl */, 459CBAAA2C7647410003743B /* WebCodecsControlMessage.h */, @@ -44545,6 +44550,7 @@ 9A87D9F42A8298AF00EE7A8A /* WebCodecsAudioDecoderConfig.h in Headers */, 9A87D9E62A8294BE00EE7A8A /* WebCodecsAudioDecoderSupport.h in Headers */, 9A87D9EC2A8295F100EE7A8A /* WebCodecsAudioInternalData.h in Headers */, + 51D444702D09B20A00F26553 /* WebCodecsBase.h in Headers */, 4110A21A28E3441600321D09 /* WebCodecsCodecState.h in Headers */, 9A87D9EE2A82967A00EE7A8A /* WebCodecsEncodedAudioChunk.h in Headers */, 9A87D9F82A829ED500EE7A8A /* WebCodecsEncodedAudioChunkData.h in Headers */,