diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index f3f6436..0375a03 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -4,18 +4,18 @@ "name": "Linux", "includePath": [ "/usr/include", - "${workspaceFolder}/_build/ubuntu-20.04_x86_64/release/zakuro", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/sora/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/webrtc/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/webrtc/include/third_party/abseil-cpp", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/webrtc/include/third_party/boringssl/src/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/webrtc/include/third_party/libyuv/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/llvm/libcxx/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/boost/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/cli11/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/blend2d/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/openh264/include", - "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/yaml/include" + "${workspaceFolder}/_build/ubuntu-22.04_x86_64/release/zakuro", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/sora/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/webrtc/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/webrtc/include/third_party/abseil-cpp", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/webrtc/include/third_party/boringssl/src/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/webrtc/include/third_party/libyuv/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/llvm/libcxx/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/boost/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/cli11/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/blend2d/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/openh264/include", + "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/yaml/include" ], "defines": [ "WEBRTC_POSIX", @@ -24,9 +24,10 @@ "_LIBCPP_ABI_VERSION=2", "_LIBCPP_DISABLE_AVAILABILITY", "_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE", - "OPENSSL_IS_BORINGSSL" + "OPENSSL_IS_BORINGSSL", + "RTC_ENABLE_H265" ], - "compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/llvm/clang/bin/clang++", + "compilerPath": "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], "cStandard": "gnu17", "cppStandard": "gnu++17", diff --git a/.vscode/settings.json b/.vscode/settings.json index 1115a58..c08d31e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -35,6 +35,7 @@ "chrono": "cpp", "string": "cpp", "__mutex_base": "cpp", - "vector": "cpp" + "vector": "cpp", + "optional": "cpp" } } \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index 59695e0..da7dc57 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,13 @@ - @melpon - [CHANGE] `--webrtc-build-dir`, `--webrtc-build-args` を `--local-webrtc-build-dir` と `--local-webrtc-build-args` に変更する - @melpon +- [UPDATE] Sora C++ SDK を 2024.7.0-canary.4 に上げる + - それに伴って以下のライブラリのバージョンも上げる + - libwebrtc のバージョンを `m125.6422.2.5` に上げる + - Boost のバージョンを `1.85.0` に上げる + - @melpon +- [UPDATE] Blend2D, AsmJit を最新版に上げる + - @melpon ## 2024.1.0 (2024-04-24) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b1d8a3..daf50ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,13 +40,11 @@ target_include_directories(zakuro PRIVATE ${OPENH264_ROOT_DIR}/include) target_sources(zakuro PRIVATE - src/dynamic_h264_video_encoder.cpp src/embedded_binary.cpp src/enable_media_with_fake_call.cpp src/fake_video_capturer.cpp src/main.cpp src/nop_video_decoder.cpp - src/sctp_transport_factory.cpp src/util.cpp src/virtual_client.cpp src/wav_reader.cpp diff --git a/VERSION b/VERSION index 1c59e73..d3c6795 100644 --- a/VERSION +++ b/VERSION @@ -1,10 +1,10 @@ ZAKURO_VERSION=2024.1.0 -SORA_CPP_SDK_VERSION=2024.6.1 -WEBRTC_BUILD_VERSION=m122.6261.1.0 -BOOST_VERSION=1.84.0 +SORA_CPP_SDK_VERSION=2024.7.0-canary.3 +WEBRTC_BUILD_VERSION=m125.6422.2.5 +BOOST_VERSION=1.85.0 CLI11_VERSION=v2.4.2 CMAKE_VERSION=3.28.1 -BLEND2D_VERSION=5a263ce51f3f880ee6c60f6345d18c3eccbe200f -ASMJIT_VERSION=3ca5c186bf8922e5fe3018432e93651fd2fa4053 +BLEND2D_VERSION=c966273935e70ce1bed328993ee40d7cfe8fef5c +ASMJIT_VERSION=062e69ca81defa35eb0ee15f7412f49a0dad3cdb OPENH264_VERSION=v2.4.1 YAML_CPP_VERSION=0.8.0 diff --git a/src/dynamic_h264_video_encoder.cpp b/src/dynamic_h264_video_encoder.cpp deleted file mode 100644 index c65fc91..0000000 --- a/src/dynamic_h264_video_encoder.cpp +++ /dev/null @@ -1,677 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -// modules/video_coding/codecs/h264/h264_encoder_impl.{h,cc} の -// OpenH264 の関数を動的に読むようにしただけ - -#include "dynamic_h264_video_encoder.h" - -#include -#include - -// Linux -#include - -// WebRTC -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace webrtc { - -namespace { - -const bool kOpenH264EncoderDetailedLogging = false; - -// QP scaling thresholds. -static const int kLowH264QpThreshold = 24; -static const int kHighH264QpThreshold = 37; - -// Used by histograms. Values of entries should not be changed. -enum H264EncoderImplEvent { - kH264EncoderEventInit = 0, - kH264EncoderEventError = 1, - kH264EncoderEventMax = 16, -}; - -int NumberOfThreads(int width, int height, int number_of_cores) { - // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac, - // see crbug.com/583348. Until further investigated, only use one thread. - // if (width * height >= 1920 * 1080 && number_of_cores > 8) { - // return 8; // 8 threads for 1080p on high perf machines. - // } else if (width * height > 1280 * 960 && number_of_cores >= 6) { - // return 3; // 3 threads for 1080p. - // } else if (width * height > 640 * 480 && number_of_cores >= 3) { - // return 2; // 2 threads for qHD/HD. - // } else { - // return 1; // 1 thread for VGA or less. - // } - // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(), - // before enabling multithreading here. - return 1; -} - -VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) { - switch (type) { - case videoFrameTypeIDR: - return VideoFrameType::kVideoFrameKey; - case videoFrameTypeSkip: - case videoFrameTypeI: - case videoFrameTypeP: - case videoFrameTypeIPMixed: - return VideoFrameType::kVideoFrameDelta; - case videoFrameTypeInvalid: - break; - } - RTC_DCHECK_NOTREACHED() << "Unexpected/invalid frame type: " << type; - return VideoFrameType::kEmptyFrame; -} - -} // namespace - -// Helper method used by DynamicH264VideoEncoder::Encode. -// Copies the encoded bytes from |info| to |encoded_image|. The -// |encoded_image->_buffer| may be deleted and reallocated if a bigger buffer is -// required. -// -// After OpenH264 encoding, the encoded bytes are stored in |info| spread out -// over a number of layers and "NAL units". Each NAL unit is a fragment starting -// with the four-byte start code {0,0,0,1}. All of this data (including the -// start codes) is copied to the |encoded_image->_buffer|. -static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { - // Calculate minimum buffer size required to hold encoded data. - size_t required_capacity = 0; - size_t fragments_count = 0; - for (int layer = 0; layer < info->iLayerNum; ++layer) { - const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; - for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) { - RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0); - // Ensure |required_capacity| will not overflow. - RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal], - std::numeric_limits::max() - required_capacity); - required_capacity += layerInfo.pNalLengthInByte[nal]; - } - } - // TODO(nisse): Use a cache or buffer pool to avoid allocation? - auto buffer = EncodedImageBuffer::Create(required_capacity); - encoded_image->SetEncodedData(buffer); - - // Iterate layers and NAL units, note each NAL unit as a fragment and copy - // the data to |encoded_image->_buffer|. - const uint8_t start_code[4] = {0, 0, 0, 1}; - size_t frag = 0; - encoded_image->set_size(0); - for (int layer = 0; layer < info->iLayerNum; ++layer) { - const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; - // Iterate NAL units making up this layer, noting fragments. - size_t layer_len = 0; - for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) { - // Because the sum of all layer lengths, |required_capacity|, fits in a - // |size_t|, we know that any indices in-between will not overflow. - RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4); - RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]); - RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]); - RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 2], start_code[2]); - RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 3], start_code[3]); - layer_len += layerInfo.pNalLengthInByte[nal]; - } - // Copy the entire layer's data (including start codes). - memcpy(buffer->data() + encoded_image->size(), layerInfo.pBsBuf, layer_len); - encoded_image->set_size(encoded_image->size() + layer_len); - } -} - -DynamicH264VideoEncoder::DynamicH264VideoEncoder( - const cricket::VideoCodec& codec, - std::string openh264) - : packetization_mode_(H264PacketizationMode::SingleNalUnit), - max_payload_size_(0), - number_of_cores_(0), - encoded_image_callback_(nullptr), - has_reported_init_(false), - has_reported_error_(false), - openh264_(std::move(openh264)) { - RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName)); - std::string packetization_mode_string; - if (codec.GetParam(cricket::kH264FmtpPacketizationMode, - &packetization_mode_string) && - packetization_mode_string == "1") { - packetization_mode_ = H264PacketizationMode::NonInterleaved; - } - downscaled_buffers_.reserve(kMaxSimulcastStreams - 1); - encoded_images_.reserve(kMaxSimulcastStreams); - encoders_.reserve(kMaxSimulcastStreams); - configurations_.reserve(kMaxSimulcastStreams); - tl0sync_limit_.reserve(kMaxSimulcastStreams); -} - -DynamicH264VideoEncoder::~DynamicH264VideoEncoder() { - Release(); - ReleaseOpenH264(); -} - -bool DynamicH264VideoEncoder::InitOpenH264() { - if (openh264_handle_ != nullptr) { - return true; - } - - void* handle = ::dlopen(openh264_.c_str(), RTLD_LAZY); - if (handle == nullptr) { - return false; - } - create_encoder_ = (CreateEncoderFunc)::dlsym(handle, "WelsCreateSVCEncoder"); - if (create_encoder_ == nullptr) { - ::dlclose(handle); - return false; - } - destroy_encoder_ = - (DestroyEncoderFunc)::dlsym(handle, "WelsDestroySVCEncoder"); - if (destroy_encoder_ == nullptr) { - ::dlclose(handle); - return false; - } - openh264_handle_ = handle; - return true; -} - -void DynamicH264VideoEncoder::ReleaseOpenH264() { - if (openh264_handle_ != nullptr) { - ::dlclose(openh264_handle_); - openh264_handle_ = nullptr; - } -} - -int32_t DynamicH264VideoEncoder::InitEncode( - const VideoCodec* inst, - const VideoEncoder::Settings& settings) { - ReportInit(); - if (!inst || inst->codecType != kVideoCodecH264) { - ReportError(); - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - if (inst->maxFramerate == 0) { - ReportError(); - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - if (inst->width < 1 || inst->height < 1) { - ReportError(); - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - - int32_t release_ret = Release(); - if (release_ret != WEBRTC_VIDEO_CODEC_OK) { - ReportError(); - return release_ret; - } - - if (!InitOpenH264()) { - ReportError(); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst); - bool doing_simulcast = (number_of_streams > 1); - - if (doing_simulcast && - !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) { - return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; - } - downscaled_buffers_.resize(number_of_streams - 1); - encoded_images_.resize(number_of_streams); - encoders_.resize(number_of_streams); - pictures_.resize(number_of_streams); - configurations_.resize(number_of_streams); - tl0sync_limit_.resize(number_of_streams); - - number_of_cores_ = settings.number_of_cores; - max_payload_size_ = settings.max_payload_size; - codec_ = *inst; - - // Code expects simulcastStream resolutions to be correct, make sure they are - // filled even when there are no simulcast layers. - if (codec_.numberOfSimulcastStreams == 0) { - codec_.simulcastStream[0].width = codec_.width; - codec_.simulcastStream[0].height = codec_.height; - } - - for (int i = 0, idx = number_of_streams - 1; i < number_of_streams; - ++i, --idx) { - ISVCEncoder* openh264_encoder; - // Create encoder. - if (create_encoder_(&openh264_encoder) != 0) { - // Failed to create encoder. - RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder"; - RTC_DCHECK(!openh264_encoder); - Release(); - ReportError(); - return WEBRTC_VIDEO_CODEC_ERROR; - } - RTC_DCHECK(openh264_encoder); - if (kOpenH264EncoderDetailedLogging) { - int trace_level = WELS_LOG_DETAIL; - openh264_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level); - } - // else WELS_LOG_DEFAULT is used by default. - - // Store h264 encoder. - encoders_[i] = openh264_encoder; - - // Set internal settings from codec_settings - configurations_[i].simulcast_idx = idx; - configurations_[i].sending = false; - configurations_[i].width = codec_.simulcastStream[idx].width; - configurations_[i].height = codec_.simulcastStream[idx].height; - configurations_[i].max_frame_rate = static_cast(codec_.maxFramerate); - configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval; - configurations_[i].num_temporal_layers = - codec_.simulcastStream[idx].numberOfTemporalLayers; - - // Create downscaled image buffers. - if (i > 0) { - downscaled_buffers_[i - 1] = I420Buffer::Create( - configurations_[i].width, configurations_[i].height, - configurations_[i].width, configurations_[i].width / 2, - configurations_[i].width / 2); - } - - // Codec_settings uses kbits/second; encoder uses bits/second. - configurations_[i].max_bps = codec_.maxBitrate * 1000; - configurations_[i].target_bps = codec_.startBitrate * 1000; - - // Create encoder parameters based on the layer configuration. - SEncParamExt encoder_params = CreateEncoderParams(i); - - // Initialize. - if (openh264_encoder->InitializeExt(&encoder_params) != 0) { - RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder"; - Release(); - ReportError(); - return WEBRTC_VIDEO_CODEC_ERROR; - } - // TODO(pbos): Base init params on these values before submitting. - int video_format = EVideoFormatType::videoFormatI420; - openh264_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format); - - // Initialize encoded image. Default buffer size: size of unencoded data. - - const size_t new_capacity = - CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width, - codec_.simulcastStream[idx].height); - encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity)); - encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width; - encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height; - encoded_images_[i].set_size(0); - - tl0sync_limit_[i] = configurations_[i].num_temporal_layers; - } - - SimulcastRateAllocator init_allocator(codec_); - VideoBitrateAllocation allocation = - init_allocator.Allocate(VideoBitrateAllocationParameters( - DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate)); - SetRates(RateControlParameters(allocation, codec_.maxFramerate)); - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t DynamicH264VideoEncoder::Release() { - while (!encoders_.empty()) { - ISVCEncoder* openh264_encoder = encoders_.back(); - if (openh264_encoder) { - RTC_CHECK_EQ(0, openh264_encoder->Uninitialize()); - destroy_encoder_(openh264_encoder); - } - encoders_.pop_back(); - } - downscaled_buffers_.clear(); - configurations_.clear(); - encoded_images_.clear(); - pictures_.clear(); - tl0sync_limit_.clear(); - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t DynamicH264VideoEncoder::RegisterEncodeCompleteCallback( - EncodedImageCallback* callback) { - encoded_image_callback_ = callback; - return WEBRTC_VIDEO_CODEC_OK; -} - -void DynamicH264VideoEncoder::SetRates( - const RateControlParameters& parameters) { - if (encoders_.empty()) { - RTC_LOG(LS_WARNING) << "SetRates() while uninitialized."; - return; - } - - if (parameters.framerate_fps < 1.0) { - RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps; - return; - } - - if (parameters.bitrate.get_sum_bps() == 0) { - // Encoder paused, turn off all encoding. - for (size_t i = 0; i < configurations_.size(); ++i) { - configurations_[i].SetStreamState(false); - } - return; - } - - codec_.maxFramerate = static_cast(parameters.framerate_fps); - - size_t stream_idx = encoders_.size() - 1; - for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { - // Update layer config. - configurations_[i].target_bps = - parameters.bitrate.GetSpatialLayerSum(stream_idx); - configurations_[i].max_frame_rate = parameters.framerate_fps; - - if (configurations_[i].target_bps) { - configurations_[i].SetStreamState(true); - - // Update h264 encoder. - SBitrateInfo target_bitrate; - memset(&target_bitrate, 0, sizeof(SBitrateInfo)); - target_bitrate.iLayer = SPATIAL_LAYER_ALL, - target_bitrate.iBitrate = configurations_[i].target_bps; - encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate); - encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE, - &configurations_[i].max_frame_rate); - } else { - configurations_[i].SetStreamState(false); - } - } -} - -int32_t DynamicH264VideoEncoder::Encode( - const VideoFrame& input_frame, - const std::vector* frame_types) { - if (encoders_.empty()) { - ReportError(); - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - if (!encoded_image_callback_) { - RTC_LOG(LS_WARNING) - << "InitEncode() has been called, but a callback function " - "has not been set with RegisterEncodeCompleteCallback()"; - ReportError(); - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - - rtc::scoped_refptr frame_buffer = - input_frame.video_frame_buffer()->ToI420(); - - bool send_key_frame = false; - for (size_t i = 0; i < configurations_.size(); ++i) { - if (configurations_[i].key_frame_request && configurations_[i].sending) { - send_key_frame = true; - break; - } - } - - if (!send_key_frame && frame_types) { - for (size_t i = 0; i < configurations_.size(); ++i) { - const size_t simulcast_idx = - static_cast(configurations_[i].simulcast_idx); - if (configurations_[i].sending && simulcast_idx < frame_types->size() && - (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) { - send_key_frame = true; - break; - } - } - } - - RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width()); - RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height()); - - // Encode image for each layer. - for (size_t i = 0; i < encoders_.size(); ++i) { - // EncodeFrame input. - pictures_[i] = {0}; - pictures_[i].iPicWidth = configurations_[i].width; - pictures_[i].iPicHeight = configurations_[i].height; - pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420; - pictures_[i].uiTimeStamp = input_frame.ntp_time_ms(); - // Downscale images on second and ongoing layers. - if (i == 0) { - pictures_[i].iStride[0] = frame_buffer->StrideY(); - pictures_[i].iStride[1] = frame_buffer->StrideU(); - pictures_[i].iStride[2] = frame_buffer->StrideV(); - pictures_[i].pData[0] = const_cast(frame_buffer->DataY()); - pictures_[i].pData[1] = const_cast(frame_buffer->DataU()); - pictures_[i].pData[2] = const_cast(frame_buffer->DataV()); - } else { - pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY(); - pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU(); - pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV(); - pictures_[i].pData[0] = - const_cast(downscaled_buffers_[i - 1]->DataY()); - pictures_[i].pData[1] = - const_cast(downscaled_buffers_[i - 1]->DataU()); - pictures_[i].pData[2] = - const_cast(downscaled_buffers_[i - 1]->DataV()); - // Scale the image down a number of times by downsampling factor. - libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0], - pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1], - pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2], - configurations_[i - 1].width, - configurations_[i - 1].height, pictures_[i].pData[0], - pictures_[i].iStride[0], pictures_[i].pData[1], - pictures_[i].iStride[1], pictures_[i].pData[2], - pictures_[i].iStride[2], configurations_[i].width, - configurations_[i].height, libyuv::kFilterBilinear); - } - - if (!configurations_[i].sending) { - continue; - } - if (frame_types != nullptr) { - // Skip frame? - if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) { - continue; - } - } - if (send_key_frame) { - // API doc says ForceIntraFrame(false) does nothing, but calling this - // function forces a key frame regardless of the |bIDR| argument's value. - // (If every frame is a key frame we get lag/delays.) - encoders_[i]->ForceIntraFrame(true); - configurations_[i].key_frame_request = false; - } - // EncodeFrame output. - SFrameBSInfo info; - memset(&info, 0, sizeof(SFrameBSInfo)); - - // Encode! - int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info); - if (enc_ret != 0) { - RTC_LOG(LS_ERROR) - << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret - << "."; - ReportError(); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - encoded_images_[i]._encodedWidth = configurations_[i].width; - encoded_images_[i]._encodedHeight = configurations_[i].height; - encoded_images_[i].SetRtpTimestamp(input_frame.timestamp()); - encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType); - encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx); - - // Split encoded image up into fragments. This also updates - // |encoded_image_|. - RtpFragmentize(&encoded_images_[i], &info); - - // Encoder can skip frames to save bandwidth in which case - // |encoded_images_[i]._length| == 0. - if (encoded_images_[i].size() > 0) { - // Parse QP. - h264_bitstream_parser_.ParseBitstream(encoded_images_[i]); - encoded_images_[i].qp_ = - h264_bitstream_parser_.GetLastSliceQp().value_or(-1); - - // Deliver encoded image. - CodecSpecificInfo codec_specific; - codec_specific.codecType = kVideoCodecH264; - codec_specific.codecSpecific.H264.packetization_mode = - packetization_mode_; - codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx; - codec_specific.codecSpecific.H264.idr_frame = - info.eFrameType == videoFrameTypeIDR; - codec_specific.codecSpecific.H264.base_layer_sync = false; - if (configurations_[i].num_temporal_layers > 1) { - const uint8_t tid = info.sLayerInfo[0].uiTemporalId; - codec_specific.codecSpecific.H264.temporal_idx = tid; - codec_specific.codecSpecific.H264.base_layer_sync = - tid > 0 && tid < tl0sync_limit_[i]; - if (codec_specific.codecSpecific.H264.base_layer_sync) { - tl0sync_limit_[i] = tid; - } - if (tid == 0) { - tl0sync_limit_[i] = configurations_[i].num_temporal_layers; - } - } - encoded_image_callback_->OnEncodedImage(encoded_images_[i], - &codec_specific); - } - } - return WEBRTC_VIDEO_CODEC_OK; -} - -// Initialization parameters. -// There are two ways to initialize. There is SEncParamBase (cleared with -// memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt -// which is a superset of SEncParamBase (cleared with GetDefaultParams) used -// in InitializeExt. -SEncParamExt DynamicH264VideoEncoder::CreateEncoderParams(size_t i) const { - SEncParamExt encoder_params; - encoders_[i]->GetDefaultParams(&encoder_params); - if (codec_.mode == VideoCodecMode::kRealtimeVideo) { - encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME; - } else if (codec_.mode == VideoCodecMode::kScreensharing) { - encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME; - } else { - RTC_DCHECK_NOTREACHED(); - } - encoder_params.iPicWidth = configurations_[i].width; - encoder_params.iPicHeight = configurations_[i].height; - encoder_params.iTargetBitrate = configurations_[i].target_bps; - // Keep unspecified. WebRTC's max codec bitrate is not the same setting - // as OpenH264's iMaxBitrate. More details in https://crbug.com/webrtc/11543 - encoder_params.iMaxBitrate = UNSPECIFIED_BIT_RATE; - // Rate Control mode - encoder_params.iRCMode = RC_BITRATE_MODE; - encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate; - - // The following parameters are extension parameters (they're in SEncParamExt, - // not in SEncParamBase). - encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on; - // |uiIntraPeriod| - multiple of GOP size - // |keyFrameInterval| - number of frames - encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval; - // Reuse SPS id if possible. This helps to avoid reset of chromium HW decoder - // on each key-frame. - // Note that WebRTC resets encoder on resolution change which makes all - // EParameterSetStrategy modes except INCREASING_ID (default) essentially - // equivalent to CONSTANT_ID. - encoder_params.eSpsPpsIdStrategy = SPS_LISTING; - encoder_params.uiMaxNalSize = 0; - // Threading model: use auto. - // 0: auto (dynamic imp. internal encoder) - // 1: single thread (default value) - // >1: number of threads - encoder_params.iMultipleThreadIdc = NumberOfThreads( - encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_); - // The base spatial layer 0 is the only one we use. - encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth; - encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight; - encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate; - encoder_params.sSpatialLayers[0].iSpatialBitrate = - encoder_params.iTargetBitrate; - encoder_params.sSpatialLayers[0].iMaxSpatialBitrate = - encoder_params.iMaxBitrate; - encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers; - if (encoder_params.iTemporalLayerNum > 1) { - encoder_params.iNumRefFrame = 1; - } - RTC_LOG(LS_INFO) << "OpenH264 version is " << OPENH264_MAJOR << "." - << OPENH264_MINOR; - switch (packetization_mode_) { - case H264PacketizationMode::SingleNalUnit: - // Limit the size of the packets produced. - encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; - encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = - SM_SIZELIMITED_SLICE; - encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = - static_cast(max_payload_size_); - RTC_LOG(LS_INFO) << "Encoder is configured with NALU constraint: " - << max_payload_size_ << " bytes"; - break; - case H264PacketizationMode::NonInterleaved: - // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto - // design it with cpu core number. - // TODO(sprang): Set to 0 when we understand why the rate controller borks - // when uiSliceNum > 1. - encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; - encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = - SM_FIXEDSLCNUM_SLICE; - break; - } - return encoder_params; -} - -void DynamicH264VideoEncoder::ReportInit() { - if (has_reported_init_) - return; - RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.DynamicH264VideoEncoder.Event", - kH264EncoderEventInit, kH264EncoderEventMax); - has_reported_init_ = true; -} - -void DynamicH264VideoEncoder::ReportError() { - if (has_reported_error_) - return; - RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.DynamicH264VideoEncoder.Event", - kH264EncoderEventError, kH264EncoderEventMax); - has_reported_error_ = true; -} - -VideoEncoder::EncoderInfo DynamicH264VideoEncoder::GetEncoderInfo() const { - EncoderInfo info; - info.supports_native_handle = false; - info.implementation_name = "OpenH264"; - info.scaling_settings = - VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); - info.is_hardware_accelerated = false; - info.supports_simulcast = true; - info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420}; - return info; -} - -void DynamicH264VideoEncoder::LayerConfig::SetStreamState(bool send_stream) { - if (send_stream && !sending) { - // Need a key frame if we have not sent this stream before. - key_frame_request = true; - } - sending = send_stream; -} - -} // namespace webrtc diff --git a/src/dynamic_h264_video_encoder.h b/src/dynamic_h264_video_encoder.h deleted file mode 100644 index a358239..0000000 --- a/src/dynamic_h264_video_encoder.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -// modules/video_coding/codecs/h264/h264_encoder_impl.{h,cc} の -// OpenH264 の関数を動的に読むようにしただけ - -#ifndef RTC_DYNAMIC_H264_VIDEO_ENCODER_H_ -#define RTC_DYNAMIC_H264_VIDEO_ENCODER_H_ - -#include -#include - -// WebRTC -#include -#include -#include -#include -#include - -// OpenH264 -#include - -class ISVCEncoder; - -namespace webrtc { - -class DynamicH264VideoEncoder : public H264Encoder { - public: - static std::unique_ptr Create(const cricket::VideoCodec& codec, - std::string openh264) { - return std::unique_ptr( - new DynamicH264VideoEncoder(codec, std::move(openh264))); - } - - public: - struct LayerConfig { - int simulcast_idx = 0; - int width = -1; - int height = -1; - bool sending = true; - bool key_frame_request = false; - float max_frame_rate = 0; - uint32_t target_bps = 0; - uint32_t max_bps = 0; - bool frame_dropping_on = false; - int key_frame_interval = 0; - int num_temporal_layers = 1; - - void SetStreamState(bool send_stream); - }; - - public: - explicit DynamicH264VideoEncoder(const cricket::VideoCodec& codec, - std::string openh264); - ~DynamicH264VideoEncoder() override; - - // |settings.max_payload_size| is ignored. - // The following members of |codec_settings| are used. The rest are ignored. - // - codecType (must be kVideoCodecH264) - // - targetBitrate - // - maxFramerate - // - width - // - height - int32_t InitEncode(const VideoCodec* codec_settings, - const VideoEncoder::Settings& settings) override; - int32_t Release() override; - - int32_t RegisterEncodeCompleteCallback( - EncodedImageCallback* callback) override; - void SetRates(const RateControlParameters& parameters) override; - - // The result of encoding - an EncodedImage and CodecSpecificInfo - are - // passed to the encode complete callback. - int32_t Encode(const VideoFrame& frame, - const std::vector* frame_types) override; - - EncoderInfo GetEncoderInfo() const override; - - // Exposed for testing. - H264PacketizationMode PacketizationModeForTesting() const { - return packetization_mode_; - } - - private: - SEncParamExt CreateEncoderParams(size_t i) const; - - webrtc::H264BitstreamParser h264_bitstream_parser_; - // Reports statistics with histograms. - void ReportInit(); - void ReportError(); - - std::vector encoders_; - std::vector pictures_; - std::vector> downscaled_buffers_; - std::vector configurations_; - std::vector encoded_images_; - - VideoCodec codec_; - H264PacketizationMode packetization_mode_; - size_t max_payload_size_; - int32_t number_of_cores_; - EncodedImageCallback* encoded_image_callback_; - - bool has_reported_init_; - bool has_reported_error_; - - std::vector tl0sync_limit_; - - private: - bool InitOpenH264(); - void ReleaseOpenH264(); - - std::string openh264_; - void* openh264_handle_ = nullptr; - using CreateEncoderFunc = int (*)(ISVCEncoder**); - using DestroyEncoderFunc = void (*)(ISVCEncoder*); - CreateEncoderFunc create_encoder_ = nullptr; - DestroyEncoderFunc destroy_encoder_ = nullptr; -}; - -} // namespace webrtc - -#endif // RTC_DYNAMIC_H264_VIDEO_ENCODER_H_ diff --git a/src/nop_video_decoder.cpp b/src/nop_video_decoder.cpp index 782cca3..22bba84 100644 --- a/src/nop_video_decoder.cpp +++ b/src/nop_video_decoder.cpp @@ -2,6 +2,7 @@ // WebRTC #include +#include #include #include #include @@ -68,8 +69,8 @@ NopVideoDecoderFactory::GetSupportedFormats() const { return supported_codecs; } -std::unique_ptr -NopVideoDecoderFactory::CreateVideoDecoder( +std::unique_ptr NopVideoDecoderFactory::Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) { return std::unique_ptr( absl::make_unique()); diff --git a/src/nop_video_decoder.h b/src/nop_video_decoder.h index fd18e86..bae9a61 100644 --- a/src/nop_video_decoder.h +++ b/src/nop_video_decoder.h @@ -26,7 +26,8 @@ class NopVideoDecoder : public webrtc::VideoDecoder { class NopVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: std::vector GetSupportedFormats() const override; - std::unique_ptr CreateVideoDecoder( + std::unique_ptr Create( + const webrtc::Environment& environment, const webrtc::SdpVideoFormat& format) override; }; diff --git a/src/sctp_transport_factory.cpp b/src/sctp_transport_factory.cpp deleted file mode 100644 index 57b0bf9..0000000 --- a/src/sctp_transport_factory.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "sctp_transport_factory.h" - -#include -#include -#include -#include - -SctpTransportFactory::SctpTransportFactory(rtc::Thread* network_thread) - : network_thread_(network_thread) {} - -std::unique_ptr -SctpTransportFactory::CreateSctpTransport( - rtc::PacketTransportInternal* transport) { - return std::unique_ptr( - new webrtc::DcSctpTransport(network_thread_, transport, - webrtc::Clock::GetRealTimeClock())); -} \ No newline at end of file diff --git a/src/sctp_transport_factory.h b/src/sctp_transport_factory.h deleted file mode 100644 index 9f07f82..0000000 --- a/src/sctp_transport_factory.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef SCTP_TRANSPORT_FACTORY_H_ -#define SCTP_TRANSPORT_FACTORY_H_ - -#include - -// WebRTC -#include -#include -#include -#include - -class SctpTransportFactory : public webrtc::SctpTransportFactoryInterface { - public: - explicit SctpTransportFactory(rtc::Thread* network_thread); - - std::unique_ptr CreateSctpTransport( - rtc::PacketTransportInternal* transport) override; - - private: - rtc::Thread* network_thread_; -}; - -#endif \ No newline at end of file diff --git a/src/zakuro.cpp b/src/zakuro.cpp index cd76a7b..48a85cf 100644 --- a/src/zakuro.cpp +++ b/src/zakuro.cpp @@ -12,14 +12,12 @@ #include #include -#include "dynamic_h264_video_encoder.h" #include "enable_media_with_fake_call.h" #include "fake_audio_key_trigger.h" #include "fake_video_capturer.h" #include "game/game_kuzushi.h" #include "nop_video_decoder.h" #include "scenario_player.h" -#include "sctp_transport_factory.h" #include "util.h" #include "virtual_client.h" #include "wav_reader.h" @@ -358,23 +356,15 @@ int Zakuro::Run() { dependencies.worker_thread->BlockingCall( [&] { dependencies.adm = adm; }); - auto sw_config = sora::GetSoftwareOnlyVideoEncoderFactoryConfig(); + auto sw_config = sora::GetSoftwareOnlyVideoEncoderFactoryConfig( + vc.openh264.empty() ? std::optional(std::nullopt) + : std::optional(vc.openh264)); sw_config.use_simulcast_adapter = true; - sw_config.encoders.push_back(sora::VideoEncoderConfig( - webrtc::kVideoCodecH264, - [openh264 = vc.openh264]( - auto format) -> std::unique_ptr { - return webrtc::DynamicH264VideoEncoder::Create( - cricket::CreateVideoCodec(format), openh264); - })); dependencies.video_encoder_factory = absl::make_unique( std::move(sw_config)); dependencies.video_decoder_factory.reset(new NopVideoDecoderFactory()); - dependencies.sctp_factory.reset( - new SctpTransportFactory(dependencies.network_thread)); - EnableMediaWithFakeCall(dependencies, vc.fake_network_send, vc.fake_network_receive); };