From 4f84acc48aebe9cd1bbf001ff9b9639f57c7777b Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 2 Apr 2024 06:15:04 +0900 Subject: [PATCH 1/2] =?UTF-8?q?=E5=8F=A4=E3=81=84=20Intel=20CPU=20?= =?UTF-8?q?=E3=81=A7=E3=82=82=20H265=20=E3=82=A8=E3=83=B3=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=80=E3=81=8C=E5=8B=95=E3=81=8F=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/c_cpp_properties.json | 23 ++- CHANGES.md | 3 + src/hwenc_vpl/vpl_video_encoder.cpp | 259 +++++++++++++++++----------- 3 files changed, 179 insertions(+), 106 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 72873002..9e91c09c 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -26,7 +26,8 @@ "_LIBCPP_DISABLE_AVAILABILITY", "OPENSSL_IS_BORINGSSL", "SORA_CPP_SDK_UBUNTU_2004", - "USE_NVCODEC_ENCODER" + "USE_NVCODEC_ENCODER", + "RTC_ENABLE_H265" ], "compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], @@ -58,7 +59,8 @@ "OPENSSL_IS_BORINGSSL", "USE_JETSON_ENCODER", "SORA_CPP_SDK_JETSON", - "HELLO_JETSON" + "HELLO_JETSON", + "RTC_ENABLE_H265" ], "compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_armv8_jetson/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], @@ -92,7 +94,8 @@ "OPENSSL_IS_BORINGSSL", "SORA_CPP_SDK_UBUNTU_2204", "USE_NVCODEC_ENCODER", - "USE_VPL_ENCODER" + "USE_VPL_ENCODER", + "RTC_ENABLE_H265" ], "compilerPath": "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], @@ -123,7 +126,8 @@ "OPENSSL_IS_BORINGSSL", "USE_NVCODEC_ENCODER=0", "SORA_CPP_SDK_ANDROID", - "HELLO_ANDROID" + "HELLO_ANDROID", + "RTC_ENABLE_H265" ], "compilerPath": "${workspaceFolder}/_install/android/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], @@ -148,7 +152,8 @@ "defines": [ "WEBRTC_POSIX", "WEBRTC_MAC", - "OPENSSL_IS_BORINGSSL" + "OPENSSL_IS_BORINGSSL", + "RTC_ENABLE_H265" ], "cStandard": "gnu17", "cppStandard": "gnu++17", @@ -174,8 +179,9 @@ "defines": [ "SORA_CPP_SDK_WINDOWS", "WEBRTC_WIN", - "USE_NVCODEC_ENCODER=1", - "USE_VPL_ENCODER=1" + "USE_NVCODEC_ENCODER", + "USE_VPL_ENCODER", + "RTC_ENABLE_H265" ], "cStandard": "gnu17", "cppStandard": "gnu++17", @@ -200,7 +206,8 @@ "defines": [ "SORA_CPP_SDK_WINDOWS", "WEBRTC_WIN", - "USE_NVCODEC_ENCODER=1" + "USE_NVCODEC_ENCODER", + "RTC_ENABLE_H265" ], "cStandard": "gnu17", "cppStandard": "gnu++17", diff --git a/CHANGES.md b/CHANGES.md index 81dd5362..97074ddb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,9 @@ ## develop +- [ADD] 古い Intel CPU でも H265 エンコーダが動くようにする + - @melpon + ## 2024.6.0 (2024-04-01) - [CHANGE] `VplVideoDecoderImpl` の `ImplementationName` を `oneVPL` から `libvpl` に変更する diff --git a/src/hwenc_vpl/vpl_video_encoder.cpp b/src/hwenc_vpl/vpl_video_encoder.cpp index 24aac7ab..a531368e 100644 --- a/src/hwenc_vpl/vpl_video_encoder.cpp +++ b/src/hwenc_vpl/vpl_video_encoder.cpp @@ -51,6 +51,24 @@ class VplVideoEncoderImpl : public VplVideoEncoder { int max_kbps, bool init); + private: + struct ExtBuffer { + mfxExtBuffer* ext_buffers[10]; + mfxExtCodingOption ext_coding_option; + mfxExtCodingOption2 ext_coding_option2; + }; + // いろいろなパターンでクエリを投げて、 + // 成功した時の param を返す + static mfxStatus Queries(MFXVideoENCODE* encoder, + mfxU32 codec, + int width, + int height, + int framerate, + int target_kbps, + int max_kbps, + mfxVideoParam& param, + ExtBuffer& ext); + private: std::mutex mutex_; webrtc::EncodedImageCallback* callback_ = nullptr; @@ -103,9 +121,51 @@ std::unique_ptr VplVideoEncoderImpl::CreateEncoder( int target_kbps, int max_kbps, bool init) { - mfxStatus sts = MFX_ERR_NONE; + std::unique_ptr encoder( + new MFXVideoENCODE(GetVplSession(session))); + + mfxPlatform platform; + memset(&platform, 0, sizeof(platform)); + MFXVideoCORE_QueryPlatform(GetVplSession(session), &platform); + RTC_LOG(LS_VERBOSE) << "--------------- codec=" << CodecToString(codec) + << " CodeName=" << platform.CodeName + << " DeviceId=" << platform.DeviceId + << " MediaAdapterType=" << platform.MediaAdapterType; mfxVideoParam param; + ExtBuffer ext; + mfxStatus sts = Queries(encoder.get(), codec, width, height, framerate, + target_kbps, max_kbps, param, ext); + if (sts < MFX_ERR_NONE) { + return nullptr; + } + if (sts > MFX_ERR_NONE) { + RTC_LOG(LS_VERBOSE) << "Supported specified codec but has warning: codec=" + << CodecToString(codec) << " sts=" << sts; + } + + if (init) { + sts = encoder->Init(¶m); + if (sts != MFX_ERR_NONE) { + RTC_LOG(LS_ERROR) << "Failed to Init: sts=" << sts; + return nullptr; + } + } + + return encoder; +} + +mfxStatus VplVideoEncoderImpl::Queries(MFXVideoENCODE* encoder, + mfxU32 codec, + int width, + int height, + int framerate, + int target_kbps, + int max_kbps, + mfxVideoParam& param, + ExtBuffer& ext) { + mfxStatus sts = MFX_ERR_NONE; + memset(¶m, 0, sizeof(param)); param.mfx.CodecId = codec; @@ -150,9 +210,9 @@ std::unique_ptr VplVideoEncoderImpl::CreateEncoder( param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; - mfxExtBuffer* ext_buffers[10]; - mfxExtCodingOption ext_coding_option; - mfxExtCodingOption2 ext_coding_option2; + mfxExtBuffer** ext_buffers = ext.ext_buffers; + mfxExtCodingOption& ext_coding_option = ext.ext_coding_option; + mfxExtCodingOption2& ext_coding_option2 = ext.ext_coding_option2; int ext_buffers_size = 0; if (codec == MFX_CODEC_AVC) { memset(&ext_coding_option, 0, sizeof(ext_coding_option)); @@ -194,106 +254,105 @@ std::unique_ptr VplVideoEncoderImpl::CreateEncoder( param.NumExtParam = ext_buffers_size; } - std::unique_ptr encoder( - new MFXVideoENCODE(GetVplSession(session))); - - mfxPlatform platform; - memset(&platform, 0, sizeof(platform)); - MFXVideoCORE_QueryPlatform(GetVplSession(session), &platform); - RTC_LOG(LS_VERBOSE) << "--------------- codec=" << CodecToString(codec) - << " CodeName=" << platform.CodeName - << " DeviceId=" << platform.DeviceId - << " MediaAdapterType=" << platform.MediaAdapterType; - - // MFX_ERR_NONE The function completed successfully. - // MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features. - // MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code. - // MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved. - mfxVideoParam bk_param; - memcpy(&bk_param, ¶m, sizeof(bk_param)); - sts = encoder->Query(¶m, ¶m); - if (sts < 0) { - RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" - << CodecToString(codec) << " sts=" << sts - << " ... Retry with low power mode"; - memcpy(¶m, &bk_param, sizeof(bk_param)); - - // 失敗したら LowPower ON にした状態でもう一度確認する - param.mfx.LowPower = MFX_CODINGOPTION_ON; - if (codec == MFX_CODEC_AVC || codec == MFX_CODEC_HEVC) { - param.mfx.RateControlMethod = MFX_RATECONTROL_CQP; - param.mfx.QPI = 25; - param.mfx.QPP = 33; - param.mfx.QPB = 40; - param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY; - } - memcpy(&bk_param, ¶m, sizeof(bk_param)); - sts = encoder->Query(¶m, ¶m); - if (sts < 0) { - RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" - << CodecToString(codec) << " sts=" << sts; - return nullptr; + // Query 関数を呼び出す。 + // 失敗した場合 param は一切書き換わらない + // 成功した場合 param は書き換わる可能性がある + auto query = [](MFXVideoENCODE* encoder, mfxVideoParam& param) { + mfxVideoParam query_param; + memcpy(&query_param, ¶m, sizeof(param)); + // ドキュメントによると、Query は以下のエラーを返す可能性がある。 + // MFX_ERR_NONE The function completed successfully. + // MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features. + // MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code. + // MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved. + mfxStatus sts = encoder->Query(&query_param, &query_param); + if (sts >= 0) { + // デバッグ用。 + // Query によってどのパラメータが変更されたかを表示する + // #define F(NAME) \ + // if (param.NAME != query_param.NAME) \ + // std::cout << "param " << #NAME << " old=" << param.NAME \ + // << " new=" << query_param.NAME << std::endl + // F(mfx.LowPower); + // F(mfx.BRCParamMultiplier); + // F(mfx.FrameInfo.FrameRateExtN); + // F(mfx.FrameInfo.FrameRateExtD); + // F(mfx.FrameInfo.FourCC); + // F(mfx.FrameInfo.ChromaFormat); + // F(mfx.FrameInfo.PicStruct); + // F(mfx.FrameInfo.CropX); + // F(mfx.FrameInfo.CropY); + // F(mfx.FrameInfo.CropW); + // F(mfx.FrameInfo.CropH); + // F(mfx.FrameInfo.Width); + // F(mfx.FrameInfo.Height); + // F(mfx.CodecId); + // F(mfx.CodecProfile); + // F(mfx.CodecLevel); + // F(mfx.GopPicSize); + // F(mfx.GopRefDist); + // F(mfx.GopOptFlag); + // F(mfx.IdrInterval); + // F(mfx.TargetUsage); + // F(mfx.RateControlMethod); + // F(mfx.InitialDelayInKB); + // F(mfx.TargetKbps); + // F(mfx.MaxKbps); + // F(mfx.BufferSizeInKB); + // F(mfx.NumSlice); + // F(mfx.NumRefFrame); + // F(mfx.EncodedOrder); + // F(mfx.DecodedOrder); + // F(mfx.ExtendedPicStruct); + // F(mfx.TimeStampCalc); + // F(mfx.SliceGroupsPresent); + // F(mfx.MaxDecFrameBuffering); + // F(mfx.EnableReallocRequest); + // F(AsyncDepth); + // F(IOPattern); + // #undef F + + memcpy(¶m, &query_param, sizeof(param)); } + return sts; + }; + + // ここからは、ひたすらパラメータを変えて query を呼び出していく + sts = query(encoder, param); + if (sts >= 0) { + return sts; } - //#define F(NAME) \ - // if (bk_param.NAME != param.NAME) \ - // std::cout << "param " << #NAME << " old=" << bk_param.NAME \ - // << " new=" << param.NAME << std::endl - // - // F(mfx.LowPower); - // F(mfx.BRCParamMultiplier); - // F(mfx.FrameInfo.FrameRateExtN); - // F(mfx.FrameInfo.FrameRateExtD); - // F(mfx.FrameInfo.FourCC); - // F(mfx.FrameInfo.ChromaFormat); - // F(mfx.FrameInfo.PicStruct); - // F(mfx.FrameInfo.CropX); - // F(mfx.FrameInfo.CropY); - // F(mfx.FrameInfo.CropW); - // F(mfx.FrameInfo.CropH); - // F(mfx.FrameInfo.Width); - // F(mfx.FrameInfo.Height); - // F(mfx.CodecId); - // F(mfx.CodecProfile); - // F(mfx.CodecLevel); - // F(mfx.GopPicSize); - // F(mfx.GopRefDist); - // F(mfx.GopOptFlag); - // F(mfx.IdrInterval); - // F(mfx.TargetUsage); - // F(mfx.RateControlMethod); - // F(mfx.InitialDelayInKB); - // F(mfx.TargetKbps); - // F(mfx.MaxKbps); - // F(mfx.BufferSizeInKB); - // F(mfx.NumSlice); - // F(mfx.NumRefFrame); - // F(mfx.EncodedOrder); - // F(mfx.DecodedOrder); - // F(mfx.ExtendedPicStruct); - // F(mfx.TimeStampCalc); - // F(mfx.SliceGroupsPresent); - // F(mfx.MaxDecFrameBuffering); - // F(mfx.EnableReallocRequest); - // F(AsyncDepth); - // F(IOPattern); - //#undef F - - if (sts != MFX_ERR_NONE) { - RTC_LOG(LS_VERBOSE) << "Supported specified codec but has warning: codec=" - << CodecToString(codec) << " sts=" << sts; + // IOPattern を MFX_IOPATTERN_IN_SYSTEM_MEMORY のみにしてみる + // Coffee Lake の H265 はこのパターンでないと通らない + RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" + << CodecToString(codec) << " sts=" << sts + << " ... Retry with IOPattern IN_SYSTEM_MEMORY only"; + param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY; + sts = query(encoder, param); + if (sts >= 0) { + return sts; } - if (init) { - sts = encoder->Init(¶m); - if (sts != MFX_ERR_NONE) { - RTC_LOG(LS_VERBOSE) << "Failed to Init: sts=" << sts; - return nullptr; - } + // LowPower ON にして、更に H264/H265 は固定 QP モードにしてみる + RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" + << CodecToString(codec) << " sts=" << sts + << " ... Retry with low power mode"; + param.mfx.LowPower = MFX_CODINGOPTION_ON; + if (codec == MFX_CODEC_AVC || codec == MFX_CODEC_HEVC) { + param.mfx.RateControlMethod = MFX_RATECONTROL_CQP; + param.mfx.QPI = 25; + param.mfx.QPP = 33; + param.mfx.QPB = 40; + } + sts = query(encoder, param); + if (sts >= 0) { + return sts; } + RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" + << CodecToString(codec) << " sts=" << sts; - return encoder; + return sts; } int32_t VplVideoEncoderImpl::InitEncode( @@ -610,7 +669,11 @@ bool VplVideoEncoder::IsSupported(std::shared_ptr session, auto encoder = VplVideoEncoderImpl::CreateEncoder( session, ToMfxCodec(codec), 1920, 1080, 30, 10, 20, false); - return encoder != nullptr; + bool result = encoder != nullptr; + RTC_LOG(LS_VERBOSE) << "IsSupported: codec=" + << CodecToString(ToMfxCodec(codec)) + << " result=" << result; + return result; } std::unique_ptr VplVideoEncoder::Create( From 097cebf00fa152feaa67b63cecfdc0ef8e920438 Mon Sep 17 00:00:00 2001 From: enm10k Date: Tue, 7 May 2024 13:53:47 +0900 Subject: [PATCH 2/2] =?UTF-8?q?=E3=83=9E=E3=83=BC=E3=82=B8=E3=82=92?= =?UTF-8?q?=E3=83=9F=E3=82=B9=E3=81=97=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE?= =?UTF-8?q?=E3=81=A7=E4=BF=AE=E6=AD=A3=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 29d4345d..e4c1bee7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,7 +33,6 @@ - [CHANGE] テストモジュールについて `SoraSignalingConfig` の `sora_client` に値を設定しないようにする - @enm10k - [FIX] VPL デコーダで遅延が起きたりセグフォすることがあるのを修正 ->>>>>>> origin/develop - @melpon ## 2024.6.0 (2024-04-01)