Skip to content

Commit

Permalink
Merge pull request #89 from shiguredo/feature/vpl-h265-encoder
Browse files Browse the repository at this point in the history 
古い Intel CPU でも H265 エンコーダが動くようにする
  • Loading branch information
@enm10k
enm10k authored May 7, 2024
2 parents 8a8d29f + 097cebf commit d86d8af
Show file tree
Hide file tree
Showing 3 changed files with 178 additions and 106 deletions.
23 changes: 15 additions & 8 deletions .vscode/c_cpp_properties.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
"_LIBCPP_DISABLE_AVAILABILITY",
"OPENSSL_IS_BORINGSSL",
"SORA_CPP_SDK_UBUNTU_2004",
"USE_NVCODEC_ENCODER"
"USE_NVCODEC_ENCODER",
"RTC_ENABLE_H265"
],
"compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/llvm/clang/bin/clang++",
"compilerArgs": ["-nostdinc++"],
Expand Down Expand Up @@ -58,7 +59,8 @@
"OPENSSL_IS_BORINGSSL",
"USE_JETSON_ENCODER",
"SORA_CPP_SDK_JETSON",
"HELLO_JETSON"
"HELLO_JETSON",
"RTC_ENABLE_H265"
],
"compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_armv8_jetson/release/llvm/clang/bin/clang++",
"compilerArgs": ["-nostdinc++"],
Expand Down Expand Up @@ -92,7 +94,8 @@
"OPENSSL_IS_BORINGSSL",
"SORA_CPP_SDK_UBUNTU_2204",
"USE_NVCODEC_ENCODER",
"USE_VPL_ENCODER"
"USE_VPL_ENCODER",
"RTC_ENABLE_H265"
],
"compilerPath": "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/llvm/clang/bin/clang++",
"compilerArgs": ["-nostdinc++"],
Expand Down Expand Up @@ -123,7 +126,8 @@
"OPENSSL_IS_BORINGSSL",
"USE_NVCODEC_ENCODER=0",
"SORA_CPP_SDK_ANDROID",
"HELLO_ANDROID"
"HELLO_ANDROID",
"RTC_ENABLE_H265"
],
"compilerPath": "${workspaceFolder}/_install/android/release/llvm/clang/bin/clang++",
"compilerArgs": ["-nostdinc++"],
Expand All @@ -148,7 +152,8 @@
"defines": [
"WEBRTC_POSIX",
"WEBRTC_MAC",
"OPENSSL_IS_BORINGSSL"
"OPENSSL_IS_BORINGSSL",
"RTC_ENABLE_H265"
],
"cStandard": "gnu17",
"cppStandard": "gnu++17",
Expand All @@ -174,8 +179,9 @@
"defines": [
"SORA_CPP_SDK_WINDOWS",
"WEBRTC_WIN",
"USE_NVCODEC_ENCODER=1",
"USE_VPL_ENCODER=1"
"USE_NVCODEC_ENCODER",
"USE_VPL_ENCODER",
"RTC_ENABLE_H265"
],
"cStandard": "gnu17",
"cppStandard": "gnu++17",
Expand All @@ -200,7 +206,8 @@
"defines": [
"SORA_CPP_SDK_WINDOWS",
"WEBRTC_WIN",
"USE_NVCODEC_ENCODER=1"
"USE_NVCODEC_ENCODER",
"RTC_ENABLE_H265"
],
"cStandard": "gnu17",
"cppStandard": "gnu++17",
Expand Down
2 changes: 2 additions & 0 deletions CHANGES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
- @enm10k
- [ADD] SoraSignalingConfig に `video_h265_params` を追加
- @enm10k
- [ADD] 古い Intel CPU でも H265 エンコーダが動くようにする
- @melpon

## 2024.6.1 (2024-04-16)

Expand Down
259 changes: 161 additions & 98 deletions src/hwenc_vpl/vpl_video_encoder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,24 @@ class VplVideoEncoderImpl : public VplVideoEncoder {
int max_kbps,
bool init);

private:
struct ExtBuffer {
mfxExtBuffer* ext_buffers[10];
mfxExtCodingOption ext_coding_option;
mfxExtCodingOption2 ext_coding_option2;
};
// いろいろなパターンでクエリを投げて、
// 成功した時の param を返す
static mfxStatus Queries(MFXVideoENCODE* encoder,
mfxU32 codec,
int width,
int height,
int framerate,
int target_kbps,
int max_kbps,
mfxVideoParam& param,
ExtBuffer& ext);

private:
std::mutex mutex_;
webrtc::EncodedImageCallback* callback_ = nullptr;
Expand Down Expand Up @@ -103,9 +121,51 @@ std::unique_ptr<MFXVideoENCODE> VplVideoEncoderImpl::CreateEncoder(
int target_kbps,
int max_kbps,
bool init) {
mfxStatus sts = MFX_ERR_NONE;
std::unique_ptr<MFXVideoENCODE> encoder(
new MFXVideoENCODE(GetVplSession(session)));

mfxPlatform platform;
memset(&platform, 0, sizeof(platform));
MFXVideoCORE_QueryPlatform(GetVplSession(session), &platform);
RTC_LOG(LS_VERBOSE) << "--------------- codec=" << CodecToString(codec)
<< " CodeName=" << platform.CodeName
<< " DeviceId=" << platform.DeviceId
<< " MediaAdapterType=" << platform.MediaAdapterType;

mfxVideoParam param;
ExtBuffer ext;
mfxStatus sts = Queries(encoder.get(), codec, width, height, framerate,
target_kbps, max_kbps, param, ext);
if (sts < MFX_ERR_NONE) {
return nullptr;
}
if (sts > MFX_ERR_NONE) {
RTC_LOG(LS_VERBOSE) << "Supported specified codec but has warning: codec="
<< CodecToString(codec) << " sts=" << sts;
}

if (init) {
sts = encoder->Init(&param);
if (sts != MFX_ERR_NONE) {
RTC_LOG(LS_ERROR) << "Failed to Init: sts=" << sts;
return nullptr;
}
}

return encoder;
}

mfxStatus VplVideoEncoderImpl::Queries(MFXVideoENCODE* encoder,
mfxU32 codec,
int width,
int height,
int framerate,
int target_kbps,
int max_kbps,
mfxVideoParam& param,
ExtBuffer& ext) {
mfxStatus sts = MFX_ERR_NONE;

memset(&param, 0, sizeof(param));

param.mfx.CodecId = codec;
Expand Down Expand Up @@ -150,9 +210,9 @@ std::unique_ptr<MFXVideoENCODE> VplVideoEncoderImpl::CreateEncoder(
param.IOPattern =
MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY;

mfxExtBuffer* ext_buffers[10];
mfxExtCodingOption ext_coding_option;
mfxExtCodingOption2 ext_coding_option2;
mfxExtBuffer** ext_buffers = ext.ext_buffers;
mfxExtCodingOption& ext_coding_option = ext.ext_coding_option;
mfxExtCodingOption2& ext_coding_option2 = ext.ext_coding_option2;
int ext_buffers_size = 0;
if (codec == MFX_CODEC_AVC) {
memset(&ext_coding_option, 0, sizeof(ext_coding_option));
Expand Down Expand Up @@ -194,106 +254,105 @@ std::unique_ptr<MFXVideoENCODE> VplVideoEncoderImpl::CreateEncoder(
param.NumExtParam = ext_buffers_size;
}

std::unique_ptr<MFXVideoENCODE> encoder(
new MFXVideoENCODE(GetVplSession(session)));

mfxPlatform platform;
memset(&platform, 0, sizeof(platform));
MFXVideoCORE_QueryPlatform(GetVplSession(session), &platform);
RTC_LOG(LS_VERBOSE) << "--------------- codec=" << CodecToString(codec)
<< " CodeName=" << platform.CodeName
<< " DeviceId=" << platform.DeviceId
<< " MediaAdapterType=" << platform.MediaAdapterType;

// MFX_ERR_NONE The function completed successfully.
// MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features.
// MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code.
// MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved.
mfxVideoParam bk_param;
memcpy(&bk_param, &param, sizeof(bk_param));
sts = encoder->Query(&param, &param);
if (sts < 0) {
RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec="
<< CodecToString(codec) << " sts=" << sts
<< " ... Retry with low power mode";
memcpy(&param, &bk_param, sizeof(bk_param));

// 失敗したら LowPower ON にした状態でもう一度確認する
param.mfx.LowPower = MFX_CODINGOPTION_ON;
if (codec == MFX_CODEC_AVC || codec == MFX_CODEC_HEVC) {
param.mfx.RateControlMethod = MFX_RATECONTROL_CQP;
param.mfx.QPI = 25;
param.mfx.QPP = 33;
param.mfx.QPB = 40;
param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY;
}
memcpy(&bk_param, &param, sizeof(bk_param));
sts = encoder->Query(&param, &param);
if (sts < 0) {
RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec="
<< CodecToString(codec) << " sts=" << sts;
return nullptr;
// Query 関数を呼び出す。
// 失敗した場合 param は一切書き換わらない
// 成功した場合 param は書き換わる可能性がある
auto query = [](MFXVideoENCODE* encoder, mfxVideoParam& param) {
mfxVideoParam query_param;
memcpy(&query_param, &param, sizeof(param));
// ドキュメントによると、Query は以下のエラーを返す可能性がある。
// MFX_ERR_NONE The function completed successfully.
// MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features.
// MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code.
// MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved.
mfxStatus sts = encoder->Query(&query_param, &query_param);
if (sts >= 0) {
// デバッグ用。
// Query によってどのパラメータが変更されたかを表示する
// #define F(NAME) \
// if (param.NAME != query_param.NAME) \
// std::cout << "param " << #NAME << " old=" << param.NAME \
// << " new=" << query_param.NAME << std::endl
// F(mfx.LowPower);
// F(mfx.BRCParamMultiplier);
// F(mfx.FrameInfo.FrameRateExtN);
// F(mfx.FrameInfo.FrameRateExtD);
// F(mfx.FrameInfo.FourCC);
// F(mfx.FrameInfo.ChromaFormat);
// F(mfx.FrameInfo.PicStruct);
// F(mfx.FrameInfo.CropX);
// F(mfx.FrameInfo.CropY);
// F(mfx.FrameInfo.CropW);
// F(mfx.FrameInfo.CropH);
// F(mfx.FrameInfo.Width);
// F(mfx.FrameInfo.Height);
// F(mfx.CodecId);
// F(mfx.CodecProfile);
// F(mfx.CodecLevel);
// F(mfx.GopPicSize);
// F(mfx.GopRefDist);
// F(mfx.GopOptFlag);
// F(mfx.IdrInterval);
// F(mfx.TargetUsage);
// F(mfx.RateControlMethod);
// F(mfx.InitialDelayInKB);
// F(mfx.TargetKbps);
// F(mfx.MaxKbps);
// F(mfx.BufferSizeInKB);
// F(mfx.NumSlice);
// F(mfx.NumRefFrame);
// F(mfx.EncodedOrder);
// F(mfx.DecodedOrder);
// F(mfx.ExtendedPicStruct);
// F(mfx.TimeStampCalc);
// F(mfx.SliceGroupsPresent);
// F(mfx.MaxDecFrameBuffering);
// F(mfx.EnableReallocRequest);
// F(AsyncDepth);
// F(IOPattern);
// #undef F

memcpy(&param, &query_param, sizeof(param));
}
return sts;
};

// ここからは、ひたすらパラメータを変えて query を呼び出していく
sts = query(encoder, param);
if (sts >= 0) {
return sts;
}

//#define F(NAME) \
// if (bk_param.NAME != param.NAME) \
// std::cout << "param " << #NAME << " old=" << bk_param.NAME \
// << " new=" << param.NAME << std::endl
//
// F(mfx.LowPower);
// F(mfx.BRCParamMultiplier);
// F(mfx.FrameInfo.FrameRateExtN);
// F(mfx.FrameInfo.FrameRateExtD);
// F(mfx.FrameInfo.FourCC);
// F(mfx.FrameInfo.ChromaFormat);
// F(mfx.FrameInfo.PicStruct);
// F(mfx.FrameInfo.CropX);
// F(mfx.FrameInfo.CropY);
// F(mfx.FrameInfo.CropW);
// F(mfx.FrameInfo.CropH);
// F(mfx.FrameInfo.Width);
// F(mfx.FrameInfo.Height);
// F(mfx.CodecId);
// F(mfx.CodecProfile);
// F(mfx.CodecLevel);
// F(mfx.GopPicSize);
// F(mfx.GopRefDist);
// F(mfx.GopOptFlag);
// F(mfx.IdrInterval);
// F(mfx.TargetUsage);
// F(mfx.RateControlMethod);
// F(mfx.InitialDelayInKB);
// F(mfx.TargetKbps);
// F(mfx.MaxKbps);
// F(mfx.BufferSizeInKB);
// F(mfx.NumSlice);
// F(mfx.NumRefFrame);
// F(mfx.EncodedOrder);
// F(mfx.DecodedOrder);
// F(mfx.ExtendedPicStruct);
// F(mfx.TimeStampCalc);
// F(mfx.SliceGroupsPresent);
// F(mfx.MaxDecFrameBuffering);
// F(mfx.EnableReallocRequest);
// F(AsyncDepth);
// F(IOPattern);
//#undef F

if (sts != MFX_ERR_NONE) {
RTC_LOG(LS_VERBOSE) << "Supported specified codec but has warning: codec="
<< CodecToString(codec) << " sts=" << sts;
// IOPattern を MFX_IOPATTERN_IN_SYSTEM_MEMORY のみにしてみる
// Coffee Lake の H265 はこのパターンでないと通らない
RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec="
<< CodecToString(codec) << " sts=" << sts
<< " ... Retry with IOPattern IN_SYSTEM_MEMORY only";
param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY;
sts = query(encoder, param);
if (sts >= 0) {
return sts;
}

if (init) {
sts = encoder->Init(&param);
if (sts != MFX_ERR_NONE) {
RTC_LOG(LS_VERBOSE) << "Failed to Init: sts=" << sts;
return nullptr;
}
// LowPower ON にして、更に H264/H265 は固定 QP モードにしてみる
RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec="
<< CodecToString(codec) << " sts=" << sts
<< " ... Retry with low power mode";
param.mfx.LowPower = MFX_CODINGOPTION_ON;
if (codec == MFX_CODEC_AVC || codec == MFX_CODEC_HEVC) {
param.mfx.RateControlMethod = MFX_RATECONTROL_CQP;
param.mfx.QPI = 25;
param.mfx.QPP = 33;
param.mfx.QPB = 40;
}
sts = query(encoder, param);
if (sts >= 0) {
return sts;
}
RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec="
<< CodecToString(codec) << " sts=" << sts;

return encoder;
return sts;
}

int32_t VplVideoEncoderImpl::InitEncode(
Expand Down Expand Up @@ -610,7 +669,11 @@ bool VplVideoEncoder::IsSupported(std::shared_ptr<VplSession> session,

auto encoder = VplVideoEncoderImpl::CreateEncoder(
session, ToMfxCodec(codec), 1920, 1080, 30, 10, 20, false);
return encoder != nullptr;
bool result = encoder != nullptr;
RTC_LOG(LS_VERBOSE) << "IsSupported: codec="
<< CodecToString(ToMfxCodec(codec))
<< " result=" << result;
return result;
}

std::unique_ptr<VplVideoEncoder> VplVideoEncoder::Create(
Expand Down

0 comments on commit d86d8af

Please sign in to comment.