diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d05ee8b8..bc57aa72 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: shell: bash run: | sudo apt update - sudo apt install -yq yasm ccache libstdc++-10-dev-arm64-cross libc6-dev-arm64-cross binutils-aarch64-linux-gnu + sudo apt install -yq yasm ccache libstdc++-10-dev-arm64-cross libc6-dev-arm64-cross binutils-aarch64-linux-gnu libva-dev libdrm-dev sudo rm -rf /var/lib/apt/lists/* env: DEBIAN_FRONTEND: noninteractive @@ -49,6 +49,7 @@ jobs: run: | source VERSION echo "libvpx_version=${LIBVPX_VERSION}" >> "$GITHUB_OUTPUT" + echo "svt_av1_version=${SVT_AV1_VERSION}" >> "$GITHUB_OUTPUT" - name: Cache libvpx ${{ steps.versions.outputs.libvpx_version }} id: cache-libvpx uses: actions/cache@v3 @@ -59,6 +60,16 @@ jobs: - name: Clear libvpx if: steps.cache-libvpx.outputs.cache-hit != 'true' run: rm -rf third_party/libvpx + - name: Cache SVT-AV1 ${{ steps.versions.outputs.svt_av1_version }} + id: cache-svt-av1 + uses: actions/cache@v3 + with: + path: third_party/SVT-AV1 + key: ${{ matrix.os }}-third_party-svt-av1-${{ hashFiles('third_party/SVT-AV1/Bin/Release/libSvtAv1Dec.a', 'third_party/SVT-AV1/Bin/Release/libSvtAv1Enc.a') }} + restore-keys: ${{ matrix.os }}-third_party-svt-av1- + - name: Clear SVT-AV1 + if: steps.cache-svt-av1.outputs.cache-hit != 'true' + run: rm -rf third_party/SVT-AV1 - name: Cache release directory id: cache-release uses: actions/cache@v3 @@ -82,6 +93,13 @@ jobs: restore-keys: ${{ matrix.os }}-ccache-dir- - name: Ccache stat run: ccache -s + - name: Cache ~/.cache/bazel directory + uses: actions/cache@v3 + with: + path: "~/.cache/bazel" + key: ${{ matrix.os }}-cache-bazel-dir + - name: Install numpy + run: pip install numpy - name: Build hisui run: ./build.bash ${{ matrix.os }} --package --use-ccache ${{ contains(matrix.os, '_x86_64') && '--with-test' || '' }} timeout-minutes: 120 diff --git a/.gitignore b/.gitignore index b38f67f5..dfe7ecf1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ /third_party/libvpx +/third_party/lyra/bazel-* +/third_party/lyra/lyra +/third_party/SVT-AV1 /test/integration/input/caminandes3.webm /test/integration/input/lilac_blossom_bloom_spring.jpg /test/integration/input/Big_Buck_Bunny_360_10s_1MB.webm diff --git a/CHANGES.md b/CHANGES.md index 10c0d06b..d7822d18 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,16 @@ ## develop +- [UPDATE] 依存ライブラリの更新をする + - `boost` を `1.83.0` にあげる + - `cpp-mp4` を `2023.2.1` にあげる + - `opus` を `1.4` にあげる +- [CHANGE] one VPL での H.264 エンコードに対応する + - @haruyama +- [CHANGE] Lyra の デコードに対応する + - @haruyama +- [CHANGE] SVT-AV1 での AV1 デコード/エンコードに対応する + - @haruyama - [CHANGE] OpenH264 での エンコードに対応する - `--out-video-codec` オプションに H.264 の指定を追加する - 画面共有合成機能では H.264 はサポートされない diff --git a/CMakeLists.txt b/CMakeLists.txt index b45ce0c8..c85f8f21 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,11 +7,22 @@ cmake_policy(SET CMP0091 NEW) include(cmake/env.cmake) -set_cache_string_from_env(BOOST_VERSION 1.81.0 "Boost のバージョン") +if(DEFINED CMAKE_BUILD_TYPE) + if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") + set(SVT_AV1_BUILD_TYPE "Debug") + else() + set(SVT_AV1_BUILD_TYPE "Release") + endif() +else() + set(SVT_AV1_BUILD_TYPE "Release") +endif() + +set_cache_string_from_env(BOOST_VERSION 1.83.0 "Boost のバージョン") set_cache_string_from_env(CLI11_VERSION 2.3.2 "CLI11 のバージョン") -set_cache_string_from_env(CPP_MP4_VERSION 7158ce0c19c1a7e9c8fcc34339e381766235abb5 "cpp-mp4 のバージョン") +set_cache_string_from_env(CPP_MP4_VERSION 2023.2.1 "cpp-mp4 のバージョン") set_cache_string_from_env(FMT_VERSION 9.1.0 "fmt のバージョン") -set_cache_string_from_env(OPUS_VERSION 1.3.1 "Opus のバージョン") +set_cache_string_from_env(ONEVPL_VERSION 2023.3.1 "oneVPL のバージョン") +set_cache_string_from_env(OPUS_VERSION 1.4 "Opus のバージョン") set_cache_string_from_env(SPDLOG_VERSION 1.11.0 "spdlog のバージョン") set_cache_string_from_env(STB_VERSION 5736b15f7ea0ffb08dd38af21067c314d6a3aae9 "stb のバージョン") @@ -43,6 +54,7 @@ set(CMAKE_C_FLAGS_PROFILE "-pg -O3 -DNDEBUG -march=native -mtune=native") include(cmake/tools.cmake) + CPMAddPackage( NAME boost_align GITHUB_REPOSITORY boostorg/align @@ -71,6 +83,13 @@ CPMAddPackage( DOWNLOAD_ONLY YES ) +CPMAddPackage( + NAME boost_container_hash + GITHUB_REPOSITORY boostorg/container_hash + GIT_TAG boost-${BOOST_VERSION} + DOWNLOAD_ONLY YES + ) + CPMAddPackage( NAME boost_core GITHUB_REPOSITORY boostorg/core @@ -255,11 +274,53 @@ if(USE_FDK_AAC) ) endif() +option(USE_ONEVPL "Use oneVPL" OFF) + +if(USE_ONEVPL) + target_compile_definitions(hisui + PRIVATE + USE_ONEVPL + ) + + CPMAddPackage( + NAME vpl + GITHUB_REPOSITORY oneapi-src/oneVPL + VERSION ${ONEVPL_VERSION} + OPTIONS "BUILD_SHARED_LIBS Off" "BUILD_TOOLS Off" "BUILD_EXAMPLES Off" "BUILD_PREVIEW Off" "BUILD_TOOLS_ONEVPL_EXPERIMENTAL Off" + ) + + target_sources(hisui + PRIVATE + src/layout/vpl_video_producer.cpp + src/muxer/vpl_video_producer.cpp + src/video/vaapi_utils.cpp + src/video/vaapi_utils_drm.cpp + src/video/vpl.cpp + src/video/vpl_encoder.cpp + src/video/vpl_session.cpp + ) + + target_include_directories(hisui + PRIVATE + ${VPL_SOURCE_DIR}/api + ) + + target_link_libraries(hisui + PRIVATE + drm + va + va-drm + VPL + ) +endif() + target_sources(hisui PRIVATE src/archive_item.cpp src/audio/basic_sequencer.cpp src/audio/buffer_opus_encoder.cpp + src/audio/lyra_decoder.cpp + src/audio/lyra_handler.cpp src/audio/mixer.cpp src/audio/opus.cpp src/audio/opus_decoder.cpp @@ -268,6 +329,7 @@ target_sources(hisui src/datetime.cpp src/hisui.cpp src/layout/archive.cpp + src/layout/av1_video_producer.cpp src/layout/cell.cpp src/layout/cell_util.cpp src/layout/compose.cpp @@ -284,6 +346,7 @@ target_sources(hisui src/metadata.cpp src/muxer/async_webm_muxer.cpp src/muxer/audio_producer.cpp + src/muxer/av1_video_producer.cpp src/muxer/faststart_mp4_muxer.cpp src/muxer/mp4_muxer.cpp src/muxer/multi_channel_vpx_video_producer.cpp @@ -300,11 +363,15 @@ target_sources(hisui src/util/json.cpp src/util/wildcard.cpp src/version/version.cpp + src/video/av1_decoder.cpp src/video/basic_sequencer.cpp + src/video/buffer_av1_encoder.cpp src/video/buffer_openh264_encoder.cpp src/video/buffer_vpx_encoder.cpp + src/video/codec_engine.cpp src/video/composer.cpp src/video/decoder.cpp + src/video/decoder_factory.cpp src/video/grid_composer.cpp src/video/image_source.cpp src/video/multi_channel_sequencer.cpp @@ -342,6 +409,9 @@ target_sources(hisui third_party/libvpx/third_party/libyuv/source/scale_neon64.cc third_party/libvpx/third_party/libyuv/source/scale_win.cc third_party/libvpx/third_party/libyuv/source/convert.cc + third_party/libvpx/third_party/libyuv/source/convert_argb.cc + third_party/libvpx/third_party/libyuv/source/convert_from.cc + third_party/libvpx/third_party/libyuv/source/video_common.cc third_party/libvpx/third_party/libwebm/mkvparser/mkvparser.cc third_party/libvpx/third_party/libwebm/mkvparser/mkvreader.cc third_party/libvpx/third_party/libwebm/mkvmuxer/mkvmuxer.cc @@ -352,10 +422,12 @@ target_sources(hisui target_include_directories(hisui PRIVATE src + ${abseil_cpp_SOURCE_DIR}/include ${boost_align_SOURCE_DIR}/include ${boost_assert_SOURCE_DIR}/include ${boost_config_SOURCE_DIR}/include ${boost_container_SOURCE_DIR}/include + ${boost_container_hash_SOURCE_DIR}/include ${boost_core_SOURCE_DIR}/include ${boost_describe_SOURCE_DIR}/include ${boost_exception_SOURCE_DIR}/include @@ -376,6 +448,8 @@ target_include_directories(hisui ${cli11_SOURCE_DIR}/include ${cpp-mp4_SOURCE_DIR}/include ${fmt_SOURCE_DIR}/include + ${google_glog_SOURCE_DIR}/src + ${gulrak_filesystem_SOURCE_DIR} ${opus_SOURCE_DIR}/include ${spdlog_SOURCE_DIR}/include ${stb_SOURCE_DIR} @@ -384,6 +458,8 @@ target_include_directories(hisui third_party/libvpx/third_party/libyuv/include third_party/libvpx/third_party/libwebm third_party/libvpx + third_party/SVT-AV1/Source/API + third_party/lyra ) set_target_properties(hisui PROPERTIES CXX_STANDARD 20 C_STANDARD 11) @@ -397,6 +473,9 @@ target_link_libraries(hisui shiguredo-mp4 spdlog ${CMAKE_SOURCE_DIR}/third_party/libvpx/${HISUI_PACKAGE}/libvpx.a + ${CMAKE_SOURCE_DIR}/third_party/SVT-AV1/Bin/${SVT_AV1_BUILD_TYPE}/libSvtAv1Dec.a + ${CMAKE_SOURCE_DIR}/third_party/SVT-AV1/Bin/${SVT_AV1_BUILD_TYPE}/libSvtAv1Enc.a + ${CMAKE_SOURCE_DIR}/third_party/lyra/bazel-bin/liblyra.a ) if(WITH_TEST) @@ -416,13 +495,6 @@ if(WITH_TEST) DOWNLOAD_ONLY YES ) - CPMAddPackage( - NAME boost_container_hash - GITHUB_REPOSITORY boostorg/container_hash - GIT_TAG boost-${BOOST_VERSION} - DOWNLOAD_ONLY YES - ) - CPMAddPackage( NAME boost_detail GITHUB_REPOSITORY boostorg/detail diff --git a/NOTICE.md b/NOTICE.md index ed6d074f..08409a30 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -394,6 +394,244 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ``` +## Lyra + +https://github.com/google/lyra + +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` + +## oneVPL + +https://github.com/oneapi-src/oneVPL + +``` +MIT License + +Copyright (c) 2020 Intel Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + + ## OpenH264 https://github.com/cisco/openh264 @@ -581,3 +819,25 @@ AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` + +## SVT-AV1 + +https://gitlab.com/AOMediaCodec/SVT-AV1 + +``` +BSD 3-Clause Clear License The Clear BSD License + +Copyright (c) 2021, Alliance for Open Media + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted (subject to the limitations in the disclaimer below) provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + + Neither the name of the Alliance for Open Media nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` diff --git a/VERSION b/VERSION index b1589248..02d24ba4 100644 --- a/VERSION +++ b/VERSION @@ -1,11 +1,14 @@ HISUI_VERSION=2023.1.1 -BOOST_VERSION=1.81.0 +BOOST_VERSION=1.83.0 CLI11_VERSION=2.3.2 -CPP_MP4_VERSION=7158ce0c19c1a7e9c8fcc34339e381766235abb5 +CPP_MP4_VERSION=2023.2.1 FMT_VERSION=9.1.0 LIBVPX_VERSION=1.13.0 -OPUS_VERSION=1.3.1 +LYRA_VERSION=1.3.2 +ONEVPL_VERSION=2023.3.1 +OPUS_VERSION=1.4 PROGRESSCPP_ORIGINAL_VERSION=7bfba0d22d19c41323aa35541618b6ebec9d737c SPDLOG_VERSION=1.11.0 STB_VERSION=5736b15f7ea0ffb08dd38af21067c314d6a3aae9 +SVT_AV1_VERSION=1.7.0 diff --git a/build.bash b/build.bash index 85ce25f7..595ceb26 100755 --- a/build.bash +++ b/build.bash @@ -124,13 +124,14 @@ fi cd third_party || exit 1 +# libvpx if [ -d libvpx ] ; then cd libvpx || exit 1 patch -p1 -R < ../libwebm.patch || echo "reverse patch failed" git checkout main git pull else - git clone https://chromium.googlesource.com/webm/libvpx + git clone --filter=tree:0 https://chromium.googlesource.com/webm/libvpx cd libvpx || exit 1 fi git checkout v"${LIBVPX_VERSION}" @@ -146,6 +147,7 @@ CMAKE_FLAGS+=("-DHISUI_PACKAGE=$PACKAGE") case "$PACKAGE" in *_x86_64 ) CMAKE_FLAGS+=("-DCMAKE_TOOLCHAIN_FILE=../../cmake/clang-x86_64-toolchain.cmake") + CMAKE_FLAGS+=('-DUSE_ONEVPL=YES') ;; *_arm64 ) CMAKE_FLAGS+=("-DCMAKE_TOOLCHAIN_FILE=../../cmake/clang-aarch64-toolchain.cmake") @@ -163,8 +165,110 @@ cd "$PACKAGE" || exit 1 CXX="$CXX" CC="$CC" ../configure "${libvpx_configure_options[@]}" || (cat config.log && exit 1) make - cd ../../.. + +# SVT-AV1 +cd third_party || exit 1 +if [ -d SVT-AV1 ] ; then + cd SVT-AV1 || exit 1 + git checkout master + git pull +else + git clone --filter=tree:0 https://gitlab.com/AOMediaCodec/SVT-AV1.git + cd SVT-AV1 || exit 1 +fi +git checkout v"${SVT_AV1_VERSION}" + +stv_av1_configure_options=('--static' '--no-apps') +if [ "${BUILD_TYPE}" = "Native" ]; then + SVT_AV1_BUILD_TYPE="Release" + stv_av1_configure_options+=('--native' 'release') +elif [ "${BUILD_TYPE}" = "Release" ]; then + SVT_AV1_BUILD_TYPE="Release" + stv_av1_configure_options+=('release') +elif [ "${BUILD_TYPE}" = "Debug" ]; then + SVT_AV1_BUILD_TYPE="Debug" + stv_av1_configure_options+=('debug') +fi + +case "$PACKAGE" in + *_x86_64 ) + stv_av1_configure_options+=('-t' '../../../../cmake/clang-x86_64-toolchain.cmake') + objcopy=/usr/bin/objcopy + ;; + *_arm64 ) + stv_av1_configure_options+=('-s' 'aarch64-linux-gnu' '-t' '../../../../cmake/clang-aarch64-toolchain.cmake') + objcopy=/usr/aarch64-linux-gnu/bin/objcopy +esac +cd Build/linux || exit 1 + +./build.sh "${stv_av1_configure_options[@]}" || exit 1 + +cd ../../../.. + +${objcopy} --redefine-sym cpuinfo_is_initialized=local_cpuinfo_is_initialized third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_initialize=local_cpuinfo_initialize third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_deinitialize=local_cpuinfo_deinitialize third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_get_core=local_cpuinfo_get_core third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_isa=local_cpuinfo_isa third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_x86_linux_init=local_cpuinfo_x86_linux_init third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_x86_decode_vendor=local_cpuinfo_x86_decode_vendor third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_x86_init_processor=local_cpuinfo_x86_init_processor third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a +${objcopy} --redefine-sym cpuinfo_x86_detect_isa=local_cpuinfo_x86_detect_isa third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Dec.a + +${objcopy} --redefine-sym cpuinfo_is_initialized=local_cpuinfo_is_initialized third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_initialize=local_cpuinfo_initialize third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_deinitialize=local_cpuinfo_deinitialize third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_get_core=local_cpuinfo_get_core third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_isa=local_cpuinfo_isa third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_x86_linux_init=local_cpuinfo_x86_linux_init third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_x86_decode_vendor=local_cpuinfo_x86_decode_vendor third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_x86_init_processor=local_cpuinfo_x86_init_processor third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a +${objcopy} --redefine-sym cpuinfo_x86_detect_isa=local_cpuinfo_x86_detect_isa third_party/SVT-AV1/Bin/"${SVT_AV1_BUILD_TYPE}"/libSvtAv1Enc.a + +# Lyra +cd third_party/lyra || exit 1 + +if [ -d lyra ] ; then + cd lyra || exit 1 + git checkout main + git pull +else + git clone --filter=tree:0 https://github.com/google/lyra.git + cd lyra || exit 1 +fi +git checkout v"${LYRA_VERSION}" + +cd .. + +lyra_bazel_options=('-c') +if [ "${BUILD_TYPE}" = "Debug" ]; then + lyra_bazel_options+=('dbg') +elif [ "${BUILD_TYPE}" = "Release" ]; then + lyra_bazel_options+=('opt') +fi + +case "$PACKAGE" in + *_x86_64 ) + lyra_bazel_sysroot='' + ;; + *_arm64 ) + lyra_bazel_options+=('--config=jetson') + lyra_bazel_sysroot='/usr/aarch64-linux-gnu' +esac + +clang_raw_version=$(clang -v |& /usr/bin/grep version | rev | cut -d ' ' -f 1 | rev) +clang_version=$(echo "$clang_raw_version" | cut -d '-' -f 1) +llvm_version=$(echo "$clang_version" | cut -d '.' -f 1) + +BAZEL_SYSROOT=${lyra_bazel_sysroot} BAZEL_LLVM_DIR=/usr/lib/llvm-${llvm_version} CLANG_VERSION=${clang_version} USE_BAZEL_VERSION=5.4.1 bazelisk build "${lyra_bazel_options[@]}" :lyra || exit 1 +# chmod 755 bazel-bin/liblyra.a +# objcopy --redefine-sym cpuinfo_is_initialized=local_cpuinfo_is_initialized bazel-bin/liblyra.a +# objcopy --redefine-sym cpuinfo_initialize=local_cpuinfo_initialize bazel-bin/liblyra.a +# objcopy --redefine-sym cpuinfo_deinitialize=local_cpuinfo_deinitialize bazel-bin/liblyra.a + +cd ../.. + if [ "${BUILD_TYPE}" = "Native" ]; then mkdir -p "native/$PACKAGE" cd "native/$PACKAGE" || exit 1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 1be3a963..b265f78c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,10 +8,12 @@ RUN apt update \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/cache/apt/archives/* +ENV HISUI_LYRA_MODEL_COEFFS_PATH /usr/local/lib/lyra/model_coeffs ARG UBUNTU_VERSION ARG TARGETARCH COPY release/ubuntu-${UBUNTU_VERSION}_x86_64/hisui /tmp/hisui-amd64 COPY release/ubuntu-${UBUNTU_VERSION}_arm64/hisui /tmp/hisui-arm64 +COPY third_party/lyra/lyra/lyra/model_coeffs /usr/local/lib/lyra/model_coeffs RUN cp /tmp/hisui-$TARGETARCH /usr/local/bin/hisui ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/hisui"] diff --git a/src/audio/basic_sequencer.hpp b/src/audio/basic_sequencer.hpp index 9f5333e3..8da97e20 100644 --- a/src/audio/basic_sequencer.hpp +++ b/src/audio/basic_sequencer.hpp @@ -22,7 +22,7 @@ class BasicSequencer : public Sequencer { explicit BasicSequencer(const std::vector&); void getSamples(std::vector>*, - const std::uint64_t); + const std::uint64_t) override; private: std::vector< diff --git a/src/audio/buffer_fdk_aac_encoder.hpp b/src/audio/buffer_fdk_aac_encoder.hpp index 086de956..698dd15c 100644 --- a/src/audio/buffer_fdk_aac_encoder.hpp +++ b/src/audio/buffer_fdk_aac_encoder.hpp @@ -25,8 +25,8 @@ class BufferFDKAACEncoder : public Encoder { BufferFDKAACEncoder(std::queue*, const BufferFDKAACEncoderParameters&); ~BufferFDKAACEncoder(); - void addSample(const std::int16_t, const std::int16_t); - void flush(); + void addSample(const std::int16_t, const std::int16_t) override; + void flush() override; private: std::queue* m_buffer; diff --git a/src/audio/buffer_opus_encoder.hpp b/src/audio/buffer_opus_encoder.hpp index 78bc2be1..95e5e89d 100644 --- a/src/audio/buffer_opus_encoder.hpp +++ b/src/audio/buffer_opus_encoder.hpp @@ -28,8 +28,8 @@ class BufferOpusEncoder : public Encoder { explicit BufferOpusEncoder(std::queue*, const BufferOpusEncoderParameters&); ~BufferOpusEncoder(); - void addSample(const std::int16_t, const std::int16_t); - void flush(); + void addSample(const std::int16_t, const std::int16_t) override; + void flush() override; ::opus_int32 getSkip() const; diff --git a/src/audio/lyra_decoder.cpp b/src/audio/lyra_decoder.cpp new file mode 100644 index 00000000..6193bc54 --- /dev/null +++ b/src/audio/lyra_decoder.cpp @@ -0,0 +1,58 @@ +#include "audio/lyra_decoder.hpp" + +#include +#include + +#include + +#include "constants.hpp" + +namespace hisui::audio { + +LyraDecoder::LyraDecoder(const int t_channles, const std::string& model_path) + : m_channels(t_channles) { + if (m_channels != 1) { + throw std::invalid_argument( + fmt::format("invalid number of channels: {}", m_channels)); + } + m_decoder = lyra_decoder_create(hisui::Constants::PCM_SAMPLE_RATE, m_channels, + model_path.c_str()); + if (m_decoder == nullptr) { + throw std::runtime_error("could not create lyra decoder"); + } + + m_lyra_buffer = new std::int16_t[hisui::Constants::PCM_SAMPLE_RATE / + hisui::Constants::LYRA_FRAME_RATE]; +} + +LyraDecoder::~LyraDecoder() { + if (m_lyra_buffer) { + delete[] m_lyra_buffer; + } + if (m_decoder) { + lyra_decoder_destroy(m_decoder); + } +} + +std::pair LyraDecoder::decode( + const unsigned char* src_buffer, + const std::size_t src_buffer_length) { + auto r = + lyra_decoder_set_encoded_packet(m_decoder, src_buffer, src_buffer_length); + if (!r) { + throw std::runtime_error("lyra_decoder_set_encoded_packet() failed"); + } + auto v = lyra_decoder_decode_samples( + m_decoder, + hisui::Constants::PCM_SAMPLE_RATE / hisui::Constants::LYRA_FRAME_RATE); + if (v == nullptr) { + throw std::runtime_error("lyra_decoder_decode_samples() failed"); + } + auto samples = lyra_vector_s16_get_size(v); + auto p = lyra_vector_s16_get_data(v); + std::memcpy(m_lyra_buffer, p, samples * 2); + lyra_vector_s16_destroy(v); + return {m_lyra_buffer, samples}; +} + +} // namespace hisui::audio diff --git a/src/audio/lyra_decoder.hpp b/src/audio/lyra_decoder.hpp new file mode 100644 index 00000000..97c318dc --- /dev/null +++ b/src/audio/lyra_decoder.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include "audio/decoder.hpp" + +namespace hisui::audio { + +class LyraDecoder : public Decoder { + public: + explicit LyraDecoder(const int t_channles, const std::string& model_path); + ~LyraDecoder(); + + std::pair decode( + const unsigned char*, + const std::size_t) override; + + private: + lyra_decoder* m_decoder; + int m_channels; + std::int16_t* m_lyra_buffer = nullptr; +}; + +} // namespace hisui::audio diff --git a/src/audio/lyra_handler.cpp b/src/audio/lyra_handler.cpp new file mode 100644 index 00000000..c2942ed4 --- /dev/null +++ b/src/audio/lyra_handler.cpp @@ -0,0 +1,48 @@ +#include "audio/lyra_handler.hpp" + +#include + +#include +#include + +namespace hisui::audio { + +void LyraHandler::setModelPath(const std::string& model_path) { + if (!m_handler) { + m_handler = new LyraHandler(model_path); + } +} + +bool LyraHandler::hasInstance() { + return m_handler != nullptr; +} + +LyraHandler& LyraHandler::getInstance() { + return *m_handler; +} + +LyraHandler::LyraHandler(const std::string& model_path) { + if (!std::empty(m_model_path)) { + return; + } + + if (!std::filesystem::is_directory(model_path)) { + throw std::invalid_argument(fmt::format("{} is not directory", model_path)); + } + + m_model_path = model_path; +} + +void LyraHandler::close() { + delete m_handler; + m_handler = nullptr; +} + +std::string LyraHandler::getModelPath() const { + if (!hasInstance()) { + throw std::runtime_error("lyra model path is not set"); + } + return m_model_path; +} + +} // namespace hisui::audio diff --git a/src/audio/lyra_handler.hpp b/src/audio/lyra_handler.hpp new file mode 100644 index 00000000..58274c63 --- /dev/null +++ b/src/audio/lyra_handler.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include + +namespace hisui::audio { + +class LyraHandler { + public: + LyraHandler(const LyraHandler&) = delete; + LyraHandler& operator=(const LyraHandler&) = delete; + LyraHandler(LyraHandler&&) = delete; + LyraHandler& operator=(LyraHandler&&) = delete; + + std::string getModelPath() const; + + static void setModelPath(const std::string&); + static bool hasInstance(); + static LyraHandler& getInstance(); + static void close(); + + private: + std::string m_model_path = ""; + inline static LyraHandler* m_handler = nullptr; + + explicit LyraHandler(const std::string&); +}; + +} // namespace hisui::audio diff --git a/src/audio/opus_decoder.hpp b/src/audio/opus_decoder.hpp index 1a377564..3ce57c0d 100644 --- a/src/audio/opus_decoder.hpp +++ b/src/audio/opus_decoder.hpp @@ -15,8 +15,9 @@ class OpusDecoder : public Decoder { explicit OpusDecoder(const int t_channles); ~OpusDecoder(); - std::pair decode(const unsigned char*, - const std::size_t); + std::pair decode( + const unsigned char*, + const std::size_t) override; private: ::OpusDecoder* m_decoder = nullptr; diff --git a/src/audio/webm_source.cpp b/src/audio/webm_source.cpp index 179f88f0..0aea8943 100644 --- a/src/audio/webm_source.cpp +++ b/src/audio/webm_source.cpp @@ -8,6 +8,8 @@ #include #include "audio/decoder.hpp" +#include "audio/lyra_decoder.hpp" +#include "audio/lyra_handler.hpp" #include "audio/opus_decoder.hpp" #include "constants.hpp" #include "report/reporter.hpp" @@ -42,6 +44,18 @@ WebMSource::WebMSource(const std::string& t_file_path) { .duration = m_webm->getDuration()}); } break; + case hisui::webm::input::AudioCodec::Lyra: + m_channels = m_webm->getChannels(), + m_sampling_rate = static_cast(m_webm->getSamplingRate()); + m_decoder = std::make_shared( + m_channels, LyraHandler::getInstance().getModelPath()); + if (hisui::report::Reporter::hasInstance()) { + hisui::report::Reporter::getInstance().registerAudioDecoder( + m_webm->getFilePath(), {.codec = "lyra", + .channels = m_webm->getChannels(), + .duration = m_webm->getDuration()}); + } + break; default: // 対応していない WebM の場合は {0, 0} を返す m_webm = nullptr; @@ -86,7 +100,8 @@ std::pair WebMSource::getSample( void WebMSource::readFrame() { if (m_webm->readFrame()) { m_current_position = static_cast(m_webm->getTimestamp()) * - m_sampling_rate / hisui::Constants::NANO_SECOND; + hisui::Constants::PCM_SAMPLE_RATE / + hisui::Constants::NANO_SECOND; const auto decoded = m_decoder->decode(m_webm->getBuffer(), m_webm->getBufferSize()); if (decoded.second > 0) { diff --git a/src/audio/webm_source.hpp b/src/audio/webm_source.hpp index 454fb188..80e9d4ce 100644 --- a/src/audio/webm_source.hpp +++ b/src/audio/webm_source.hpp @@ -21,7 +21,7 @@ class Decoder; class WebMSource : public Source { public: explicit WebMSource(const std::string&); - std::pair getSample(const std::uint64_t); + std::pair getSample(const std::uint64_t) override; private: std::shared_ptr m_webm = nullptr; diff --git a/src/config.cpp b/src/config.cpp index a1d28f4e..13bde4ac 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -113,9 +113,10 @@ void set_cli_options(CLI::App* app, Config* config) { {"VP8", config::OutVideoCodec::VP8}, {"VP9", config::OutVideoCodec::VP9}, {"H264", config::OutVideoCodec::H264}, + {"AV1", config::OutVideoCodec::AV1}, }; app->add_option("--out-video-codec", config->out_video_codec, - "Video codec (VP8/VP9/H264). default: VP9") + "Video codec (VP8/VP9/H264/AV1). default: VP9") ->transform(CLI::CheckedTransformer(out_video_codec_assoc)); std::vector> out_audio_codec{ @@ -195,6 +196,21 @@ void set_cli_options(CLI::App* app, Config* config) { ->check(CLI::ExistingDirectory) ->group(EXPERIMENTAL_OPTIONS); + app->add_flag("--video-codec-engines", config->video_codec_engines, + "Show video codec engines and exit."); + + std::vector> h264_encoder_assoc{ +#ifdef USE_ONEVPL + {"OneVPL", config::H264Encoder::OneVPL}, +#endif + {"OpenH264", config::H264Encoder::OpenH264}, + }; + + app->add_option("--h264-encoder", config->h264_encoder, + "H264 encoder (OneVPL/OpenH264). default: OneVPL") + ->transform( + CLI::CheckedTransformer(h264_encoder_assoc, CLI::ignore_case)); + app->add_option("--show-progress-bar", config->show_progress_bar, "Toggle to show progress bar. default: true"); @@ -301,6 +317,9 @@ void set_cli_options(CLI::App* app, Config* config) { CLI::CheckedTransformer(openh264_level_assoc, CLI::ignore_case)) ->group(OPTIONS_FOR_TUNING); + app->add_option("--lyra-model-path", config->lyra_model_path, + "Path to directory containing Lyra TFLite files"); + std::vector> log_level_assoc{ {"trace", spdlog::level::trace}, diff --git a/src/config.hpp b/src/config.hpp index d4a63f8d..ba8596ae 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -32,6 +32,7 @@ enum OutVideoCodec { VP8 = hisui::Constants::VP8_FOURCC, VP9 = hisui::Constants::VP9_FOURCC, H264 = hisui::Constants::H264_FOURCC, + AV1 = hisui::Constants::AV1_FOURCC, }; enum struct VideoComposer { @@ -59,6 +60,14 @@ enum struct OutAudioCodec { FDK_AAC, }; +enum struct H264Encoder { + Unspecified, +#ifdef USE_ONEVPL + OneVPL, +#endif + OpenH264, +}; + } // namespace config class Config { @@ -97,8 +106,11 @@ class Config { std::string layout = ""; // 以降は SPEC.rst にないオプション + bool video_codec_engines = false; bool show_progress_bar = true; + config::H264Encoder h264_encoder = config::H264Encoder::Unspecified; + #ifdef NDEBUG spdlog::level::level_enum log_level = spdlog::level::info; #else @@ -123,6 +135,8 @@ class Config { ::EProfileIdc openh264_profile = ::PRO_BASELINE; ::ELevelIdc openh264_level = ::LEVEL_3_1; + std::string lyra_model_path = ""; + libyuv::FilterMode libyuv_filter_mode = libyuv::kFilterBox; config::VideoComposer video_composer = config::VideoComposer::Grid; diff --git a/src/constants.hpp b/src/constants.hpp index 31f04625..8d3700ef 100644 --- a/src/constants.hpp +++ b/src/constants.hpp @@ -8,8 +8,11 @@ namespace hisui { class Constants { public: static const std::uint32_t PCM_SAMPLE_RATE = 48000; + static const std::uint32_t LYRA_FRAME_RATE = + 50; // from chromemedia::codec::kFrameRate static const std::uint32_t VP8_FOURCC = 0x30385056; static const std::uint32_t VP9_FOURCC = 0x30395056; + static const std::uint32_t AV1_FOURCC = 0x31305641; static const std::uint32_t H264_FOURCC = 0x34363248; static const std::uint32_t I420_FOURCC = 0x30323449; static const std::uint32_t VIDEO_H264_OUTPUT_BITRATE = 512000; diff --git a/src/hisui.cpp b/src/hisui.cpp index 02bc46a4..423bfcb3 100644 --- a/src/hisui.cpp +++ b/src/hisui.cpp @@ -13,7 +13,9 @@ #include #include +#include "audio/lyra_handler.hpp" #include "config.hpp" +#include "constants.hpp" #include "datetime.hpp" #include "layout/compose.hpp" #include "metadata.hpp" @@ -23,12 +25,47 @@ #include "muxer/simple_mp4_muxer.hpp" #include "report/reporter.hpp" #include "version/version.hpp" +#include "video/codec_engine.hpp" +#include "video/decoder_factory.hpp" #include "video/openh264_handler.hpp" +#ifdef USE_ONEVPL +#include "video/vpl_decoder.hpp" +#include "video/vpl_encoder.hpp" +#include "video/vpl_session.hpp" +#endif + +static void closeHandlersAndSession() { + if (hisui::video::OpenH264Handler::hasInstance()) { + hisui::video::OpenH264Handler::close(); + } + + if (hisui::audio::LyraHandler::hasInstance()) { + hisui::audio::LyraHandler::close(); + } + +#ifdef USE_ONEVPL + if (hisui::video::VPLSession::hasInstance()) { + hisui::video::VPLSession::close(); + } +#endif +} + int main(int argc, char** argv) { CLI::App app{"hisui"}; hisui::Config config; + ::setenv("SVT_LOG", "-2", 1); + ::setenv("LIBVA_MESSAGING_LEVEL", "0", 1); + +#ifdef USE_ONEVPL + try { + hisui::video::VPLSession::open(); + } catch (const std::exception& e) { + spdlog::debug("failed to open VPL session: {}", e.what()); + } +#endif + try { hisui::set_cli_options(&app, &config); @@ -45,7 +82,15 @@ int main(int argc, char** argv) { } else { spdlog::set_level(config.log_level); } - spdlog::debug("log level={}", config.log_level); + spdlog::debug("log level={}", static_cast(config.log_level)); + + if (std::empty(config.lyra_model_path)) { + if (const auto hisui_lyra_model_coeffs_path = + std::getenv("HISUI_LYRA_MODEL_COEFFS_PATH")) { + config.lyra_model_path = hisui_lyra_model_coeffs_path; + } + spdlog::debug("config.lyra_model_path={}", config.lyra_model_path); + } if (!std::empty(config.openh264)) { try { @@ -55,6 +100,15 @@ int main(int argc, char** argv) { } } + if (!std::empty(config.lyra_model_path)) { + try { + hisui::audio::LyraHandler::setModelPath(config.lyra_model_path); + } catch (const std::exception& e) { + spdlog::warn("failed to set lyra model path: {}", e.what()); + return EXIT_FAILURE; + } + } + if (config.enabledReport()) { hisui::report::Reporter::open(); } @@ -64,16 +118,28 @@ int main(int argc, char** argv) { } if (!std::empty(config.layout)) { - return hisui::layout::compose(config); + hisui::video::DecoderFactory::setup(config); + auto ret = hisui::layout::compose(config); + + closeHandlersAndSession(); + + return ret; } config.validate(); + if (config.video_codec_engines) { + hisui::video::showCodecEngines(); + return EXIT_SUCCESS; + } + if (std::empty(config.in_metadata_filename)) { spdlog::error("-f,--in-metadata-file is required"); return EXIT_FAILURE; } + hisui::video::DecoderFactory::setup(config); + hisui::muxer::Muxer* muxer = nullptr; boost::json::string normal_recording_id; @@ -165,9 +231,7 @@ int main(int argc, char** argv) { } delete muxer; - if (!config.openh264.empty()) { - hisui::video::OpenH264Handler::close(); - } + closeHandlersAndSession(); if (config.enabledSuccessReport()) { try { @@ -185,3 +249,4 @@ int main(int argc, char** argv) { return EXIT_SUCCESS; } + diff --git a/src/layout/av1_video_producer.cpp b/src/layout/av1_video_producer.cpp new file mode 100644 index 00000000..a0469d2d --- /dev/null +++ b/src/layout/av1_video_producer.cpp @@ -0,0 +1,105 @@ +#include "layout/av1_video_producer.hpp" + +#include + +#include +#include + +#include +#include + +#include "config.hpp" +#include "layout/metadata.hpp" +#include "metadata.hpp" +#include "muxer/video_producer.hpp" +#include "video/basic_sequencer.hpp" +#include "video/buffer_av1_encoder.hpp" +#include "video/composer.hpp" +#include "video/grid_composer.hpp" +#include "video/parallel_grid_composer.hpp" +#include "video/sequencer.hpp" + +namespace hisui::layout { + +AV1VideoProducer::AV1VideoProducer(const hisui::Config& t_config, + const AV1VideoProducerParameters& params) + : VideoProducer({.show_progress_bar = t_config.show_progress_bar}), + m_resolution(params.resolution) { + m_frame_rate = t_config.out_video_frame_rate; + m_duration = params.duration; + + hisui::video::AV1EncoderConfig av1_config(m_resolution.width, + m_resolution.height, t_config); + + for (auto& r : params.regions) { + r->setEncodingInterval(); + } + + m_layout_composer = std::make_shared(ComposerParameters{ + .regions = params.regions, .resolution = m_resolution}); + + m_encoder = std::make_shared( + &m_buffer, av1_config, params.timescale); +} + +void AV1VideoProducer::produce() { + if (isFinished()) { + return; + } + + try { + std::vector raw_image; + + raw_image.resize(m_resolution.width * m_resolution.height * 3 >> 1); + + const std::uint64_t max_time = static_cast( + std::ceil(m_duration * hisui::Constants::NANO_SECOND)); + + progresscpp::ProgressBar progress_bar(max_time, 60); + + for (std::uint64_t t = 0, step = hisui::Constants::NANO_SECOND * + m_frame_rate.denominator() / + m_frame_rate.numerator(); + t < max_time; t += step) { + m_layout_composer->compose(&raw_image, t); + { + std::lock_guard lock(m_mutex_buffer); + m_encoder->outputImage(raw_image); + } + + if (m_show_progress_bar) { + progress_bar.setTicks(t); + progress_bar.display(); + } + } + + { + std::lock_guard lock(m_mutex_buffer); + m_encoder->flush(); + m_is_finished = true; + } + + if (m_show_progress_bar) { + progress_bar.setTicks(max_time); + progress_bar.done(); + } + } catch (const std::exception& e) { + spdlog::error("VideoProducer::produce() failed: what={}", e.what()); + m_is_finished = true; + throw; + } +} + +std::uint32_t AV1VideoProducer::getWidth() const { + return m_resolution.width; +} + +std::uint32_t AV1VideoProducer::getHeight() const { + return m_resolution.height; +} + +const std::vector& AV1VideoProducer::getExtraData() const { + return m_encoder->getExtraData(); +} + +} // namespace hisui::layout diff --git a/src/layout/av1_video_producer.hpp b/src/layout/av1_video_producer.hpp new file mode 100644 index 00000000..d504cf67 --- /dev/null +++ b/src/layout/av1_video_producer.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +#include "constants.hpp" +#include "layout/cell_util.hpp" +#include "layout/composer.hpp" +#include "layout/metadata.hpp" +#include "muxer/video_producer.hpp" + +namespace hisui { + +class Config; + +} // namespace hisui + +namespace hisui::layout { + +struct AV1VideoProducerParameters { + const std::vector>& regions; + const Resolution& resolution; + const double duration; + const std::uint64_t timescale = hisui::Constants::NANO_SECOND; +}; + +class AV1VideoProducer : public hisui::muxer::VideoProducer { + public: + AV1VideoProducer(const hisui::Config&, const AV1VideoProducerParameters&); + void produce() override; + std::uint32_t getWidth() const override; + std::uint32_t getHeight() const override; + const std::vector& getExtraData() const override; + + private: + Resolution m_resolution; + std::shared_ptr m_layout_composer; +}; + +} // namespace hisui::layout diff --git a/src/layout/compose.cpp b/src/layout/compose.cpp index 8832f751..f9886352 100644 --- a/src/layout/compose.cpp +++ b/src/layout/compose.cpp @@ -7,6 +7,7 @@ #include "config.hpp" #include "constants.hpp" #include "datetime.hpp" +#include "layout/av1_video_producer.hpp" #include "layout/metadata.hpp" #include "layout/openh264_video_producer.hpp" #include "layout/vpx_video_producer.hpp" @@ -16,6 +17,13 @@ #include "muxer/no_video_producer.hpp" #include "muxer/simple_mp4_muxer.hpp" #include "report/reporter.hpp" +#include "video/openh264_handler.hpp" + +#ifdef USE_ONEVPL +#include "layout/vpl_video_producer.hpp" +#include "video/vpl_encoder.hpp" +#include "video/vpl_session.hpp" +#endif namespace hisui::layout { @@ -33,8 +41,85 @@ int compose(const hisui::Config& t_config) { video_producer = std::make_shared(); } else { if (config.out_video_codec == hisui::config::OutVideoCodec::H264) { - video_producer = std::make_shared( - config, OpenH264VideoProducerParameters{ + if (config.h264_encoder == hisui::config::H264Encoder::OpenH264) { + if (!hisui::video::OpenH264Handler::hasInstance()) { + throw std::runtime_error("OpenH264 library is not loaded"); + } + video_producer = std::make_shared( + config, OpenH264VideoProducerParameters{ + .regions = metadata.getRegions(), + .resolution = metadata.getResolution(), + .duration = metadata.getMaxEndTime(), + .timescale = config.out_container == + hisui::config::OutContainer::WebM + ? hisui::Constants::NANO_SECOND + : 16000, // TODO(haruyama): 整理する + }); + } +#ifdef USE_ONEVPL + if (config.h264_encoder == hisui::config::H264Encoder::OneVPL) { + auto fourcc = hisui::Constants::H264_FOURCC; + if (!(hisui::video::VPLSession::hasInstance() && + hisui::video::VPLEncoder::isSupported(fourcc))) { + throw std::runtime_error("oneVPL H.264 encoder is not supported"); + } + video_producer = std::make_shared( + config, + VPLVideoProducerParameters{ + .regions = metadata.getRegions(), + .resolution = metadata.getResolution(), + .duration = metadata.getMaxEndTime(), + .timescale = + config.out_container == hisui::config::OutContainer::WebM + ? hisui::Constants::NANO_SECOND + : 16000, // TODO(haruyama): 整理する + }, + fourcc); + } +#endif + + // Unspecified + if (!video_producer) { +#ifdef USE_ONEVPL + if (hisui::video::VPLSession::hasInstance() && + hisui::video::VPLEncoder::isSupported( + hisui::Constants::H264_FOURCC)) { + auto fourcc = hisui::Constants::H264_FOURCC; + spdlog::debug("use VPLVideoProducer"); + video_producer = std::make_shared( + config, + VPLVideoProducerParameters{ + .regions = metadata.getRegions(), + .resolution = metadata.getResolution(), + .duration = metadata.getMaxEndTime(), + .timescale = config.out_container == + hisui::config::OutContainer::WebM + ? hisui::Constants::NANO_SECOND + : 16000, // TODO(haruyama): 整理する + }, + fourcc); + } else // NOLINT +#endif + if (hisui::video::OpenH264Handler::hasInstance()) { + spdlog::debug("use OpenH264VideoProducer"); + video_producer = std::make_shared( + config, + OpenH264VideoProducerParameters{ + .regions = metadata.getRegions(), + .resolution = metadata.getResolution(), + .duration = metadata.getMaxEndTime(), + .timescale = config.out_container == + hisui::config::OutContainer::WebM + ? hisui::Constants::NANO_SECOND + : 16000, // TODO(haruyama): 整理する + }); + } else { + throw std::runtime_error("H.264 dncoder is unavailable"); + } + } + } else if (config.out_video_codec == hisui::config::OutVideoCodec::AV1) { + video_producer = std::make_shared( + config, AV1VideoProducerParameters{ .regions = metadata.getRegions(), .resolution = metadata.getResolution(), .duration = metadata.getMaxEndTime(), diff --git a/src/layout/openh264_video_producer.hpp b/src/layout/openh264_video_producer.hpp index 45c5a915..369c0863 100644 --- a/src/layout/openh264_video_producer.hpp +++ b/src/layout/openh264_video_producer.hpp @@ -29,9 +29,9 @@ class OpenH264VideoProducer : public hisui::muxer::VideoProducer { public: OpenH264VideoProducer(const hisui::Config&, const OpenH264VideoProducerParameters&); - virtual void produce(); - virtual std::uint32_t getWidth() const; - virtual std::uint32_t getHeight() const; + void produce() override; + std::uint32_t getWidth() const override; + std::uint32_t getHeight() const override; private: Resolution m_resolution; diff --git a/src/layout/video_source.hpp b/src/layout/video_source.hpp index 5e0f9a18..ce2ac3d2 100644 --- a/src/layout/video_source.hpp +++ b/src/layout/video_source.hpp @@ -13,6 +13,7 @@ class YUVImage; } // namespace hisui::video namespace hisui::layout { + class VideoSource : public Source { public: explicit VideoSource(const SourceParameters&); diff --git a/src/layout/vpl_video_producer.cpp b/src/layout/vpl_video_producer.cpp new file mode 100644 index 00000000..7d522ae1 --- /dev/null +++ b/src/layout/vpl_video_producer.cpp @@ -0,0 +1,102 @@ +#include "layout/vpl_video_producer.hpp" + +#include + +#include +#include + +#include +#include + +#include "config.hpp" +#include "layout/metadata.hpp" +#include "metadata.hpp" +#include "muxer/video_producer.hpp" +#include "video/basic_sequencer.hpp" +#include "video/composer.hpp" +#include "video/grid_composer.hpp" +#include "video/parallel_grid_composer.hpp" +#include "video/sequencer.hpp" +#include "video/vpl_encoder.hpp" + +namespace hisui::layout { + +VPLVideoProducer::VPLVideoProducer(const hisui::Config& t_config, + const VPLVideoProducerParameters& params, + const std::uint32_t t_fourcc) + : VideoProducer({.show_progress_bar = t_config.show_progress_bar}), + m_resolution(params.resolution) { + m_frame_rate = t_config.out_video_frame_rate; + m_duration = params.duration; + + hisui::video::VPLEncoderConfig vpl_config(m_resolution.width, + m_resolution.height, t_config); + + for (auto& r : params.regions) { + r->setEncodingInterval(); + } + + m_layout_composer = std::make_shared(ComposerParameters{ + .regions = params.regions, .resolution = m_resolution}); + + m_encoder = std::make_shared( + t_fourcc, &m_buffer, vpl_config, params.timescale); +} + +void VPLVideoProducer::produce() { + if (isFinished()) { + return; + } + + try { + std::vector raw_image; + + raw_image.resize(m_resolution.width * m_resolution.height * 3 >> 1); + + const std::uint64_t max_time = static_cast( + std::ceil(m_duration * hisui::Constants::NANO_SECOND)); + + progresscpp::ProgressBar progress_bar(max_time, 60); + + for (std::uint64_t t = 0, step = hisui::Constants::NANO_SECOND * + m_frame_rate.denominator() / + m_frame_rate.numerator(); + t < max_time; t += step) { + m_layout_composer->compose(&raw_image, t); + { + std::lock_guard lock(m_mutex_buffer); + m_encoder->outputImage(raw_image); + } + + if (m_show_progress_bar) { + progress_bar.setTicks(t); + progress_bar.display(); + } + } + + { + std::lock_guard lock(m_mutex_buffer); + m_encoder->flush(); + m_is_finished = true; + } + + if (m_show_progress_bar) { + progress_bar.setTicks(max_time); + progress_bar.done(); + } + } catch (const std::exception& e) { + spdlog::error("VideoProducer::produce() failed: what={}", e.what()); + m_is_finished = true; + throw; + } +} + +std::uint32_t VPLVideoProducer::getWidth() const { + return m_resolution.width; +} + +std::uint32_t VPLVideoProducer::getHeight() const { + return m_resolution.height; +} + +} // namespace hisui::layout diff --git a/src/layout/vpl_video_producer.hpp b/src/layout/vpl_video_producer.hpp new file mode 100644 index 00000000..70c98494 --- /dev/null +++ b/src/layout/vpl_video_producer.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include "constants.hpp" +#include "layout/cell_util.hpp" +#include "layout/composer.hpp" +#include "layout/metadata.hpp" +#include "muxer/video_producer.hpp" + +namespace hisui { + +class Config; + +} // namespace hisui + +namespace hisui::layout { + +struct VPLVideoProducerParameters { + const std::vector>& regions; + const Resolution& resolution; + const double duration; + const std::uint64_t timescale = hisui::Constants::NANO_SECOND; +}; + +class VPLVideoProducer : public hisui::muxer::VideoProducer { + public: + VPLVideoProducer(const hisui::Config&, + const VPLVideoProducerParameters&, + const std::uint32_t); + void produce() override; + std::uint32_t getWidth() const override; + std::uint32_t getHeight() const override; + + private: + Resolution m_resolution; + std::shared_ptr m_layout_composer; +}; + +} // namespace hisui::layout diff --git a/src/layout/vpx_video_producer.hpp b/src/layout/vpx_video_producer.hpp index d01d3537..fd1ea4f6 100644 --- a/src/layout/vpx_video_producer.hpp +++ b/src/layout/vpx_video_producer.hpp @@ -28,9 +28,9 @@ struct VPXVideoProducerParameters { class VPXVideoProducer : public hisui::muxer::VideoProducer { public: VPXVideoProducer(const hisui::Config&, const VPXVideoProducerParameters&); - virtual void produce(); - virtual std::uint32_t getWidth() const; - virtual std::uint32_t getHeight() const; + void produce() override; + std::uint32_t getWidth() const override; + std::uint32_t getHeight() const override; private: Resolution m_resolution; diff --git a/src/muxer/async_webm_muxer.cpp b/src/muxer/async_webm_muxer.cpp index f6691e96..ef0e5bad 100644 --- a/src/muxer/async_webm_muxer.cpp +++ b/src/muxer/async_webm_muxer.cpp @@ -1,5 +1,7 @@ #include "muxer/async_webm_muxer.hpp" +#include + #include #include #include @@ -14,6 +16,7 @@ #include "constants.hpp" #include "frame.hpp" #include "muxer/audio_producer.hpp" +#include "muxer/av1_video_producer.hpp" #include "muxer/multi_channel_vpx_video_producer.hpp" #include "muxer/no_video_producer.hpp" #include "muxer/openh264_video_producer.hpp" @@ -21,8 +24,15 @@ #include "muxer/video_producer.hpp" #include "muxer/vpx_video_producer.hpp" #include "report/reporter.hpp" +#include "video/openh264_handler.hpp" #include "webm/output/context.hpp" +#ifdef USE_ONEVPL +#include "muxer/vpl_video_producer.hpp" +#include "video/vpl_encoder.hpp" +#include "video/vpl_session.hpp" +#endif + namespace hisui::muxer { AsyncWebMMuxer::AsyncWebMMuxer(const hisui::Config& t_config, @@ -73,25 +83,13 @@ void AsyncWebMMuxer::setUp() { .duration = m_duration, }); } else { - if (m_config.out_video_codec == hisui::config::OutVideoCodec::H264) { - m_video_producer = std::make_shared( - m_config, - OpenH264VideoProducerParameters{.archives = m_normal_archives, - .duration = m_duration}); - } else { - m_video_producer = std::make_shared( - m_config, - VPXVideoProducerParameters{.archives = m_normal_archives, - .duration = m_duration}); - } + m_video_producer = makeVideoProducer(); } } } if (!m_config.audio_only) { - m_context->setVideoTrack(m_video_producer->getWidth(), - m_video_producer->getHeight(), - m_video_producer->getFourcc()); + setVideoTrack(); } auto audio_producer = std::make_shared( @@ -138,4 +136,74 @@ void AsyncWebMMuxer::cleanUp() {} void AsyncWebMMuxer::muxFinalize() {} +std::shared_ptr AsyncWebMMuxer::makeVideoProducer() { + if (m_config.out_video_codec == hisui::config::OutVideoCodec::H264) { + if (m_config.h264_encoder == hisui::config::H264Encoder::OpenH264) { + if (!hisui::video::OpenH264Handler::hasInstance()) { + throw std::runtime_error("OpenH264 library is not loaded"); + } + return std::make_shared( + m_config, OpenH264VideoProducerParameters{ + .archives = m_normal_archives, .duration = m_duration}); + } +#ifdef USE_ONEVPL + if (m_config.h264_encoder == hisui::config::H264Encoder::OneVPL) { + if (!(hisui::video::VPLSession::hasInstance() && + hisui::video::VPLEncoder::isSupported( + hisui::Constants::H264_FOURCC))) { + throw std::runtime_error("oneVPL H.264 encoder is not supported"); + } + auto fourcc = hisui::Constants::H264_FOURCC; + return std::make_shared( + m_config, + VPLVideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration}, + fourcc); + } +#endif + + // Unspecified +#ifdef USE_ONEVPL + if (hisui::video::VPLSession::hasInstance() && + hisui::video::VPLEncoder::isSupported(hisui::Constants::H264_FOURCC)) { + spdlog::debug("use VPLVideoProducer"); + return std::make_shared( + m_config, + VPLVideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration}, + hisui::config::OutVideoCodec::H264); + } else // NOLINT +#endif + if (hisui::video::OpenH264Handler::hasInstance()) { + spdlog::debug("use OpenH264VideoProducer"); + return std::make_shared( + m_config, OpenH264VideoProducerParameters{ + .archives = m_normal_archives, .duration = m_duration}); + } + throw std::runtime_error("H.264 encoder is unavailable"); + } else if (m_config.out_video_codec == hisui::config::OutVideoCodec::AV1) { + return std::make_shared( + m_config, AV1VideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration}); + } else { + return std::make_shared( + m_config, VPXVideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration}); + } +} + +void AsyncWebMMuxer::setVideoTrack() { + if (m_config.out_video_codec == hisui::config::OutVideoCodec::AV1) { + const std::array private_data{0x81, 0x00, 0x06, 0x00}; + m_context->setVideoTrack(m_video_producer->getWidth(), + m_video_producer->getHeight(), + m_video_producer->getFourcc(), private_data.data(), + std::size(private_data)); + } else { + m_context->setVideoTrack(m_video_producer->getWidth(), + m_video_producer->getHeight(), + m_video_producer->getFourcc(), nullptr, 0); + } +} + } // namespace hisui::muxer diff --git a/src/muxer/async_webm_muxer.hpp b/src/muxer/async_webm_muxer.hpp index 075b08b4..19984ed6 100644 --- a/src/muxer/async_webm_muxer.hpp +++ b/src/muxer/async_webm_muxer.hpp @@ -54,6 +54,8 @@ class AsyncWebMMuxer : public Muxer { std::vector m_preferred_archives; double m_duration; std::size_t m_normal_archive_size; + std::shared_ptr makeVideoProducer(); + void setVideoTrack(); }; } // namespace hisui::muxer diff --git a/src/muxer/av1_video_producer.cpp b/src/muxer/av1_video_producer.cpp new file mode 100644 index 00000000..a2d4f07d --- /dev/null +++ b/src/muxer/av1_video_producer.cpp @@ -0,0 +1,61 @@ +#include "muxer/av1_video_producer.hpp" + +#include +#include + +#include + +#include "config.hpp" +#include "metadata.hpp" +#include "muxer/video_producer.hpp" +#include "video/basic_sequencer.hpp" +#include "video/buffer_av1_encoder.hpp" +#include "video/composer.hpp" +#include "video/grid_composer.hpp" +#include "video/parallel_grid_composer.hpp" +#include "video/sequencer.hpp" + +namespace hisui::muxer { + +AV1VideoProducer::AV1VideoProducer(const hisui::Config& t_config, + const AV1VideoProducerParameters& params) + : VideoProducer({.show_progress_bar = t_config.show_progress_bar}) { + m_sequencer = std::make_shared(params.archives); + + const auto scaling_width = t_config.scaling_width != 0 + ? t_config.scaling_width + : m_sequencer->getMaxWidth(); + const auto scaling_height = t_config.scaling_height != 0 + ? t_config.scaling_height + : m_sequencer->getMaxHeight(); + + switch (t_config.video_composer) { + case hisui::config::VideoComposer::Grid: + m_composer = std::make_shared( + scaling_width, scaling_height, m_sequencer->getSize(), + t_config.max_columns, t_config.video_scaler, + t_config.libyuv_filter_mode); + break; + case hisui::config::VideoComposer::ParallelGrid: + m_composer = std::make_shared( + scaling_width, scaling_height, m_sequencer->getSize(), + t_config.max_columns, t_config.video_scaler, + t_config.libyuv_filter_mode); + break; + } + + hisui::video::AV1EncoderConfig av1_config(m_composer->getWidth(), + m_composer->getHeight(), t_config); + + m_encoder = std::make_shared( + &m_buffer, av1_config, params.timescale); + + m_duration = params.duration; + m_frame_rate = t_config.out_video_frame_rate; +} + +const std::vector& AV1VideoProducer::getExtraData() const { + return m_encoder->getExtraData(); +} + +} // namespace hisui::muxer diff --git a/src/muxer/av1_video_producer.hpp b/src/muxer/av1_video_producer.hpp new file mode 100644 index 00000000..6ce12b48 --- /dev/null +++ b/src/muxer/av1_video_producer.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +#include "archive_item.hpp" +#include "constants.hpp" +#include "muxer/video_producer.hpp" + +namespace hisui { + +class Config; +class Metadata; + +} // namespace hisui + +namespace hisui::muxer { + +struct AV1VideoProducerParameters { + const std::vector& archives; + const double duration; + const std::uint64_t timescale = hisui::Constants::NANO_SECOND; +}; + +class AV1VideoProducer : public VideoProducer { + public: + AV1VideoProducer(const hisui::Config&, const AV1VideoProducerParameters&); + const std::vector& getExtraData() const override; +}; + +} // namespace hisui::muxer diff --git a/src/muxer/mp4_muxer.cpp b/src/muxer/mp4_muxer.cpp index 75a672b1..b24c5587 100644 --- a/src/muxer/mp4_muxer.cpp +++ b/src/muxer/mp4_muxer.cpp @@ -1,5 +1,7 @@ #include "muxer/mp4_muxer.hpp" +#include + #include #include #include @@ -14,6 +16,7 @@ #include "constants.hpp" #include "metadata.hpp" #include "muxer/audio_producer.hpp" +#include "muxer/av1_video_producer.hpp" #include "muxer/multi_channel_vpx_video_producer.hpp" #include "muxer/no_video_producer.hpp" #include "muxer/openh264_video_producer.hpp" @@ -21,18 +24,26 @@ #include "muxer/video_producer.hpp" #include "muxer/vpx_video_producer.hpp" #include "report/reporter.hpp" +#include "shiguredo/mp4/track/av1.hpp" #include "shiguredo/mp4/track/h264.hpp" #include "shiguredo/mp4/track/opus.hpp" #include "shiguredo/mp4/track/soun.hpp" #include "shiguredo/mp4/track/vide.hpp" #include "shiguredo/mp4/track/vpx.hpp" #include "shiguredo/mp4/writer/writer.hpp" +#include "video/openh264_handler.hpp" #ifdef USE_FDK_AAC #include "muxer/fdk_aac_audio_producer.hpp" #include "shiguredo/mp4/track/aac.hpp" #endif +#ifdef USE_ONEVPL +#include "muxer/vpl_video_producer.hpp" +#include "video/vpl_encoder.hpp" +#include "video/vpl_session.hpp" +#endif + namespace hisui::muxer { MP4Muxer::MP4Muxer(const MP4MuxerParameters& params) @@ -91,11 +102,67 @@ void MP4Muxer::initialize( }); } else { if (config.out_video_codec == config::OutVideoCodec::H264) { - m_video_producer = std::make_shared( - config, - OpenH264VideoProducerParameters{.archives = m_normal_archives, - .duration = m_duration, - .timescale = 16000}); + if (config.h264_encoder == hisui::config::H264Encoder::OpenH264) { + if (!hisui::video::OpenH264Handler::hasInstance()) { + throw std::runtime_error("OpenH264 library is not loaded"); + } + m_video_producer = std::make_shared( + config, + OpenH264VideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration, + .timescale = 16000}); + } +#ifdef USE_ONEVPL + if (config.h264_encoder == hisui::config::H264Encoder::OneVPL) { + if (!(hisui::video::VPLSession::hasInstance() && + hisui::video::VPLEncoder::isSupported( + hisui::Constants::H264_FOURCC))) { + throw std::runtime_error("oneVPL H.264 encoder is not supported"); + } + auto fourcc = hisui::Constants::H264_FOURCC; + m_video_producer = std::make_shared( + config, + VPLVideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration, + .timescale = 16000}, + fourcc); + } +#endif + + // Unspecified + + if (!m_video_producer) { +#ifdef USE_ONEVPL + if (hisui::video::VPLSession::hasInstance() && + hisui::video::VPLEncoder::isSupported( + hisui::Constants::H264_FOURCC)) { + auto fourcc = hisui::Constants::H264_FOURCC; + spdlog::debug("use VPLVideoProducer"); + m_video_producer = std::make_shared( + config, + VPLVideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration, + .timescale = 16000}, + fourcc); + } else // NOLINT +#endif + if (hisui::video::OpenH264Handler::hasInstance()) { + spdlog::debug("use OpenH264VideoProducer"); + m_video_producer = std::make_shared( + config, + OpenH264VideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration, + .timescale = 16000}); + } else { + throw std::runtime_error("H.264 encoder is unavailable"); + } + } + } else if (config.out_video_codec == + hisui::config::OutVideoCodec::AV1) { + m_video_producer = std::make_shared( + config, AV1VideoProducerParameters{.archives = m_normal_archives, + .duration = m_duration, + .timescale = 16000}); } else { m_video_producer = std::make_shared( config, VPXVideoProducerParameters{.archives = m_normal_archives, @@ -113,6 +180,16 @@ void MP4Muxer::initialize( .width = m_video_producer->getWidth(), .height = m_video_producer->getHeight(), .writer = m_writer.get()}); + } else if (config.out_video_codec == config::OutVideoCodec::AV1) { + m_vide_track = std::make_shared( + shiguredo::mp4::track::AV1TrackParameters{ + .timescale = 16000, + .duration = static_cast(m_duration), + .track_id = m_writer->getAndUpdateNextTrackID(), + .width = m_video_producer->getWidth(), + .height = m_video_producer->getHeight(), + .config_OBUs = m_video_producer->getExtraData(), + .writer = m_writer.get()}); } else { m_vide_track = std::make_shared( shiguredo::mp4::track::VPXTrackParameters{ diff --git a/src/muxer/muxer.cpp b/src/muxer/muxer.cpp index 626c4f0a..4e82e11c 100644 --- a/src/muxer/muxer.cpp +++ b/src/muxer/muxer.cpp @@ -105,6 +105,9 @@ std::string Muxer::getVideoCodecName(const hisui::Config& config) { if (fourcc == hisui::Constants::VP8_FOURCC) { return "vp8"; } + if (fourcc == hisui::Constants::AV1_FOURCC) { + return "av1"; + } if (fourcc == hisui::Constants::H264_FOURCC) { return "h264"; } diff --git a/src/muxer/muxer.hpp b/src/muxer/muxer.hpp index fc86fdf6..6c3eecb1 100644 --- a/src/muxer/muxer.hpp +++ b/src/muxer/muxer.hpp @@ -30,7 +30,7 @@ class Muxer { void mux(); std::string getVideoCodecName(const hisui::Config& config); - std::shared_ptr m_video_producer; + std::shared_ptr m_video_producer = nullptr; std::shared_ptr m_audio_producer; boost::rational m_timescale_ratio = 1; diff --git a/src/muxer/video_producer.cpp b/src/muxer/video_producer.cpp index 4a88c432..e56b553e 100644 --- a/src/muxer/video_producer.cpp +++ b/src/muxer/video_producer.cpp @@ -112,4 +112,8 @@ std::uint32_t VideoProducer::getFourcc() const { return m_encoder->getFourcc(); } +const std::vector& VideoProducer::getExtraData() const { + throw std::logic_error("VideoProducer::getExtraData() should not be called"); +} + } // namespace hisui::muxer diff --git a/src/muxer/video_producer.hpp b/src/muxer/video_producer.hpp index 73784b4e..756476b2 100644 --- a/src/muxer/video_producer.hpp +++ b/src/muxer/video_producer.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,8 @@ class VideoProducer { virtual std::uint32_t getHeight() const; std::uint32_t getFourcc() const; + virtual const std::vector& getExtraData() const; + protected: std::shared_ptr m_sequencer; std::shared_ptr m_encoder; diff --git a/src/muxer/vpl_video_producer.cpp b/src/muxer/vpl_video_producer.cpp new file mode 100644 index 00000000..f2414ae1 --- /dev/null +++ b/src/muxer/vpl_video_producer.cpp @@ -0,0 +1,58 @@ +#include "muxer/vpl_video_producer.hpp" + +#include +#include + +#include + +#include "config.hpp" +#include "metadata.hpp" +#include "muxer/video_producer.hpp" +#include "video/basic_sequencer.hpp" +#include "video/composer.hpp" +#include "video/grid_composer.hpp" +#include "video/parallel_grid_composer.hpp" +#include "video/sequencer.hpp" +#include "video/vpl_encoder.hpp" + +namespace hisui::muxer { + +VPLVideoProducer::VPLVideoProducer(const hisui::Config& t_config, + const VPLVideoProducerParameters& params, + const std::uint32_t t_fourcc) + : VideoProducer({.show_progress_bar = t_config.show_progress_bar}) { + m_sequencer = std::make_shared(params.archives); + + const auto scaling_width = t_config.scaling_width != 0 + ? t_config.scaling_width + : m_sequencer->getMaxWidth(); + const auto scaling_height = t_config.scaling_height != 0 + ? t_config.scaling_height + : m_sequencer->getMaxHeight(); + + switch (t_config.video_composer) { + case hisui::config::VideoComposer::Grid: + m_composer = std::make_shared( + scaling_width, scaling_height, m_sequencer->getSize(), + t_config.max_columns, t_config.video_scaler, + t_config.libyuv_filter_mode); + break; + case hisui::config::VideoComposer::ParallelGrid: + m_composer = std::make_shared( + scaling_width, scaling_height, m_sequencer->getSize(), + t_config.max_columns, t_config.video_scaler, + t_config.libyuv_filter_mode); + break; + } + + hisui::video::VPLEncoderConfig vpl_config(m_composer->getWidth(), + m_composer->getHeight(), t_config); + + m_encoder = std::make_shared( + t_fourcc, &m_buffer, vpl_config, params.timescale); + + m_duration = params.duration; + m_frame_rate = t_config.out_video_frame_rate; +} + +} // namespace hisui::muxer diff --git a/src/muxer/vpl_video_producer.hpp b/src/muxer/vpl_video_producer.hpp new file mode 100644 index 00000000..3acec4d1 --- /dev/null +++ b/src/muxer/vpl_video_producer.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +#include "archive_item.hpp" +#include "constants.hpp" +#include "muxer/video_producer.hpp" + +namespace hisui { + +class Config; +class Metadata; + +} // namespace hisui + +namespace hisui::muxer { + +struct VPLVideoProducerParameters { + const std::vector& archives; + const double duration; + const std::uint64_t timescale = hisui::Constants::NANO_SECOND; +}; + +class VPLVideoProducer : public VideoProducer { + public: + VPLVideoProducer(const hisui::Config&, + const VPLVideoProducerParameters&, + const std::uint32_t); +}; + +} // namespace hisui::muxer diff --git a/src/report/reporter.cpp b/src/report/reporter.cpp index 4e4b925f..11486292 100644 --- a/src/report/reporter.cpp +++ b/src/report/reporter.cpp @@ -93,6 +93,9 @@ void Reporter::collectVersions() { {"libvpx", version::get_libvpx_version()}, {"libwebm", version::get_libwebm_version()}, {"openh264", version::get_openh264_version()}, + {"SVT-AV1", version::get_svt_av1_version()}, + {"oneVPL", version::get_onevpl_version()}, + {"Lyra", version::get_lyra_version()}, #ifdef USE_FDK_AAC {"fdk-aac AACENC", version::get_fdkaac_aacenc_version()}, #endif diff --git a/src/version/version.cpp b/src/version/version.cpp index 8e398683..0605a07b 100644 --- a/src/version/version.cpp +++ b/src/version/version.cpp @@ -1,5 +1,6 @@ #include "version/version.hpp" +#include #include #include #include @@ -36,6 +37,23 @@ std::string get_openh264_version() { version.uRevision); } +std::string get_svt_av1_version() { + return fmt::format("{}.{}.{}", SVT_AV1_VERSION_MAJOR, SVT_AV1_VERSION_MINOR, + SVT_AV1_VERSION_PATCHLEVEL); +} + +std::string get_onevpl_version() { + // コードからは得られない + // VERSION と合わせること + return "2023.3.1"; +} + +std::string get_lyra_version() { + // コードから得るには third_party/lyra を変更する必要あり + // VERSION と合わせること + return "1.3.2"; +} + #ifdef USE_FDK_AAC std::string get_fdkaac_aacenc_version() { ::LIB_INFO info[FDK_MODULE_LAST]; diff --git a/src/version/version.hpp b/src/version/version.hpp index 64c017c6..6d3c47f8 100644 --- a/src/version/version.hpp +++ b/src/version/version.hpp @@ -7,6 +7,10 @@ namespace hisui::version { std::string get_libvpx_version(); std::string get_libwebm_version(); std::string get_openh264_version(); +std::string get_svt_av1_version(); +std::string get_onevpl_version(); +std::string get_lyra_version(); + #ifdef USE_FDK_AAC std::string get_fdkaac_aacenc_version(); #endif diff --git a/src/video/av1_decoder.cpp b/src/video/av1_decoder.cpp new file mode 100644 index 00000000..3cbb825a --- /dev/null +++ b/src/video/av1_decoder.cpp @@ -0,0 +1,269 @@ +#include "video/av1_decoder.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "constants.hpp" +#include "report/reporter.hpp" +#include "video/yuv.hpp" +#include "webm/input/video_context.hpp" + +namespace hisui::video { + +void set_config(::EbSvtAv1DecConfiguration* config, + std::uint32_t width, + std::uint32_t height) { + config->operating_point = -1; + config->output_all_layers = 0; + config->skip_film_grain = 0; + config->skip_frames = 0; + config->frames_to_be_decoded = 0; + config->compressed_ten_bit_format = 0; + config->eight_bit_output = 1; + + config->max_picture_width = width; + config->max_picture_height = height; + config->max_bit_depth = ::EB_EIGHT_BIT; + config->is_16bit_pipeline = 0; + config->max_color_format = ::EB_YUV420; + + config->channel_id = 0; + config->active_channel_count = 1; + config->stat_report = 0; + + config->threads = 1; + config->num_p_frames = 1; +} + +int set_picture_buffer(::EbSvtIOFormat* pic_buffer, + ::EbSvtAv1DecConfiguration* config, + std::uint32_t width, + std::uint32_t height) { + auto luma_size = width * height; + pic_buffer->luma = new std::uint8_t[luma_size]; + pic_buffer->cb = new std::uint8_t[luma_size >> 2]; + pic_buffer->cr = new std::uint8_t[luma_size >> 2]; + + pic_buffer->y_stride = width; + pic_buffer->cb_stride = (width + 1) >> 1; + pic_buffer->cr_stride = (width + 1) >> 1; + pic_buffer->width = width; + pic_buffer->height = height; + + pic_buffer->org_x = 0; + pic_buffer->org_y = 0; + pic_buffer->bit_depth = config->max_bit_depth; + return 0; +} + +void update_yuv_image_by_av1_buffer(std::shared_ptr yuv_image, + const ::EbSvtIOFormat* buffer) { + const auto bytes_per_sample = (buffer->bit_depth == ::EB_EIGHT_BIT) ? 1 : 2; + if (bytes_per_sample == 2) { + throw std::runtime_error("bytes_per_sample == 2 is not suppoted"); + } + if (buffer->color_fmt != ::EB_YUV420) { + throw std::runtime_error( + fmt::format("only EB_YUV420 format is suppoted: {}", + static_cast(buffer->color_fmt))); + } + auto* buf = buffer->luma; + auto s = buffer->y_stride; + auto w = buffer->width; + auto h = buffer->height; + + yuv_image->setWidthAndHeight(w, h); + + for (std::uint32_t y = 0; y < h; ++y) { + std::copy_n(buf + y * s, w, yuv_image->yuv[0] + y * w); + } + + buf = buffer->cb; + s = buffer->cb_stride; + w = (w + 1) >> 1; + h = (h + 1) >> 1; + for (std::uint32_t y = 0; y < h; ++y) { + std::copy_n(buf + y * s, w, yuv_image->yuv[1] + y * w); + } + + buf = buffer->cr; + s = buffer->cr_stride; + for (std::uint32_t y = 0; y < h; ++y) { + std::copy_n(buf + y * s, w, yuv_image->yuv[2] + y * w); + } +} + +AV1Decoder::AV1Decoder(std::shared_ptr t_webm) + : Decoder(t_webm) { + ::EbSvtAv1DecConfiguration config; + void* app_data = nullptr; + if (auto err = ::svt_av1_dec_init_handle(&m_handle, app_data, &config); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("::svt_av1_dec_init_handle() failed: {}", + static_cast(err))); + } + set_config(&config, m_width, m_height); + + if (auto err = ::svt_av1_dec_set_parameter(m_handle, &config); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("::svt_av1_dec_set_parameter() failed: {}", + static_cast(err))); + } + if (auto err = ::svt_av1_dec_init(m_handle); err != ::EB_ErrorNone) { + if (auto err_deinit = ::svt_av1_dec_deinit_handle(m_handle); + err_deinit != ::EB_ErrorNone) { + spdlog::error("::svt_av1_dec_deinit_handle() failed: {}", + static_cast(err_deinit)); + } + throw std::runtime_error(fmt::format("::svt_av1_dec_init() failed: {}", + static_cast(err))); + } + + m_recon_buffer = new ::EbBufferHeaderType(); + m_recon_buffer->p_buffer = + reinterpret_cast(::malloc(sizeof(::EbSvtIOFormat))); + ::EbSvtIOFormat* buffer = + reinterpret_cast<::EbSvtIOFormat*>(m_recon_buffer->p_buffer); + set_picture_buffer(buffer, &config, m_width, m_height); + + m_stream_info = new ::EbAV1StreamInfo(); + m_frame_info = new ::EbAV1FrameInfo(); + + m_current_yuv_image = std::make_shared(m_width, m_height); + + if (hisui::report::Reporter::hasInstance()) { + m_report_enabled = true; + + hisui::report::Reporter::getInstance().registerVideoDecoder( + m_webm->getFilePath(), + {.codec = "av1", .duration = m_webm->getDuration()}); + + hisui::report::Reporter::getInstance().registerResolutionChange( + m_webm->getFilePath(), + {.timestamp = 0, .width = m_width, .height = m_height}); + } + + updateAV1ImageByTimestamp(0); +} + +AV1Decoder::~AV1Decoder() { + // 作法的には実施すべきだが, segmentation fault してしまうので実施しない + // https://gitlab.com/AOMediaCodec/SVT-AV1/-/issues/2005#note_1181213012 + // if (m_handle) { + // if (auto err = ::svt_av1_dec_deinit(m_handle); err != ::EB_ErrorNone) { + // spdlog::error("::svt_av1_dec_deinit() failed: {}", + // static_cast(err)); + // } + // } + + if (m_stream_info) { + delete m_stream_info; + } + + if (m_frame_info) { + delete m_frame_info; + } + + if (m_recon_buffer) { + if (m_recon_buffer->p_buffer) { + ::EbSvtIOFormat* buffer = + reinterpret_cast<::EbSvtIOFormat*>(m_recon_buffer->p_buffer); + if (buffer->luma) { + delete[] buffer->luma; + } + if (buffer->cb) { + delete[] buffer->cb; + } + if (buffer->cr) { + delete[] buffer->cr; + } + ::free(m_recon_buffer->p_buffer); + } + delete m_recon_buffer; + } + + if (m_handle) { + if (auto err = ::svt_av1_dec_deinit_handle(m_handle); + err != ::EB_ErrorNone) { + spdlog::error("::svt_av1_dec_deinit_handle() failed: {}", + static_cast(err)); + } + } +} + +const std::shared_ptr AV1Decoder::getImage( + const std::uint64_t timestamp) { + // 非対応 WebM or 時間超過 + if (!m_webm || m_is_time_over) { + return m_black_yuv_image; + } + // 時間超過した + if (m_duration <= timestamp) { + m_is_time_over = true; + return m_black_yuv_image; + } + + updateAV1Image(timestamp); + return m_current_yuv_image; +} + +void AV1Decoder::updateAV1Image(const std::uint64_t timestamp) { + // 次のブロックに逹していない + if (timestamp < m_next_timestamp) { + return; + } + // 次以降のブロックに逹した + updateAV1ImageByTimestamp(timestamp); +} + +void AV1Decoder::updateAV1ImageByTimestamp(const std::uint64_t timestamp) { + if (m_finished_webm) { + return; + } + + do { + m_current_timestamp = m_next_timestamp; + if (m_webm->readFrame()) { + spdlog::trace("webm->getBufferSize(): {}", m_webm->getBufferSize()); + if (auto err = ::svt_av1_dec_frame(m_handle, m_webm->getBuffer(), + m_webm->getBufferSize(), 0); + err != ::EB_ErrorNone) { + throw std::runtime_error(fmt::format("::svt_av1_dec_frame() failed: {}", + static_cast(err))); + } + if (::svt_av1_dec_get_picture(m_handle, m_recon_buffer, m_stream_info, + m_frame_info) != ::EB_DecNoOutputPicture) { + ::EbSvtIOFormat* buffer = + reinterpret_cast<::EbSvtIOFormat*>(m_recon_buffer->p_buffer); + + if (m_report_enabled) { + if (m_current_yuv_image->getWidth(0) != buffer->width || + m_current_yuv_image->getHeight(0) != buffer->height) { + hisui::report::Reporter::getInstance().registerResolutionChange( + m_webm->getFilePath(), {.timestamp = m_next_timestamp, + .width = buffer->width, + .height = buffer->height}); + } + } + + update_yuv_image_by_av1_buffer(m_current_yuv_image, buffer); + } + m_next_timestamp = static_cast(m_webm->getTimestamp()); + } else { + m_finished_webm = true; + m_next_timestamp = std::numeric_limits::max(); + return; + } + } while (timestamp >= m_next_timestamp); +} + +} // namespace hisui::video diff --git a/src/video/av1_decoder.hpp b/src/video/av1_decoder.hpp new file mode 100644 index 00000000..7d27e5c2 --- /dev/null +++ b/src/video/av1_decoder.hpp @@ -0,0 +1,45 @@ +#pragma once + +// EbSvtAv1Dec.h の前に必要 +#include + +#include +#include +#include + +#include "video/decoder.hpp" + +namespace hisui::webm::input { + +class VideoContext; + +} + +namespace hisui::video { + +class YUVImage; + +class AV1Decoder : public Decoder { + public: + explicit AV1Decoder(std::shared_ptr); + + ~AV1Decoder(); + + const std::shared_ptr getImage(const std::uint64_t) override; + + private: + std::uint64_t m_current_timestamp = 0; + std::uint64_t m_next_timestamp = 0; + std::shared_ptr m_current_yuv_image = nullptr; + bool m_report_enabled = false; + ::EbComponentType* m_handle; + ::EbBufferHeaderType* m_recon_buffer; + ::EbAV1StreamInfo* m_stream_info; + ::EbAV1FrameInfo* m_frame_info; + + void updateAV1Image(const std::uint64_t); + + void updateAV1ImageByTimestamp(const std::uint64_t); +}; + +} // namespace hisui::video diff --git a/src/video/basic_sequencer.hpp b/src/video/basic_sequencer.hpp index 583fb499..5500ede7 100644 --- a/src/video/basic_sequencer.hpp +++ b/src/video/basic_sequencer.hpp @@ -21,7 +21,7 @@ class BasicSequencer : public Sequencer { explicit BasicSequencer(const std::vector&); SequencerGetYUVsResult getYUVs(std::vector>*, - const std::uint64_t); + const std::uint64_t) override; private: std::shared_ptr m_black_yuv_image; diff --git a/src/video/buffer_av1_encoder.cpp b/src/video/buffer_av1_encoder.cpp new file mode 100644 index 00000000..47f10556 --- /dev/null +++ b/src/video/buffer_av1_encoder.cpp @@ -0,0 +1,277 @@ +#include "video/buffer_av1_encoder.hpp" + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include "config.hpp" +#include "frame.hpp" + +namespace hisui::video { + +AV1EncoderConfig::AV1EncoderConfig(const std::uint32_t t_width, + const std::uint32_t t_height, + const hisui::Config& config) + : width(t_width), + height(t_height), + fps(config.out_video_frame_rate), + fourcc(config.out_video_codec), + bitrate(config.out_video_bit_rate) {} + +BufferAV1Encoder::BufferAV1Encoder(std::queue* t_buffer, + const AV1EncoderConfig& config, + const std::uint64_t t_timescale) + : m_buffer(t_buffer), m_timescale(t_timescale) { + m_width = config.width; + m_height = config.height; + m_fps = config.fps; + m_fourcc = config.fourcc; + m_bitrate = config.bitrate; + + void* app_data = nullptr; + if (auto err = + ::svt_av1_enc_init_handle(&m_handle, app_data, &m_av1_enc_config); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("::svt_av1_enc_init_handle() failed: {}", + static_cast(err))); + } + m_av1_enc_config.rate_control_mode = ::SVT_AV1_RC_MODE_CBR; + m_av1_enc_config.target_bit_rate = m_bitrate * 1000; + m_av1_enc_config.force_key_frames = false; + m_av1_enc_config.source_width = m_width; + m_av1_enc_config.source_height = m_height; + m_av1_enc_config.frame_rate_numerator = + static_cast(m_fps.numerator()); + m_av1_enc_config.frame_rate_denominator = + static_cast(m_fps.denominator()); + // core dump する場合を予防する + m_av1_enc_config.frame_scale_evts.start_frame_nums = nullptr; + m_av1_enc_config.frame_scale_evts.resize_kf_denoms = nullptr; + m_av1_enc_config.frame_scale_evts.resize_denoms = nullptr; + + if (auto err = ::svt_av1_enc_set_parameter(m_handle, &m_av1_enc_config); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("::svt_av1_enc_set_parameter() failed: {}", + static_cast(err))); + } + + if (auto err = ::svt_av1_enc_init(m_handle); err != ::EB_ErrorNone) { + throw std::runtime_error(fmt::format("svt_av1_enc_init() failed: {}", + static_cast(err))); + } + + m_input_buffer = new ::EbBufferHeaderType(); + m_input_buffer->p_buffer = + reinterpret_cast(::malloc(sizeof(::EbSvtIOFormat))); + m_input_buffer->size = sizeof(::EbBufferHeaderType); + m_input_buffer->p_app_private = nullptr; + m_input_buffer->pic_type = ::EB_AV1_INVALID_PICTURE; + m_input_buffer->metadata = nullptr; + + ::EbSvtIOFormat* buffer = + reinterpret_cast<::EbSvtIOFormat*>(m_input_buffer->p_buffer); + auto luma_size = m_width * m_height; + buffer->luma = new std::uint8_t[luma_size]; + buffer->cb = new std::uint8_t[luma_size >> 2]; + buffer->cr = new std::uint8_t[luma_size >> 2]; + + ::EbBufferHeaderType* stream_header = nullptr; + if (auto err = ::svt_av1_enc_stream_header(m_handle, &stream_header); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("svt_av1_enc_stream_header() failed: {}", + static_cast(err))); + } + + std::copy_n(stream_header->p_buffer, stream_header->n_filled_len, + std::back_inserter(m_extra_data)); + + ::svt_av1_enc_stream_header_release(stream_header); + + spdlog::debug("AV1 extra_data: [{:02x}]", fmt::join(m_extra_data, ", ")); +} + +void BufferAV1Encoder::outputImage(const std::vector& yuv) { + ::EbSvtIOFormat* buffer = + reinterpret_cast<::EbSvtIOFormat*>(m_input_buffer->p_buffer); + const auto luma_size = m_width * m_height; + std::copy_n(std::begin(yuv), luma_size, buffer->luma); + std::copy_n(std::begin(yuv) + luma_size, luma_size >> 2, buffer->cb); + std::copy_n(std::begin(yuv) + luma_size + (luma_size >> 2), luma_size >> 2, + buffer->cr); + m_input_buffer->flags = 0; + m_input_buffer->p_app_private = nullptr; + m_input_buffer->pts = m_frame; + m_input_buffer->pic_type = ::EB_AV1_INVALID_PICTURE; + m_input_buffer->metadata = nullptr; + buffer->y_stride = m_width; + buffer->cb_stride = m_width >> 1; + buffer->cr_stride = m_width >> 1; + buffer->width = m_width; + buffer->height = m_height; + + if (auto err = ::svt_av1_enc_send_picture(m_handle, m_input_buffer); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("::svt_av1_enc_send_picture() failed: {}", + static_cast(err))); + } + + outputFrame(m_frame++, 0); +} + +void BufferAV1Encoder::outputFrame(const std::int64_t frame_index, + const std::uint8_t done_sending_pics) { + ::EbBufferHeaderType* output_buf = nullptr; + + while (true) { + auto status = + ::svt_av1_enc_get_packet(m_handle, &output_buf, done_sending_pics); + if (status == ::EB_ErrorMax) { + throw std::runtime_error( + fmt::format("::svt_av1_enc_send_picture() failed: {}", + static_cast(status))); + } else if (status == ::EB_NoErrorEmptyQueue) { + return; + } + const std::uint64_t timestamp = + static_cast(output_buf->pts) * m_timescale * + m_fps.denominator() / m_fps.numerator(); + std::uint8_t* data = new std::uint8_t[output_buf->n_filled_len]; + std::copy_n(output_buf->p_buffer, output_buf->n_filled_len, data); + m_buffer->push(hisui::Frame{ + .timestamp = timestamp, + .data = data, + .data_size = output_buf->n_filled_len, + .is_key = output_buf->pic_type == ::EB_AV1_KEY_PICTURE || + output_buf->pic_type == ::EB_AV1_INTRA_ONLY_PICTURE}); + + m_sum_of_bits += output_buf->n_filled_len * 8; + + ::svt_av1_enc_release_out_buffer(&output_buf); + + if (m_frame > 0 && m_frame % 100 == 0 && frame_index > 0) { + spdlog::trace("AV1: frame index: {}", m_frame); + spdlog::trace("AV1: average bitrate (kbps): {}", + m_sum_of_bits * m_fps.numerator() / m_fps.denominator() / + static_cast(m_frame) / 1024); + } + } +} + +void BufferAV1Encoder::flush() { + ::EbBufferHeaderType input_buffer; + input_buffer.n_alloc_len = 0; + input_buffer.n_filled_len = 0; + input_buffer.n_tick_count = 0; + input_buffer.p_app_private = nullptr; + input_buffer.flags = EB_BUFFERFLAG_EOS; + input_buffer.p_buffer = nullptr; + input_buffer.metadata = nullptr; + + if (auto err = ::svt_av1_enc_send_picture(m_handle, &input_buffer); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("::svt_av1_enc_send_picture() failed: {}", + static_cast(err))); + } + + outputFrame(m_frame, 1); +} + +BufferAV1Encoder::~BufferAV1Encoder() { + if (m_frame > 0) { + spdlog::debug("AV1Encoder: number of frames: {}", m_frame); + spdlog::debug("AV1Encoder: final average bitrate (kbps): {}", + m_sum_of_bits * m_fps.numerator() / m_fps.denominator() / + static_cast(m_frame) / 1024); + } + if (m_input_buffer) { + if (m_input_buffer->p_buffer) { + ::EbSvtIOFormat* buffer = + reinterpret_cast<::EbSvtIOFormat*>(m_input_buffer->p_buffer); + if (buffer->luma) { + delete[] buffer->luma; + } + if (buffer->cb) { + delete[] buffer->cb; + } + if (buffer->cr) { + delete[] buffer->cr; + } + ::free(m_input_buffer->p_buffer); + } + delete m_input_buffer; + } + + if (auto err = ::svt_av1_enc_deinit(m_handle); err != ::EB_ErrorNone) { + spdlog::error("::svt_av1_enc_deinit() failed: {}", + static_cast(err)); + } + if (auto err = ::svt_av1_enc_deinit_handle(m_handle); err != ::EB_ErrorNone) { + spdlog::error("::svt_av1_enc_deinit_handle() failed: {}", + static_cast(err)); + } +} + +std::uint32_t BufferAV1Encoder::getFourcc() const { + return m_fourcc; +} + +void BufferAV1Encoder::setResolutionAndBitrate(const std::uint32_t width, + const std::uint32_t height, + const std::uint32_t bitrate) { + if (m_width == width && m_height == height && m_bitrate == bitrate) { + return; + } + spdlog::debug("width: {}, height: {}", width, height); + flush(); + m_width = width; + m_height = height; + m_bitrate = bitrate; + + m_av1_enc_config.target_bit_rate = m_bitrate; + m_av1_enc_config.source_width = m_width; + m_av1_enc_config.source_height = m_height; + + if (auto err = ::svt_av1_enc_set_parameter(m_handle, &m_av1_enc_config); + err != ::EB_ErrorNone) { + throw std::runtime_error( + fmt::format("::svt_av1_enc_set_parameter() failed: {}", + static_cast(err))); + } + + ::EbSvtIOFormat* buffer = + reinterpret_cast<::EbSvtIOFormat*>(m_input_buffer->p_buffer); + if (buffer->luma) { + delete[] buffer->luma; + } + if (buffer->cb) { + delete[] buffer->cb; + } + if (buffer->cr) { + delete[] buffer->cr; + } + auto luma_size = sizeof(std::uint8_t) * m_width * m_height; + buffer->luma = new std::uint8_t[luma_size]; + buffer->cb = new std::uint8_t[luma_size >> 2]; + buffer->cr = new std::uint8_t[luma_size >> 2]; +} + +const std::vector& BufferAV1Encoder::getExtraData() const { + return m_extra_data; +} + +} // namespace hisui::video diff --git a/src/video/buffer_av1_encoder.hpp b/src/video/buffer_av1_encoder.hpp new file mode 100644 index 00000000..91a73851 --- /dev/null +++ b/src/video/buffer_av1_encoder.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "constants.hpp" +#include "video/encoder.hpp" + +namespace hisui { + +class Config; +struct Frame; + +} // namespace hisui + +namespace hisui::video { + +class AV1EncoderConfig { + public: + AV1EncoderConfig(const std::uint32_t, + const std::uint32_t, + const hisui::Config&); + const std::uint32_t width; + const std::uint32_t height; + const boost::rational fps; + const std::uint32_t fourcc; + const std::uint32_t bitrate; +}; + +class BufferAV1Encoder : public Encoder { + public: + BufferAV1Encoder( + std::queue*, + const AV1EncoderConfig& config, + const std::uint64_t timescale = hisui::Constants::NANO_SECOND); + ~BufferAV1Encoder(); + + void outputImage(const std::vector&) override; + void flush() override; + std::uint32_t getFourcc() const override; + void setResolutionAndBitrate(const std::uint32_t, + const std::uint32_t, + const std::uint32_t) override; + const std::vector& getExtraData() const override; + + private: + std::queue* m_buffer; + std::uint32_t m_width; + std::uint32_t m_height; + std::uint32_t m_bitrate; + boost::rational m_fps; + std::uint32_t m_fourcc; + std::int64_t m_frame = 0; + std::uint64_t m_sum_of_bits = 0; + const std::uint64_t m_timescale; + ::EbComponentType* m_handle; + ::EbBufferHeaderType* m_input_buffer; + ::EbSvtAv1EncConfiguration m_av1_enc_config; + std::vector m_extra_data = {}; + + void outputFrame(const std::int64_t, const std::uint8_t); +}; + +} // namespace hisui::video diff --git a/src/video/buffer_openh264_encoder.hpp b/src/video/buffer_openh264_encoder.hpp index 4b92aea8..0245de8a 100644 --- a/src/video/buffer_openh264_encoder.hpp +++ b/src/video/buffer_openh264_encoder.hpp @@ -30,12 +30,12 @@ class BufferOpenH264Encoder : public Encoder { const std::uint64_t timescale = hisui::Constants::NANO_SECOND); ~BufferOpenH264Encoder(); - void outputImage(const std::vector&); - void flush(); - std::uint32_t getFourcc() const; + void outputImage(const std::vector&) override; + void flush() override; + std::uint32_t getFourcc() const override; void setResolutionAndBitrate(const std::uint32_t, const std::uint32_t, - const std::uint32_t); + const std::uint32_t) override; private: ::ISVCEncoder* m_encoder = nullptr; diff --git a/src/video/buffer_vpx_encoder.hpp b/src/video/buffer_vpx_encoder.hpp index 460681dc..2f9288c1 100644 --- a/src/video/buffer_vpx_encoder.hpp +++ b/src/video/buffer_vpx_encoder.hpp @@ -32,12 +32,12 @@ class BufferVPXEncoder : public Encoder { const std::uint64_t timescale = hisui::Constants::NANO_SECOND); ~BufferVPXEncoder(); - void outputImage(const std::vector&); - void flush(); - std::uint32_t getFourcc() const; + void outputImage(const std::vector&) override; + void flush() override; + std::uint32_t getFourcc() const override; void setResolutionAndBitrate(const std::uint32_t, const std::uint32_t, - const std::uint32_t); + const std::uint32_t) override; private: std::queue* m_buffer; diff --git a/src/video/codec_engine.cpp b/src/video/codec_engine.cpp new file mode 100644 index 00000000..80d2fb69 --- /dev/null +++ b/src/video/codec_engine.cpp @@ -0,0 +1,89 @@ +#include "video/codec_engine.hpp" + +#include + +#include +#include + +#include "constants.hpp" +#include "video/openh264_handler.hpp" + +#ifdef USE_ONEVPL +#include "video/vpl_encoder.hpp" +#include "video/vpl_session.hpp" +#endif + +namespace hisui::video { + +void printEngine(const std::string& name, + const std::string& type, + const bool is_default) { + std::cout << fmt::format(" - {} [{}]", name, type); + if (is_default) { + std::cout << " (default)"; + } + std::cout << std::endl; +} + +void showCodecEngines() { + std::cout << "VP8:" << std::endl; + std::cout << " Encoder:" << std::endl; + { + bool is_default = true; + printEngine("libvpx", "software", is_default); + } + std::cout << " Decoder:" << std::endl; + { + bool is_default = true; + printEngine("libvpx", "software", is_default); + } + + std::cout << "VP9:" << std::endl; + std::cout << " Encoder:" << std::endl; + { + bool is_default = true; + printEngine("libvpx", "software", is_default); + } + std::cout << " Decoder:" << std::endl; + { + bool is_default = true; + printEngine("libvpx", "software", is_default); + } + + std::cout << "AV1:" << std::endl; + std::cout << " Encoder:" << std::endl; + { + bool is_default = true; + printEngine("SVT-AV1", "software", is_default); + } + std::cout << " Decoder:" << std::endl; + { + bool is_default = true; + printEngine("SVT-AV1", "software", is_default); + } + + std::cout << "H264:" << std::endl; + std::cout << " Encoder:" << std::endl; + { + bool is_default = true; +#ifdef USE_ONEVPL + if (VPLSession::hasInstance() && + VPLEncoder::isSupported(hisui::Constants::H264_FOURCC)) { + printEngine("Intel oneVPL", "intel", is_default); + is_default = false; + } +#endif + if (OpenH264Handler::hasInstance()) { + printEngine("OpenH264", "software", is_default); + } + } + std::cout << " Decoder:" << std::endl; + { + bool is_default = true; + if (OpenH264Handler::hasInstance()) { + printEngine("OpenH264", "software", is_default); + } + } +} + +} // namespace hisui::video diff --git a/src/video/codec_engine.hpp b/src/video/codec_engine.hpp new file mode 100644 index 00000000..089df77c --- /dev/null +++ b/src/video/codec_engine.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace hisui::video { + +void showCodecEngines(); + +} diff --git a/src/video/decoder_factory.cpp b/src/video/decoder_factory.cpp new file mode 100644 index 00000000..6b5fdbd2 --- /dev/null +++ b/src/video/decoder_factory.cpp @@ -0,0 +1,47 @@ +#include "video/decoder_factory.hpp" + +#include +#include + +#include "config.hpp" +#include "constants.hpp" +#include "video/av1_decoder.hpp" +#include "video/openh264_decoder.hpp" +#include "video/openh264_handler.hpp" +#include "video/vpx_decoder.hpp" +#include "webm/input/video_context.hpp" + +#ifdef USE_ONEVPL +#include "video/vpl_session.hpp" +#endif + +namespace hisui::video { + +void DecoderFactory::setup(const hisui::Config& config) { + auto factory = new DecoderFactory(config); + m_instance = std::unique_ptr(factory); +} + +DecoderFactory::DecoderFactory(const hisui::Config& t_config) + : m_config(t_config) {} + +std::shared_ptr DecoderFactory::create( + std::shared_ptr webm) { + auto fourcc = webm->getFourcc(); + switch (fourcc) { + case hisui::Constants::VP8_FOURCC: /* fall through */ + case hisui::Constants::VP9_FOURCC: + return std::make_shared(webm); + case hisui::Constants::AV1_FOURCC: + return std::make_shared(webm); + case hisui::Constants::H264_FOURCC: + if (OpenH264Handler::hasInstance()) { + return std::make_shared(webm); + } + throw std::runtime_error("H.264 decoder is unavailable"); + default: + throw std::runtime_error(fmt::format("unknown fourcc: {}", fourcc)); + } +} + +} // namespace hisui::video diff --git a/src/video/decoder_factory.hpp b/src/video/decoder_factory.hpp new file mode 100644 index 00000000..f61cd52d --- /dev/null +++ b/src/video/decoder_factory.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +#include "config.hpp" +#include "video/decoder.hpp" + +namespace hisui::webm::input { + +class VideoContext; + +} + +namespace hisui::video { + +class DecoderFactory { + public: + DecoderFactory(const DecoderFactory&) = delete; + DecoderFactory& operator=(const DecoderFactory&) = delete; + DecoderFactory(DecoderFactory&&) = delete; + DecoderFactory& operator=(DecoderFactory&&) = delete; + + static std::shared_ptr create( + std::shared_ptr); + static void setup(const hisui::Config&); + + private: + explicit DecoderFactory(const hisui::Config&); + + inline static std::unique_ptr m_instance = nullptr; + hisui::Config m_config; +}; + +} // namespace hisui::video diff --git a/src/video/encoder.hpp b/src/video/encoder.hpp index 6d1b0cad..5496f296 100644 --- a/src/video/encoder.hpp +++ b/src/video/encoder.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace hisui::video { @@ -15,6 +16,9 @@ class Encoder { const std::uint32_t) {} virtual std::uint32_t getFourcc() const = 0; + virtual const std::vector& getExtraData() const { + throw std::logic_error("Encoder::getExtraData() should not be called"); + } }; } // namespace hisui::video diff --git a/src/video/grid_composer.hpp b/src/video/grid_composer.hpp index e26dae14..1ab64403 100644 --- a/src/video/grid_composer.hpp +++ b/src/video/grid_composer.hpp @@ -28,7 +28,7 @@ class GridComposer : public Composer { ~GridComposer(); void compose(std::vector*, - const std::vector>&); + const std::vector>&) override; private: std::uint32_t m_single_width; diff --git a/src/video/image_source.hpp b/src/video/image_source.hpp index 6584122e..6ad22b34 100644 --- a/src/video/image_source.hpp +++ b/src/video/image_source.hpp @@ -13,9 +13,9 @@ class YUVImage; class ImageSource : public Source { public: explicit ImageSource(const std::string&); - const std::shared_ptr getYUV(const std::uint64_t); - std::uint32_t getWidth() const; - std::uint32_t getHeight() const; + const std::shared_ptr getYUV(const std::uint64_t) override; + std::uint32_t getWidth() const override; + std::uint32_t getHeight() const override; private: std::uint32_t m_width; diff --git a/src/video/multi_channel_sequencer.hpp b/src/video/multi_channel_sequencer.hpp index f837dd1c..5bc31984 100644 --- a/src/video/multi_channel_sequencer.hpp +++ b/src/video/multi_channel_sequencer.hpp @@ -24,7 +24,7 @@ class MultiChannelSequencer : public Sequencer { const std::vector&); SequencerGetYUVsResult getYUVs(std::vector>*, - const std::uint64_t); + const std::uint64_t) override; private: std::shared_ptr m_black_yuv_image; diff --git a/src/video/openh264_decoder.cpp b/src/video/openh264_decoder.cpp index 65f8439c..720a384d 100644 --- a/src/video/openh264_decoder.cpp +++ b/src/video/openh264_decoder.cpp @@ -70,6 +70,7 @@ OpenH264Decoder::~OpenH264Decoder() { m_decoder->Uninitialize(); OpenH264Handler::getInstance().destroyDecoder(m_decoder); } + m_decoder = nullptr; } const std::shared_ptr OpenH264Decoder::getImage( @@ -120,9 +121,11 @@ void OpenH264Decoder::updateImageByTimestamp(const std::uint64_t timestamp) { m_tmp_yuv, &buffer_info); if (ret != 0) { spdlog::error( - "OpenH264Decoder DecodeFrameNoDelay failed: error_code={}", ret); - throw std::runtime_error(fmt::format( - "m_decoder->DecodeFrameNoDelay() failed: error_code={}", ret)); + "OpenH264Decoder DecodeFrameNoDelay failed: error_code={}", + static_cast(ret)); + throw std::runtime_error( + fmt::format("m_decoder->DecodeFrameNoDelay() failed: error_code={}", + static_cast(ret))); } m_next_timestamp = static_cast(m_webm->getTimestamp()); if (buffer_info.iBufferStatus == 1) { diff --git a/src/video/openh264_decoder.hpp b/src/video/openh264_decoder.hpp index ae7a0ed5..34b37bcb 100644 --- a/src/video/openh264_decoder.hpp +++ b/src/video/openh264_decoder.hpp @@ -22,7 +22,7 @@ class OpenH264Decoder : public Decoder { explicit OpenH264Decoder(std::shared_ptr); ~OpenH264Decoder(); - const std::shared_ptr getImage(const std::uint64_t); + const std::shared_ptr getImage(const std::uint64_t) override; private: ::ISVCDecoder* m_decoder = nullptr; diff --git a/src/video/parallel_grid_composer.hpp b/src/video/parallel_grid_composer.hpp index 422f94c2..0dfbd365 100644 --- a/src/video/parallel_grid_composer.hpp +++ b/src/video/parallel_grid_composer.hpp @@ -28,7 +28,7 @@ class ParallelGridComposer : public Composer { ~ParallelGridComposer(); void compose(std::vector*, - const std::vector>&); + const std::vector>&) override; private: std::uint32_t m_single_width; diff --git a/src/video/preserve_aspect_ratio_scaler.hpp b/src/video/preserve_aspect_ratio_scaler.hpp index 3f303cc9..be69ddcc 100644 --- a/src/video/preserve_aspect_ratio_scaler.hpp +++ b/src/video/preserve_aspect_ratio_scaler.hpp @@ -23,7 +23,8 @@ class PreserveAspectRatioScaler : public Scaler { PreserveAspectRatioScaler(const std::uint32_t, const std::uint32_t, const libyuv::FilterMode); - const std::shared_ptr scale(const std::shared_ptr); + const std::shared_ptr scale( + const std::shared_ptr) override; private: const libyuv::FilterMode m_filter_mode; diff --git a/src/video/simple_scaler.hpp b/src/video/simple_scaler.hpp index e22d74f7..ed4774fd 100644 --- a/src/video/simple_scaler.hpp +++ b/src/video/simple_scaler.hpp @@ -16,7 +16,8 @@ class SimpleScaler : public Scaler { SimpleScaler(const std::uint32_t, const std::uint32_t, const libyuv::FilterMode); - const std::shared_ptr scale(const std::shared_ptr src); + const std::shared_ptr scale( + const std::shared_ptr src) override; const libyuv::FilterMode m_filter_mode; }; diff --git a/src/video/vaapi_utils.cpp b/src/video/vaapi_utils.cpp new file mode 100644 index 00000000..7d17ca74 --- /dev/null +++ b/src/video/vaapi_utils.cpp @@ -0,0 +1,57 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/src/vaapi_utils.cpp より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ + +#include "vaapi_utils.h" + +namespace hisui::video { + +mfxStatus va_to_mfx_status(VAStatus va_res) { + mfxStatus mfxRes = MFX_ERR_NONE; + + switch (va_res) { + case VA_STATUS_SUCCESS: + mfxRes = MFX_ERR_NONE; + break; + case VA_STATUS_ERROR_ALLOCATION_FAILED: + mfxRes = MFX_ERR_MEMORY_ALLOC; + break; + case VA_STATUS_ERROR_ATTR_NOT_SUPPORTED: + case VA_STATUS_ERROR_UNSUPPORTED_PROFILE: + case VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT: + case VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT: + case VA_STATUS_ERROR_UNSUPPORTED_BUFFERTYPE: + case VA_STATUS_ERROR_FLAG_NOT_SUPPORTED: + case VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED: + mfxRes = MFX_ERR_UNSUPPORTED; + break; + case VA_STATUS_ERROR_INVALID_DISPLAY: + case VA_STATUS_ERROR_INVALID_CONFIG: + case VA_STATUS_ERROR_INVALID_CONTEXT: + case VA_STATUS_ERROR_INVALID_SURFACE: + case VA_STATUS_ERROR_INVALID_BUFFER: + case VA_STATUS_ERROR_INVALID_IMAGE: + case VA_STATUS_ERROR_INVALID_SUBPICTURE: + mfxRes = MFX_ERR_NOT_INITIALIZED; + break; + case VA_STATUS_ERROR_INVALID_PARAMETER: + mfxRes = MFX_ERR_INVALID_VIDEO_PARAM; + default: + mfxRes = MFX_ERR_UNKNOWN; + break; + } + return mfxRes; +} + +} // namespace hisui::video diff --git a/src/video/vaapi_utils.h b/src/video/vaapi_utils.h new file mode 100644 index 00000000..6bbc4775 --- /dev/null +++ b/src/video/vaapi_utils.h @@ -0,0 +1,45 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/include/vaapi_utils.h より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ + +#pragma once + +// libva +#include + +// oneVPL +#include + +namespace hisui::video { + +class CLibVA { + public: + virtual ~CLibVA(void) {} + + VADisplay GetVADisplay() { return m_va_dpy; } + + protected: + CLibVA() : m_va_dpy(NULL) {} + VADisplay m_va_dpy; + + private: + CLibVA(CLibVA&&) = delete; + CLibVA(const CLibVA&) = delete; + CLibVA& operator=(CLibVA&&) = delete; + CLibVA& operator=(const CLibVA&) = delete; +}; + +mfxStatus va_to_mfx_status(VAStatus va_res); + +} // namespace hisui::video diff --git a/src/video/vaapi_utils_drm.cpp b/src/video/vaapi_utils_drm.cpp new file mode 100644 index 00000000..21d62d7c --- /dev/null +++ b/src/video/vaapi_utils_drm.cpp @@ -0,0 +1,134 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/src/vaapi_utils_drm.cpp より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ +#include "vaapi_utils_drm.h" + +// Linux +#include +#include +#include +#include +#include + +// DRM +#include + +// libva +#include + +namespace hisui::video { + +constexpr mfxU32 MFX_DRI_MAX_NODES_NUM = 16; +constexpr mfxU32 MFX_DRI_RENDER_START_INDEX = 128; +constexpr mfxU32 MFX_DRI_CARD_START_INDEX = 0; +constexpr mfxU32 MFX_DRM_DRIVER_NAME_LEN = 4; +const char* MFX_DRM_INTEL_DRIVER_NAME = "i915"; +const char* MFX_DRI_PATH = "/dev/dri/"; +const char* MFX_DRI_NODE_RENDER = "renderD"; +const char* MFX_DRI_NODE_CARD = "card"; + +int get_drm_driver_name(int fd, char* name, int name_size) { + drm_version_t version = {}; + version.name_len = name_size; + version.name = name; + return ioctl(fd, DRM_IOWR(0, drm_version), &version); +} + +int open_first_intel_adapter() { + std::string adapterPath = MFX_DRI_PATH; + char driverName[MFX_DRM_DRIVER_NAME_LEN + 1] = {}; + mfxU32 nodeIndex; + + adapterPath += MFX_DRI_NODE_RENDER; + nodeIndex = MFX_DRI_RENDER_START_INDEX; + + for (mfxU32 i = 0; i < MFX_DRI_MAX_NODES_NUM; ++i) { + std::string curAdapterPath = adapterPath + std::to_string(nodeIndex + i); + + int fd = open(curAdapterPath.c_str(), O_RDWR); + if (fd < 0) + continue; + + if (!get_drm_driver_name(fd, driverName, MFX_DRM_DRIVER_NAME_LEN) && + !strcmp(driverName, MFX_DRM_INTEL_DRIVER_NAME)) { + return fd; + } + close(fd); + } + + return -1; +} + +int open_intel_adapter(const std::string& devicePath) { + if (devicePath.empty()) + return open_first_intel_adapter(); + + int fd = open(devicePath.c_str(), O_RDWR); + + if (fd < 0) { + return -1; + } + + char driverName[MFX_DRM_DRIVER_NAME_LEN + 1] = {}; + if (!get_drm_driver_name(fd, driverName, MFX_DRM_DRIVER_NAME_LEN) && + !strcmp(driverName, MFX_DRM_INTEL_DRIVER_NAME)) { + return fd; + } else { + close(fd); + return -1; + } +} + +DRMLibVA::DRMLibVA(const std::string& devicePath) : CLibVA(), m_fd(-1) { + mfxStatus sts = MFX_ERR_NONE; + + m_fd = open_intel_adapter(devicePath); + if (m_fd < 0) + throw std::range_error("Intel GPU was not found"); + + m_va_dpy = vaGetDisplayDRM(m_fd); + if (m_va_dpy) { + int major_version = 0, minor_version = 0; + VAStatus va_res = vaInitialize(m_va_dpy, &major_version, &minor_version); + sts = va_to_mfx_status(va_res); + } else { + sts = MFX_ERR_NULL_PTR; + } + + if (MFX_ERR_NONE != sts) { + if (m_va_dpy) + vaTerminate(m_va_dpy); + close(m_fd); + throw std::runtime_error("Loading of VA display was failed"); + } +} + +DRMLibVA::~DRMLibVA(void) { + if (m_va_dpy) { + vaTerminate(m_va_dpy); + } + if (m_fd >= 0) { + close(m_fd); + } +} + +std::unique_ptr CreateDRMLibVA(const std::string& devicePath) { + try { + return std::unique_ptr(new DRMLibVA(devicePath)); + } catch (std::exception& e) { + return nullptr; + } +} + +} // namespace hisui::video diff --git a/src/video/vaapi_utils_drm.h b/src/video/vaapi_utils_drm.h new file mode 100644 index 00000000..0548cc72 --- /dev/null +++ b/src/video/vaapi_utils_drm.h @@ -0,0 +1,44 @@ +// https://github.com/Intel-Media-SDK/MediaSDK/blob/master/samples/sample_common/include/vaapi_utils_drm.h より。 +// オリジナルのライセンスは以下。 +/******************************************************************************\ +Copyright (c) 2005-2019, Intel Corporation +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +This sample was distributed or derived from the Intel's Media Samples package. +The original version of this sample may be obtained from https://software.intel.com/en-us/intel-media-server-studio +or https://software.intel.com/en-us/media-client-solutions-support. +\**********************************************************************************/ + +#pragma once + +#include +#include + +#include "vaapi_utils.h" + +namespace hisui::video { + +class DRMLibVA : public CLibVA { + public: + DRMLibVA(const std::string& devicePath); + virtual ~DRMLibVA(void); + + inline int getFD() { return m_fd; } + + protected: + int m_fd; + + private: + DRMLibVA(DRMLibVA&&) = delete; + DRMLibVA(const DRMLibVA&) = delete; + DRMLibVA& operator=(DRMLibVA&&) = delete; + DRMLibVA& operator=(const DRMLibVA&) = delete; +}; + +std::unique_ptr CreateDRMLibVA(const std::string& devicePath = ""); + +} // namespace hisui::video diff --git a/src/video/vpl.cpp b/src/video/vpl.cpp new file mode 100644 index 00000000..ced3fbb2 --- /dev/null +++ b/src/video/vpl.cpp @@ -0,0 +1,29 @@ +#include "video/vpl.hpp" + +#include +#include +#include + +#include +#include + +#include "constants.hpp" + +namespace hisui::video { + +::mfxU32 ToMfxCodec(const std::uint32_t fourcc) { + switch (fourcc) { + case hisui::Constants::VP8_FOURCC: + return MFX_CODEC_VP8; + case hisui::Constants::VP9_FOURCC: + return MFX_CODEC_VP9; + case hisui::Constants::H264_FOURCC: + return MFX_CODEC_AVC; + case hisui::Constants::AV1_FOURCC: + return MFX_CODEC_AV1; + default: + throw std::invalid_argument(fmt::format("unknown fourcc: {:x}", fourcc)); + } +} + +} // namespace hisui::video diff --git a/src/video/vpl.hpp b/src/video/vpl.hpp new file mode 100644 index 00000000..9ac2c080 --- /dev/null +++ b/src/video/vpl.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +#include +#include +#include + +#include "constants.hpp" +#include "video/decoder.hpp" +#include "video/vpl_session.hpp" + +namespace hisui::video { + +mfxU32 ToMfxCodec(const std::uint32_t fourcc); + +} diff --git a/src/video/vpl_decoder.cpp b/src/video/vpl_decoder.cpp new file mode 100644 index 00000000..1217bcac --- /dev/null +++ b/src/video/vpl_decoder.cpp @@ -0,0 +1,378 @@ +#include "video/vpl_decoder.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "report/reporter.hpp" +#include "video/vpl.hpp" +#include "video/vpl_session.hpp" +#include "video/yuv.hpp" +#include "webm/input/video_context.hpp" + +namespace hisui::video { + +bool VPLDecoder::initVpl() { + m_decoder = createDecoder(m_fourcc, {{4096, 4096}, {2048, 2048}}); + if (!m_decoder) { + throw std::runtime_error( + fmt::format("createDecoder() failed: fourcc={}", m_fourcc)); + } + + ::mfxStatus sts = MFX_ERR_NONE; + + ::mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + sts = m_decoder->GetVideoParam(¶m); + if (sts != MFX_ERR_NONE) { + return false; + } + + memset(&m_alloc_request, 0, sizeof(m_alloc_request)); + sts = m_decoder->QueryIOSurf(¶m, &m_alloc_request); + if (sts != MFX_ERR_NONE) { + throw std::runtime_error(fmt::format("QueryIOSurf() failed: sts={}", + static_cast(sts))); + } + + spdlog::debug("Decoder NumFrameSuggested={}", + m_alloc_request.NumFrameSuggested); + + // 入力ビットストリーム + m_bitstream_buffer.resize(1024 * 1024); + memset(&m_bitstream, 0, sizeof(m_bitstream)); + m_bitstream.MaxLength = static_cast(m_bitstream_buffer.size()); + m_bitstream.Data = m_bitstream_buffer.data(); + + // 入力ビットストリーム + // 必要な枚数分の出力サーフェスを作る + { + auto width = (m_alloc_request.Info.Width + 31) / 32 * 32; + auto height = (m_alloc_request.Info.Height + 31) / 32 * 32; + // 1枚あたりのバイト数 + // NV12 なので 1 ピクセルあたり 12 ビット + auto size = width * height * 12 / 8; + m_surface_buffer.resize( + static_cast(m_alloc_request.NumFrameSuggested * size)); + + m_surfaces.clear(); + m_surfaces.reserve(m_alloc_request.NumFrameSuggested); + for (int i = 0; i < m_alloc_request.NumFrameSuggested; i++) { + ::mfxFrameSurface1 surface; + memset(&surface, 0, sizeof(surface)); + surface.Info = param.mfx.FrameInfo; + surface.Data.Y = m_surface_buffer.data() + i * size; + surface.Data.U = m_surface_buffer.data() + i * size + width * height; + surface.Data.V = m_surface_buffer.data() + i * size + width * height + 1; + surface.Data.Pitch = static_cast(width); + m_surfaces.push_back(surface); + } + } + + return true; +} + +void VPLDecoder::releaseVpl() { + if (m_decoder != nullptr) { + m_decoder->Close(); + } +} + +VPLDecoder::VPLDecoder(std::shared_ptr t_webm) + : Decoder(t_webm), m_fourcc(t_webm->getFourcc()) { + initVpl(); + + m_current_yuv_image = + std::shared_ptr(create_black_yuv_image(m_width, m_height)); + m_next_yuv_image = + std::shared_ptr(create_black_yuv_image(m_width, m_height)); + + if (hisui::report::Reporter::hasInstance()) { + m_report_enabled = true; + + hisui::report::Reporter::getInstance().registerVideoDecoder( + m_webm->getFilePath(), + {.codec = "H.264", .duration = m_webm->getDuration()}); + + hisui::report::Reporter::getInstance().registerResolutionChange( + m_webm->getFilePath(), + {.timestamp = 0, .width = m_width, .height = m_height}); + } +} + +VPLDecoder::~VPLDecoder() { + releaseVpl(); +} + +const std::shared_ptr VPLDecoder::getImage( + const std::uint64_t timestamp) { + if (!m_webm || m_is_time_over) { + return m_black_yuv_image; + } + // 時間超過した + if (m_duration <= timestamp) { + m_is_time_over = true; + return m_black_yuv_image; + } + updateImage(timestamp); + return m_current_yuv_image; +} + +void VPLDecoder::updateImage(const std::uint64_t timestamp) { + // 次のブロックに逹っしていない + if (timestamp < m_next_timestamp) { + return; + } + // 次以降のブロックに逹っした + updateImageByTimestamp(timestamp); +} + +void VPLDecoder::updateImageByTimestamp(const std::uint64_t timestamp) { + if (m_finished_webm) { + return; + } + + do { + if (m_report_enabled) { + if (m_current_yuv_image->getWidth(0) != m_next_yuv_image->getWidth(0) || + m_current_yuv_image->getHeight(0) != m_next_yuv_image->getHeight(0)) { + hisui::report::Reporter::getInstance().registerResolutionChange( + m_webm->getFilePath(), {.timestamp = m_next_timestamp, + .width = m_next_yuv_image->getWidth(0), + .height = m_next_yuv_image->getHeight(0)}); + } + } + m_current_yuv_image = m_next_yuv_image; + m_current_timestamp = m_next_timestamp; + if (m_webm->readFrame()) { + decode(); + m_next_timestamp = static_cast(m_webm->getTimestamp()); + } else { + // m_duration までは m_current_image を出すので webm を読み終えても m_current_image を維持する + m_finished_webm = true; + m_next_timestamp = std::numeric_limits::max(); + return; + } + } while (timestamp >= m_next_timestamp); +} + +void VPLDecoder::decode() { + auto buffer_size = m_webm->getBufferSize(); + + if (m_bitstream.MaxLength < m_bitstream.DataLength + buffer_size) { + m_bitstream_buffer.resize(m_bitstream.DataLength + buffer_size); + m_bitstream.MaxLength = static_cast( + m_bitstream.DataLength + m_bitstream_buffer.size()); + m_bitstream.Data = m_bitstream_buffer.data(); + } + + memmove(m_bitstream.Data, m_bitstream.Data + m_bitstream.DataOffset, + m_bitstream.DataLength); + m_bitstream.DataOffset = 0; + memcpy(m_bitstream.Data + m_bitstream.DataLength, m_webm->getBuffer(), + buffer_size); + m_bitstream.DataLength += buffer_size; + + auto surface = + std::find_if(std::begin(m_surfaces), std::end(m_surfaces), + [](const mfxFrameSurface1& s) { return !s.Data.Locked; }); + if (surface == std::end(m_surfaces)) { + throw std::runtime_error("unlocked surface is not found"); + } + ::mfxStatus sts; + ::mfxSyncPoint syncp; + ::mfxFrameSurface1* out_surface = nullptr; + while (true) { + sts = m_decoder->DecodeFrameAsync(&m_bitstream, &*surface, &out_surface, + &syncp); + if (sts == MFX_WRN_DEVICE_BUSY) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + // 受信した映像のサイズが変わってたら width, height を更新する + if (sts == MFX_WRN_VIDEO_PARAM_CHANGED) { + mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + sts = m_decoder->GetVideoParam(¶m); + if (sts != MFX_ERR_NONE) { + throw std::runtime_error(fmt::format("GetVideoParam() failed: sts={}", + static_cast(sts))); + } + + if (m_width != param.mfx.FrameInfo.CropW || + m_height != param.mfx.FrameInfo.CropH) { + m_width = param.mfx.FrameInfo.CropW; + m_height = param.mfx.FrameInfo.CropH; + } + continue; + } + break; + } + if (sts == MFX_ERR_MORE_DATA) { + // もっと入力が必要なので出直す + return; + } + + if (!syncp) { + spdlog::info( + "Failed to DecodeFrameAsync: syncp is null, file_path={} sts={}", + m_webm->getFilePath(), static_cast(sts)); + return; + } + + if (sts != MFX_ERR_NONE) { + throw std::runtime_error( + fmt::format("DecodeFrameAsync() failed: file_path={}, sts={}", + m_webm->getFilePath(), static_cast(sts))); + } + + // H264 は sts == MFX_WRN_VIDEO_PARAM_CHANGED でハンドリングできるのでここではチェックしない + // VP9 は受信フレームのサイズが変わっても MFX_WRN_VIDEO_PARAM_CHANGED を返さないようなので、 + // ここで毎フレーム情報を取得してサイズを更新する。 + if (m_fourcc != Constants::H264_FOURCC) { + ::mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + sts = m_decoder->GetVideoParam(¶m); + if (sts != MFX_ERR_NONE) { + throw std::runtime_error(fmt::format("GetVideoParam() failed: sts={}", + static_cast(sts))); + } + + if (m_width != param.mfx.FrameInfo.CropW || + m_height != param.mfx.FrameInfo.CropH) { + m_width = param.mfx.FrameInfo.CropW; + m_height = param.mfx.FrameInfo.CropH; + } + } + + sts = ::MFXVideoCORE_SyncOperation( + hisui::video::VPLSession::getInstance().getSession(), syncp, 600000); + if (sts != MFX_ERR_NONE) { + throw std::runtime_error( + fmt::format("MFXVideoCORE_SyncOperation() failed: sts={}", + static_cast(sts))); + } + + m_next_yuv_image = std::make_shared(m_width, m_height); + // NV12 から I420 に変換 + libyuv::NV12ToI420(out_surface->Data.Y, out_surface->Data.Pitch, + out_surface->Data.UV, out_surface->Data.Pitch, + m_next_yuv_image->yuv[0], static_cast(m_width), + m_next_yuv_image->yuv[1], (m_width + 1) >> 1, + m_next_yuv_image->yuv[2], (m_width + 1) >> 1, + static_cast(m_width), static_cast(m_height)); + + return; +} + +std::unique_ptr<::MFXVideoDECODE> VPLDecoder::createDecoder( + const std::uint32_t fourcc, + const std::vector> sizes) { + if (!hisui::video::VPLSession::hasInstance()) { + throw std::runtime_error("VPL session is not opened"); + } + for (auto size : sizes) { + auto decoder = + createDecoderInternal(hisui::video::VPLSession::getInstance(), + ToMfxCodec(fourcc), size.first, size.second); + if (decoder) { + return decoder; + } + } + return nullptr; +} + +std::unique_ptr<::MFXVideoDECODE> VPLDecoder::createDecoderInternal( + const VPLSession& session, + ::mfxU32 codec, + std::uint32_t width, + std::uint32_t height) { + std::unique_ptr decoder( + new MFXVideoDECODE(session.getSession())); + + ::mfxStatus sts = MFX_ERR_NONE; + + ::mfxVideoParam param; + memset(¶m, 0, sizeof(param)); + + param.mfx.CodecId = codec; + param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; + param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; + param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; + param.mfx.FrameInfo.CropX = 0; + param.mfx.FrameInfo.CropY = 0; + param.mfx.FrameInfo.CropW = static_cast(width); + param.mfx.FrameInfo.CropH = static_cast(height); + param.mfx.FrameInfo.Width = + (static_cast(width) + 15) / 16 * 16; + param.mfx.FrameInfo.Height = + (static_cast(height) + 15) / 16 * 16; + + param.mfx.GopRefDist = 1; + param.AsyncDepth = 1; + param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY; + + // qmfxExtCodingOption ext_coding_option; + // qmemset(&ext_coding_option, 0, sizeof(ext_coding_option)); + // qext_coding_option.Header.BufferId = MFX_EXTBUFF_CODING_OPTION; + // qext_coding_option.Header.BufferSz = sizeof(ext_coding_option); + // qext_coding_option.MaxDecFrameBuffering = 1; + + // qmfxExtBuffer* ext_buffers[1]; + // qext_buffers[0] = (mfxExtBuffer*)&ext_coding_option; + // qparam.ExtParam = ext_buffers; + // qparam.NumExtParam = sizeof(ext_buffers) / sizeof(ext_buffers[0]); + + sts = decoder->Query(¶m, ¶m); + if (sts < 0) { + const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" + : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" + : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" + : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" + : "MFX_CODEC_UNKNOWN"; + spdlog::debug("Unsupported decoder codec: codec={}, sts={}", codec_str, + static_cast(sts)); + return nullptr; + } + + // if (sts != MFX_ERR_NONE) { + // RTC_LOG(LS_WARNING) << "Supported specified codec but has warning: sts=" + // << sts; + // } + + // Query した上で Init しても MFX_ERR_UNSUPPORTED になることがあるので + // 本来 Init が不要な時も常に呼ぶようにして確認する + /*if (init)*/ { + // Initialize the oneVPL encoder + sts = decoder->Init(¶m); + if (sts != MFX_ERR_NONE) { + const char* codec_str = codec == MFX_CODEC_VP8 ? "MFX_CODEC_VP8" + : codec == MFX_CODEC_VP9 ? "MFX_CODEC_VP9" + : codec == MFX_CODEC_AV1 ? "MFX_CODEC_AV1" + : codec == MFX_CODEC_AVC ? "MFX_CODEC_AVC" + : "MFX_CODEC_UNKNOWN"; + spdlog::warn( + "decoder->Init() failed: codec={}, std={}, width={}, height={}", + codec_str, static_cast(sts), width, height); + return nullptr; + } + } + + return decoder; +} + +bool VPLDecoder::isSupported(const std::uint32_t fourcc) { + auto decoder = createDecoder(fourcc, {{4096, 4096}, {2048, 2048}}); + + return decoder != nullptr; +} + +} // namespace hisui::video diff --git a/src/video/vpl_decoder.hpp b/src/video/vpl_decoder.hpp new file mode 100644 index 00000000..47522d88 --- /dev/null +++ b/src/video/vpl_decoder.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include "constants.hpp" +#include "video/decoder.hpp" +#include "video/vpl_session.hpp" + +namespace hisui::webm::input { + +class VideoContext; + +} + +namespace hisui::video { + +class YUVImage; + +class VPLDecoder : public Decoder { + public: + explicit VPLDecoder(std::shared_ptr); + ~VPLDecoder(); + + const std::shared_ptr getImage(const std::uint64_t) override; + + static bool isSupported(const std::uint32_t fourcc); + + private: + std::unique_ptr m_decoder; + std::uint32_t m_fourcc; + std::uint64_t m_current_timestamp = 0; + std::uint64_t m_next_timestamp = 0; + std::shared_ptr m_current_yuv_image = nullptr; + std::shared_ptr m_next_yuv_image = nullptr; + bool m_report_enabled = false; + std::vector<::mfxFrameSurface1> m_surfaces; + ::mfxFrameAllocRequest m_alloc_request; + std::vector m_surface_buffer; + std::vector m_bitstream_buffer; + ::mfxBitstream m_bitstream; + + static std::unique_ptr<::MFXVideoDECODE> createDecoder( + const ::mfxU32 codec, + const std::vector> sizes); + + static std::unique_ptr<::MFXVideoDECODE> createDecoderInternal( + const VPLSession& session, + const ::mfxU32 codec, + const std::uint32_t width, + const std::uint32_t height); + + bool initVpl(); + void releaseVpl(); + void updateImage(const std::uint64_t); + void updateImageByTimestamp(const std::uint64_t); + void decode(); +}; + +} // namespace hisui::video diff --git a/src/video/vpl_encoder.cpp b/src/video/vpl_encoder.cpp new file mode 100644 index 00000000..f595b2a1 --- /dev/null +++ b/src/video/vpl_encoder.cpp @@ -0,0 +1,405 @@ +#include "video/vpl_encoder.hpp" + +#include +#include +#include + +#include + +#include