diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4685d4c5..75208c38 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: name: - windows_x86_64 name: Build sora-cpp-sdk for ${{ matrix.name }} - runs-on: windows-2019 + runs-on: windows-2022 env: TEST_SIGNALING_URL: ${{ secrets.TEST_SIGNALING_URL }} TEST_CHANNEL_ID_PREFIX: ${{ secrets.TEST_CHANNEL_ID_PREFIX }} @@ -84,7 +84,7 @@ jobs: cd examples mkdir examples_${{ matrix.name }} foreach ($app in @('sdl_sample', 'sumomo', 'messaging_recvonly_sample')) { - python3 "$app\${{ matrix.name }}\run.py" --sora-dir .. + python3 "$app\${{ matrix.name }}\run.py" --local-sora-cpp-sdk-dir .. cp "_build\${{ matrix.name }}\release\$app\Release\$app.exe" "examples_${{ matrix.name }}" } - name: Upload Examples Artifact @@ -142,7 +142,7 @@ jobs: cd examples mkdir examples_${{ matrix.name }} for app in sdl_sample sumomo messaging_recvonly_sample; do - python3 $app/${{ matrix.name }}/run.py --sora-dir .. + python3 $app/${{ matrix.name }}/run.py --local-sora-cpp-sdk-dir .. cp _build/${{ matrix.name }}/release/$app/$app examples_${{ matrix.name }} done if: matrix.name == 'macos_arm64' @@ -156,18 +156,26 @@ jobs: strategy: fail-fast: false matrix: - name: - - ubuntu-20.04_x86_64 - - ubuntu-22.04_x86_64 - - ubuntu-20.04_armv8_jetson - - android - name: Build sora-cpp-sdk for ${{ matrix.name }} - runs-on: ${{ matrix.name == 'ubuntu-22.04_x86_64' && 'ubuntu-22.04' || 'ubuntu-20.04' }} + platform: + - name: ubuntu-20.04_x86_64 + runs-on: ubuntu-20.04 + os: ubuntu + - name: ubuntu-22.04_x86_64 + runs-on: ubuntu-22.04 + os: ubuntu + - name: ubuntu-24.04_x86_64 + runs-on: ubuntu-24.04 + os: ubuntu + - name: android + runs-on: ubuntu-22.04 + os: android + name: Build sora-cpp-sdk for ${{ matrix.platform.name }} + runs-on: ${{ matrix.platform.runs-on }} env: TEST_SIGNALING_URL: ${{ secrets.TEST_SIGNALING_URL }} TEST_CHANNEL_ID_PREFIX: ${{ secrets.TEST_CHANNEL_ID_PREFIX }} TEST_SECRET_KEY: ${{ secrets.TEST_SECRET_KEY }} - TEST_MATRIX_NAME: ${{ matrix.name }} + TEST_MATRIX_NAME: ${{ matrix.platform.name }} steps: - uses: actions/checkout@v4 - name: Disk cleanup @@ -178,7 +186,10 @@ jobs: sudo du -h -d1 /usr/local/share sudo du -h -d1 /usr/local/lib sudo du -h -d1 /usr/share - docker rmi `docker images -q -a` + RMI=`docker images -q -a` + if [ -n "$RMI" ]; then + docker rmi $RMI + fi # 4.6G sudo rm -rf /usr/local/.ghcup # 1.7G @@ -186,34 +197,49 @@ jobs: # 1.4G sudo rm -rf /usr/share/dotnet df -h - - name: Install deps for ${{ matrix.name }} - if: matrix.name == 'ubuntu-20.04_x86_64' || matrix.name == 'ubuntu-22.04_x86_64' + # Ubuntu 24.04 だと libtinfo5 が見つからない問題があるので、その修正 + # ref: https://qiita.com/gengen16k/items/88cf3c18a40a94205fab + - name: Fix CUDA issues for Ubuntu 24.04 + if: matrix.platform.name == 'ubuntu-24.04_x86_64' + run: | + sudo tee /etc/apt/sources.list.d/jammy.list << EOF + deb http://archive.ubuntu.com/ubuntu/ jammy universe + EOF + + sudo tee /etc/apt/preferences.d/pin-jammy <> $GITHUB_OUTPUT echo "boost_name=${BOOST_PACKAGE_NAME}" >> $GITHUB_OUTPUT id: package_name @@ -237,41 +263,33 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ steps.package_name.outputs.name }} - path: _package/${{ matrix.name }}/release/${{ steps.package_name.outputs.name }} + path: _package/${{ matrix.platform.name }}/release/${{ steps.package_name.outputs.name }} - name: Upload Boost Artifact uses: actions/upload-artifact@v4 with: name: ${{ steps.package_name.outputs.boost_name }} - path: _package/${{ matrix.name }}/release/${{ steps.package_name.outputs.boost_name }} + path: _package/${{ matrix.platform.name }}/release/${{ steps.package_name.outputs.boost_name }} - name: Upload Environment uses: actions/upload-artifact@v4 with: - name: ${{ matrix.name }}.env - path: _package/${{ matrix.name }}/release/sora.env + name: ${{ matrix.platform.name }}.env + path: _package/${{ matrix.platform.name }}/release/sora.env # Examples のビルド - name: Build Examples run: | cd examples - mkdir examples_${{ matrix.name }} + mkdir examples_${{ matrix.platform.name }} for app in sdl_sample sumomo messaging_recvonly_sample; do - python3 $app/${{ matrix.name }}/run.py --sora-dir .. - cp _build/${{ matrix.name }}/release/$app/$app examples_${{ matrix.name }} + python3 $app/${{ matrix.platform.name }}/run.py --local-sora-cpp-sdk-dir .. + cp _build/${{ matrix.platform.name }}/release/$app/$app examples_${{ matrix.platform.name }} done - if: matrix.name == 'ubuntu-20.04_x86_64' || matrix.name == 'ubuntu-22.04_x86_64' || matrix.name == 'ubuntu-20.04_armv8_jetson' + if: matrix.platform.os == 'ubuntu' - name: Upload Examples Artifact uses: actions/upload-artifact@v4 with: - name: examples_${{ matrix.name }} - path: examples/examples_${{ matrix.name }} - if: matrix.name == 'ubuntu-20.04_x86_64' || matrix.name == 'ubuntu-22.04_x86_64' || matrix.name == 'ubuntu-20.04_armv8_jetson' - - name: Slack Notification - if: failure() - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_CHANNEL: sora-cpp-sdk - SLACK_COLOR: danger - SLACK_TITLE: Build failed - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + name: examples_${{ matrix.platform.name }} + path: examples/examples_${{ matrix.platform.name }} + if: matrix.platform.os == 'ubuntu' create-release: name: Create Release @@ -292,15 +310,15 @@ jobs: - uses: ./.github/actions/download with: platform: ios - - uses: ./.github/actions/download - with: - platform: ubuntu-20.04_armv8_jetson - uses: ./.github/actions/download with: platform: ubuntu-20.04_x86_64 - uses: ./.github/actions/download with: platform: ubuntu-22.04_x86_64 + - uses: ./.github/actions/download + with: + platform: ubuntu-24.04_x86_64 - uses: ./.github/actions/download with: platform: android @@ -315,11 +333,25 @@ jobs: with: files: ${{ steps.env.outputs.package_paths }} prerelease: ${{ contains(github.ref, 'canary') }} - - name: Slack Notification - if: failure() - uses: rtCamp/action-slack-notify@v2 + + notification: + name: Slack Notification + runs-on: ubuntu-latest + needs: + - build-windows + - build-macos + - build-ubuntu + - create-release + if: always() + steps: + - uses: rtCamp/action-slack-notify@v2 + if: | + needs.build-windows.result == 'failure' || + needs.build-macos.result == 'failure' || + needs.build-ubuntu.result == 'failure' || + needs.create-release.result == 'failure' env: SLACK_CHANNEL: sora-cpp-sdk SLACK_COLOR: danger - SLACK_TITLE: Release failed + SLACK_TITLE: Build failed SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 72873002..c48c0204 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -26,7 +26,8 @@ "_LIBCPP_DISABLE_AVAILABILITY", "OPENSSL_IS_BORINGSSL", "SORA_CPP_SDK_UBUNTU_2004", - "USE_NVCODEC_ENCODER" + "USE_NVCODEC_ENCODER", + "RTC_ENABLE_H265" ], "compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], @@ -58,7 +59,8 @@ "OPENSSL_IS_BORINGSSL", "USE_JETSON_ENCODER", "SORA_CPP_SDK_JETSON", - "HELLO_JETSON" + "HELLO_JETSON", + "RTC_ENABLE_H265" ], "compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_armv8_jetson/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], @@ -92,9 +94,11 @@ "OPENSSL_IS_BORINGSSL", "SORA_CPP_SDK_UBUNTU_2204", "USE_NVCODEC_ENCODER", - "USE_VPL_ENCODER" + "USE_VPL_ENCODER", + "RTC_ENABLE_H265" ], - "compilerPath": "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/llvm/clang/bin/clang++", + // "compilerPath": "${workspaceFolder}/_install/ubuntu-22.04_x86_64/release/llvm/clang/bin/clang++", + "compilerPath": "/usr/bin/clang++-18", "compilerArgs": ["-nostdinc++"], "cStandard": "gnu17", "cppStandard": "gnu++17", @@ -123,7 +127,8 @@ "OPENSSL_IS_BORINGSSL", "USE_NVCODEC_ENCODER=0", "SORA_CPP_SDK_ANDROID", - "HELLO_ANDROID" + "HELLO_ANDROID", + "RTC_ENABLE_H265" ], "compilerPath": "${workspaceFolder}/_install/android/release/llvm/clang/bin/clang++", "compilerArgs": ["-nostdinc++"], @@ -148,7 +153,8 @@ "defines": [ "WEBRTC_POSIX", "WEBRTC_MAC", - "OPENSSL_IS_BORINGSSL" + "OPENSSL_IS_BORINGSSL", + "RTC_ENABLE_H265" ], "cStandard": "gnu17", "cppStandard": "gnu++17", @@ -174,8 +180,9 @@ "defines": [ "SORA_CPP_SDK_WINDOWS", "WEBRTC_WIN", - "USE_NVCODEC_ENCODER=1", - "USE_VPL_ENCODER=1" + "USE_NVCODEC_ENCODER", + "USE_VPL_ENCODER", + "RTC_ENABLE_H265" ], "cStandard": "gnu17", "cppStandard": "gnu++17", @@ -200,7 +207,8 @@ "defines": [ "SORA_CPP_SDK_WINDOWS", "WEBRTC_WIN", - "USE_NVCODEC_ENCODER=1" + "USE_NVCODEC_ENCODER", + "RTC_ENABLE_H265" ], "cStandard": "gnu17", "cppStandard": "gnu++17", diff --git a/CHANGES.md b/CHANGES.md index 564321bf..030ec8e6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,49 @@ ## develop +## 2024.7.0 (2024-07-29) + +- [CHANGE] `--sora-dir`, `--sora-args` を `--local-sora-cpp-sdk-dir` と `--local-sora-cpp-sdk-args` に変更する + - @melpon +- [CHANGE] `--webrtc-build-dir`, `--webrtc-build-args` を `--local-webrtc-build-dir` と `--local-webrtc-build-args` に変更する + - @melpon +- [CHANGE] SoraVideoDecoderFactory の CreateVideoDecoder 関数を Create に変更する + - libwebrtc に定義されている継承元クラスが変更されたことに対する追従 + - @enm10k +- [CHANGE] SoraVideoEncoderFactory の CreateVideoEncoder 関数を Create に変更する + - libwebrtc に定義されている継承元クラスが変更されたことに対する追従 + - @enm10k +- [CHANGE] `CreateOpenH264VideoEncoder()` 関数の第1引数の型を `const cricket::VideoCodec&` から `const webrtc::SdpVideoFormat&` に変更する + - @melpon +- [CHANGE] Jetson 対応をサポートブランチに移動する + - @melpon +- [UPDATE] Boost を 1.85.0 にあげる + - @enm10k +- [UPDATE] SDL2 を 2.30.3 にあげる + - @enm10k +- [UPDATE] CLI11 を 2.4.2 にあげる + - @enm10k +- [UPDATE] libwebrtc を m126.6478.1.1 にあげる + - Android の test アプリがリンクできなくなったため、リンカーを Android NDK のものから libwebrtc のものに変更 + - リンカーのバージョンの違いによる互換性の問題でエラーが発生していた + - @melpon @enm10k @torikizi +- [ADD] sumomo に `--openh264` と `--use-hardware-encoder` オプションを追加 + - @melpon +- [ADD] sumomo に `--video-h264-params` と `--video-h265-params` オプションを追加 + - @enm10k +- [ADD] SoraSignalingConfig に `video_h265_params` を追加 + - @enm10k +- [ADD] 古い Intel CPU でも H265 エンコーダが動くようにする + - @melpon +- [ADD] Intel VPL で AV1 デコーダを動くようにする + - @melpon +- [ADD] NVIDIA Video Codec SDK を H265 に対応する + - @melpon +- [ADD] Ubuntu 24.04 に対応する + - @melpon +- [ADD] WebSocket での接続時に User-Agent ヘッダーを追加する + - @melpon + ## 2024.6.1 (2024-04-16) - [CHANGE] テストモジュールについて `SoraSignalingConfig` の `sora_client` に値を設定しないようにする diff --git a/CMakeLists.txt b/CMakeLists.txt index 01d67cf9..21e76e52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,10 +45,11 @@ elseif (SORA_TARGET STREQUAL "ubuntu-22.04_x86_64") set(SORA_TARGET_OS_VERSION "22.04") set(SORA_TARGET_ARCH "x86_64") set(SORA_TARGET_DEF "SORA_CPP_SDK_UBUNTU_2204") -elseif (SORA_TARGET STREQUAL "ubuntu-20.04_armv8_jetson") - set(SORA_TARGET_OS "jetson") - set(SORA_TARGET_ARCH "armv8") - set(SORA_TARGET_DEF "SORA_CPP_SDK_JETSON") +elseif (SORA_TARGET STREQUAL "ubuntu-24.04_x86_64") + set(SORA_TARGET_OS "ubuntu") + set(SORA_TARGET_OS_VERSION "24.04") + set(SORA_TARGET_ARCH "x86_64") + set(SORA_TARGET_DEF "SORA_CPP_SDK_UBUNTU_2404") endif() list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) @@ -149,7 +150,6 @@ target_include_directories(sora INTERFACE $:USE_NVCODEC_ENCODER> - $<$:USE_JETSON_ENCODER> $<$:USE_VPL_ENCODER> ) @@ -254,7 +254,7 @@ if (SORA_TARGET_OS STREQUAL "windows") if (USE_NVCODEC_ENCODER) target_sources(sora PRIVATE - src/hwenc_nvcodec/nvcodec_h264_encoder.cpp + src/hwenc_nvcodec/nvcodec_video_encoder.cpp third_party/NvCodec/NvCodec/NvEncoder/NvEncoder.cpp third_party/NvCodec/NvCodec/NvEncoder/NvEncoderD3D11.cpp src/hwenc_nvcodec/nvcodec_video_decoder.cpp) @@ -287,6 +287,23 @@ if (SORA_TARGET_OS STREQUAL "windows") third_party/NvCodec/NvCodec/NvDecoder/NvDecoder.cpp src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp OPTIONS + # VS 2022 の 17.10.x 以上に上げると、ツールセットのバージョンが 14.40 以上になってしまい、以下のエラーが出るため -allow-unsupported-compiler を指定する + # + # G:\dev\sora-cpp-sdk\_install\windows_x86_64\release\cuda\include\crt/host_config.h(153): fatal error C1189: #error: -- unsupported Microsoft Visual Studio version! Only the versions between 20 17 and 2022 (inclusive) are supported! The nvcc flag '-allow-unsupported-compiler' can be used to override this version check; however, using an unsupported host compiler may cause compilation failure or incorrect run time execution. Use at your own risk. [G:\dev\sora-cpp-sdk\_build\windows_x86_64\release\sora\sora.vcxproj] + # + # host_config.h では以下のような記述になっている + # + # ``` + # #if _MSC_VER < 1910 || _MSC_VER >= 1940 + # + # #error -- unsupported Microsoft Visual Studio version! Only the versions between 2017 and 2022 (inclusive) are supported! (...snip...) + # ``` + # + # 17.10 は _MSC_VER が 1940 になるため、このエラーが出る。 + # + # VS のバージョンと _MSC_VER のリストは以下を参照: + # https://devblogs.microsoft.com/cppblog/msvc-toolset-minor-version-number-14-40-in-vs-2022-v17-10/ + -allow-unsupported-compiler -Xcompiler /utf-8 -Xcompiler /I${CMAKE_CURRENT_SOURCE_DIR}/third_party/NvCodec/include -Xcompiler /I${CMAKE_CURRENT_SOURCE_DIR}/third_party/NvCodec/NvCodec @@ -447,9 +464,9 @@ elseif (SORA_TARGET_OS STREQUAL "ubuntu") target_sources(sora PRIVATE src/cuda_context_cuda.cpp - src/hwenc_nvcodec/nvcodec_h264_encoder.cpp + src/hwenc_nvcodec/nvcodec_video_encoder.cpp src/hwenc_nvcodec/nvcodec_v4l2_capturer.cpp - src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp + src/hwenc_nvcodec/nvcodec_video_encoder_cuda.cpp src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp src/hwenc_nvcodec/nvcodec_video_decoder.cpp third_party/NvCodec/NvCodec/NvDecoder/NvDecoder.cpp @@ -464,7 +481,7 @@ elseif (SORA_TARGET_OS STREQUAL "ubuntu") # これらのソースは CUDA としてコンパイルする set_source_files_properties( src/cuda_context_cuda.cpp - src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp + src/hwenc_nvcodec/nvcodec_video_encoder_cuda.cpp src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp third_party/NvCodec/NvCodec/NvDecoder/NvDecoder.cpp third_party/NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp @@ -496,91 +513,6 @@ elseif (SORA_TARGET_OS STREQUAL "ubuntu") list(APPEND BUNDLE_STATIC_LIBS $) endif() -elseif (SORA_TARGET_OS STREQUAL "jetson") - target_sources(sora PRIVATE src/v4l2/v4l2_video_capturer.cpp) - - target_compile_definitions(sora - PUBLIC - WEBRTC_POSIX - WEBRTC_LINUX - _LIBCPP_ABI_NAMESPACE=Cr - _LIBCPP_ABI_VERSION=2 - _LIBCPP_DISABLE_AVAILABILITY - _LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE - # https://github.com/boostorg/container_hash/issues/22 と同じ問題が clang-15 でも起きるので、これを手動で定義して回避する - BOOST_NO_CXX98_FUNCTION_BASE - ) - - set_target_properties(sora PROPERTIES POSITION_INDEPENDENT_CODE ON) - target_link_libraries(sora - PRIVATE - X11 - #Xau - #Xdmcp - #Xtst - #xcb - #plds4 - #Xext - #expat - dl - #nss3 - #nssutil3 - #plc4 - #nspr4 - #rt - Threads::Threads - ) - - if (USE_JETSON_ENCODER) - target_sources(sora - PRIVATE - src/hwenc_jetson/jetson_buffer.cpp - src/hwenc_jetson/jetson_jpeg_decoder.cpp - src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp - src/hwenc_jetson/jetson_v4l2_capturer.cpp - src/hwenc_jetson/jetson_video_encoder.cpp - src/hwenc_jetson/jetson_video_decoder.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvBufSurface.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvBuffer.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvElement.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvElementProfiler.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvJpegDecoder.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvJpegEncoder.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvLogging.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvV4l2Element.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvV4l2ElementPlane.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvVideoDecoder.cpp - ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvVideoEncoder.cpp - ) - - target_include_directories(sora PRIVATE ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/include) - target_include_directories(sora PRIVATE ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/include/libjpeg-8b) - target_link_directories(sora PRIVATE ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/tegra) - target_link_options(sora - PRIVATE - "-Wl,-rpath-link=${CMAKE_SYSROOT}/lib/aarch64-linux-gnu" - "-Wl,-rpath-link=${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu" - "-Wl,-rpath-link=${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/tegra" - ) - target_link_libraries(sora - PUBLIC - nvv4l2 - nvv4lconvert - #nvbuf_fdmap - #nvddk_vic - #nvddk_2d_v2 - nvjpeg - nvbufsurface - nvbufsurftransform - - # nvbuf_utils を NvUtils に移行した際、ドキュメントには libnvbuf_utils.so を参照するように記載があったが、 - # そのような so ファイルは存在しないためリンクが失敗した - # nvbuf_fdmap を追加したらリンクが通った - # https://developer.nvidia.com/sites/default/files/akamai/embedded/nvbuf_utils_to_nvutils_migration_guide.pdf - nvbuf_fdmap - #nvos - ) - endif(USE_JETSON_ENCODER) endif() # 静的ライブラリを sora.lib に含める diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c65f0942..404d06e7 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -15,13 +15,13 @@ ```bash # ../webrtc-build に shiguredo-webrtc-build/webrtc-build がある場合 -python3 run.py ubuntu-20.04_x86_64 --webrtc-build-dir ../webrtc-build +python3 run.py ubuntu-20.04_x86_64 --local-webrtc-build-dir ../webrtc-build ``` webrtc-build に引数を渡してビルドする場合、以下のようにする。 ```bash -python3 run.py ubuntu-20.04_x86_64 --webrtc-build-dir ../webrtc-build --webrtc-build-args='--webrtc-fetch' +python3 run.py ubuntu-20.04_x86_64 --local-webrtc-build-dir ../webrtc-build --local-webrtc-build-args='--webrtc-fetch' ``` この時、VERSION に指定している WEBRTC_BUILD_VERSION に関係なく、現在 webrtc-build リポジトリでチェックアウトされている内容でビルドするため、バージョンの不整合に注意すること。 @@ -32,7 +32,7 @@ C++ SDK をデバッグビルドするには、libwebrtc も含めて、依存 しかし libwebrtc のバイナリはリリースビルドであるため、libwebrtc のデバッグバイナリを作るにはローカルの webrtc-build を利用する必要がある。 ```bash -python3 run.py ubuntu-20.04_x86_64 --debug --webrtc-build-dir ../webrtc-build +python3 run.py ubuntu-20.04_x86_64 --debug --local-webrtc-build-dir ../webrtc-build ``` このように `--debug` を付けると、C++ SDK だけでなく、ローカルの webrtc-build を含む全ての依存ライブラリもデバッグビルドを行う。 diff --git a/README.md b/README.md index be5aadd2..c2272579 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,11 @@ We will not respond to PRs or issues that have not been discussed on Discord. Also, Discord is only available in Japanese. -Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use. +Please read before use. ## 時雨堂のオープンソースソフトウェアについて -利用前に https://github.com/shiguredo/oss をお読みください。 +利用前に をお読みください。 ## Sora C++ SDK について @@ -21,16 +21,18 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use - 各プラットフォームで利用可能な HWA への対応 - NVIDIA Video Codec SDK (NVENC / NVDEC) - - VP9 / H.264 + - VP9 / H.264 / H.265 - NVIDIA Jetson Video HWA - - VP9 / AV1 / H.264 + - VP9 / AV1 / H.264 / H.265 - Apple macOS / iOS Video Toolbox - H.264 / H.265 - Google Android HWA - VP8 / VP9 / H.264 / H.265 - [Intel VPL](https://github.com/intel/libvpl) (Intel Media SDK の後継) - - VP9 / H.264 / H.265 - - H.265 は Ubuntu 22.04 のみ + - H.264 / H.265 / AV1 + - AV1 は デコードのみ + - H.265 は Windows と Ubuntu 22.04 のみ + - Media SDK での動作は未検証です ## ライブラリのバイナリ提供について @@ -38,7 +40,7 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use _hololens2 は無視してください_ -https://github.com/shiguredo/sora-cpp-sdk/releases + ## サンプル集 @@ -52,19 +54,32 @@ https://github.com/shiguredo/sora-cpp-sdk/releases - Windows 10.1809 x86_64 以降 - macOS 13.5 arm64 以降 -- Ubuntu 20.04 ARMv8 Jetson (JetPack 5.1.1 以降) +- Ubuntu 22.04 ARMv8 Jetson (JetPack 6.0.0 以降) - Jetson AGX Orin - - Jetson AGX Xavier - - Jetson Xavier NX + - Jetson Orin NX + - 動作未検証です - Ubuntu 20.04 x86_64 + - 動作未検証です - Ubuntu 22.04 x86_64 +- Ubuntu 24.04 x86_64 + - 動作未検証です - Android 7 arm64 以降 - iOS 13 arm64 以降 +## 既知の問題 + +[known_issues.md](doc/known_issues.md) をお読みください。 + ## FAQ [faq.md](doc/faq.md) をお読みください。 +## メンテナンスポリシー + +Sora C++ SDK のメンテナンスポリシーにはプライオリティがあります。 + +詳細については [maintenance_policy.md](doc/maintenance_policy.md) をお読みください。 + ## 優先実装 優先実装とは Sora のライセンスを契約頂いているお客様限定で Sora C++ SDK の実装予定機能を有償にて前倒しで実装することです。 @@ -72,15 +87,16 @@ https://github.com/shiguredo/sora-cpp-sdk/releases - Intel VPL H.265 対応 - [アダワープジャパン株式会社](https://adawarp.com/) 様 -### 優先実装が可能な機能一覧 +### 優先実装が可能な対応一覧 **詳細は Discord やメールなどでお気軽にお問い合わせください** +- NVIDIA Jetson JetPack 5 対応 +- NVIDIA Jetson JetPack 6 Jetson Orin Nano 対応 +- Raspberry Pi OS (64bit) arm64 対応 +- Windows arm64 対応 - Ubuntu 24.04 x86_64 -- Ubuntu 22.04 arm64 対応 - - NVIDIA Jetson JetPack 6 対応 - AMD 系 HWA 対応 -- Windows arm64 対応 ## サポートについて @@ -92,7 +108,7 @@ https://github.com/shiguredo/sora-cpp-sdk/releases 最新の状況などは Discord で共有しています。質問や相談も Discord でのみ受け付けています。 -https://discord.gg/shiguredo + ### バグ報告 @@ -121,7 +137,7 @@ limitations under the License. ## OpenH264 -https://www.openh264.org/BINARY_LICENSE.txt + ``` "OpenH264 Video Codec provided by Cisco Systems, Inc." diff --git a/VERSION b/VERSION index e778cc98..b443bc95 100644 --- a/VERSION +++ b/VERSION @@ -1,7 +1,7 @@ -SORA_CPP_SDK_VERSION=2024.6.1 -WEBRTC_BUILD_VERSION=m122.6261.1.0 -BOOST_VERSION=1.84.0 -CMAKE_VERSION=3.28.1 +SORA_CPP_SDK_VERSION=2024.7.0 +WEBRTC_BUILD_VERSION=m127.6533.1.1 +BOOST_VERSION=1.85.0 +CMAKE_VERSION=3.29.6 CUDA_VERSION=11.8.0-1 ANDROID_NDK_VERSION=r26b ANDROID_NATIVE_API_LEVEL=29 diff --git a/buildbase.py b/buildbase.py index 2bf0e9c5..b91e070b 100644 --- a/buildbase.py +++ b/buildbase.py @@ -433,19 +433,19 @@ def install_webrtc(version, source_dir, install_dir, platform: str): extract(archive, output_dir=install_dir, output_dirname="webrtc") -def build_webrtc(platform, webrtc_build_dir, webrtc_build_args, debug): - with cd(webrtc_build_dir): +def build_webrtc(platform, local_webrtc_build_dir, local_webrtc_build_args, debug): + with cd(local_webrtc_build_dir): args = ["--webrtc-nobuild-ios-framework", "--webrtc-nobuild-android-aar"] if debug: args += ["--debug"] - args += webrtc_build_args + args += local_webrtc_build_args cmd(["python3", "run.py", "build", platform, *args]) # インクルードディレクトリを増やしたくないので、 # __config_site を libc++ のディレクトリにコピーしておく - webrtc_source_dir = os.path.join(webrtc_build_dir, "_source", platform, "webrtc") + webrtc_source_dir = os.path.join(local_webrtc_build_dir, "_source", platform, "webrtc") src_config = os.path.join( webrtc_source_dir, "src", "buildtools", "third_party", "libc++", "__config_site" ) @@ -486,11 +486,11 @@ class WebrtcInfo(NamedTuple): def get_webrtc_info( - platform: str, webrtc_build_dir: Optional[str], install_dir: str, debug: bool + platform: str, local_webrtc_build_dir: Optional[str], install_dir: str, debug: bool ) -> WebrtcInfo: webrtc_install_dir = os.path.join(install_dir, "webrtc") - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: return WebrtcInfo( version_file=os.path.join(webrtc_install_dir, "VERSIONS"), deps_file=os.path.join(webrtc_install_dir, "DEPS"), @@ -501,15 +501,17 @@ def get_webrtc_info( libcxx_dir=os.path.join(install_dir, "llvm", "libcxx"), ) else: - webrtc_build_source_dir = os.path.join(webrtc_build_dir, "_source", platform, "webrtc") + webrtc_build_source_dir = os.path.join( + local_webrtc_build_dir, "_source", platform, "webrtc" + ) configuration = "debug" if debug else "release" webrtc_build_build_dir = os.path.join( - webrtc_build_dir, "_build", platform, configuration, "webrtc" + local_webrtc_build_dir, "_build", platform, configuration, "webrtc" ) return WebrtcInfo( - version_file=os.path.join(webrtc_build_dir, "VERSION"), - deps_file=os.path.join(webrtc_build_dir, "DEPS"), + version_file=os.path.join(local_webrtc_build_dir, "VERSION"), + deps_file=os.path.join(local_webrtc_build_dir, "DEPS"), webrtc_include_dir=os.path.join(webrtc_build_source_dir, "src"), webrtc_source_dir=os.path.join(webrtc_build_source_dir, "src"), webrtc_library_dir=webrtc_build_build_dir, @@ -535,6 +537,101 @@ def install_boost(version, source_dir, install_dir, sora_version, platform: str) extract(archive, output_dir=install_dir, output_dirname="boost") +# 以下の問題を解決するためのパッチ +# +# No support for msvc-toolset 14.4x (VS 2022, 17.10.x): https://github.com/boostorg/boost/issues/914 +BOOST_PATCH_SUPPORT_14_4 = r""" +diff --git a/tools/build/src/engine/config_toolset.bat b/tools/build/src/engine/config_toolset.bat +index 4ba577cac..3e3f6a3a1 100644 +--- a/tools/build/src/engine/config_toolset.bat ++++ b/tools/build/src/engine/config_toolset.bat +@@ -157,7 +157,7 @@ pushd %CD% + if "_%VSINSTALLDIR%_" == "__" call :Call_If_Exists "%B2_TOOLSET_ROOT%Auxiliary\Build\vcvarsall.bat" %B2_BUILD_ARGS% + popd + @REM set "B2_CXX="%CXX%" /nologo /MP /MT /TP /Feb2 /wd4996 /O2 /GL /EHsc" +-set "B2_CXX="%CXX%" /nologo -TP /wd4996 /wd4675 /EHs /GR /Zc:throwingNew /O2 /Ob2 /W3 /MD /Zc:forScope /Zc:wchar_t /Zc:inline /Gw /favor:blend /Feb2" ++set "B2_CXX="%CXX%" /nologo -TP /wd4996 /wd4675 /EHs /GR /Zc:throwingNew /O2 /Ob2 /W3 /MT /Zc:forScope /Zc:wchar_t /Zc:inline /Gw /favor:blend /Feb2" + set "B2_CXX_LINK=/link kernel32.lib advapi32.lib user32.lib" + set "_known_=1" + goto :Embed_Minafest_Via_Link +diff --git a/tools/build/src/tools/msvc.jam b/tools/build/src/tools/msvc.jam +index 54a6ced32..4bb3810b3 100644 +--- a/tools/build/src/tools/msvc.jam ++++ b/tools/build/src/tools/msvc.jam +@@ -1137,7 +1137,15 @@ local rule generate-setup-cmd ( version : command : parent : options * : cpu : g + } + else + { +- if [ MATCH "(14.3)" : $(version) ] ++ if [ MATCH "(14.4)" : $(version) ] ++ { ++ if $(.debug-configuration) ++ { ++ ECHO "notice: [generate-setup-cmd] $(version) is 14.4" ; ++ } ++ parent = [ path.native [ path.join $(parent) "..\\..\\..\\..\\..\\Auxiliary\\Build" ] ] ; ++ } ++ else if [ MATCH "(14.3)" : $(version) ] + { + if $(.debug-configuration) + { +@@ -1316,7 +1324,11 @@ local rule configure-really ( version ? : options * ) + # version from the path. + # FIXME: We currently detect both Microsoft Visual Studio 9.0 and + # 9.0express as 9.0 here. +- if [ MATCH "(MSVC\\\\14.3)" : $(command) ] ++ if [ MATCH "(MSVC\\\\14.4)" : $(command) ] ++ { ++ version = 14.4 ; ++ } ++ else if [ MATCH "(MSVC\\\\14.3)" : $(command) ] + { + version = 14.3 ; + } +@@ -1745,13 +1757,17 @@ local rule default-path ( version ) + # And fortunately, forward slashes do also work in native Windows. + local vswhere = "$(root)/Microsoft Visual Studio/Installer/vswhere.exe" ; + # The check for $(root) is to avoid a segmentation fault if not found. +- if $(version) in 14.1 14.2 14.3 default && $(root) && [ path.exists $(vswhere) ] ++ if $(version) in 14.1 14.2 14.3 14.4 default && $(root) && [ path.exists $(vswhere) ] + { + local req = "-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64" ; + local prop = "-property installationPath" ; + local limit ; + +- if $(version) = 14.3 ++ if $(version) = 14.4 ++ { ++ limit = "-version \"[17.0,18.0)\" -prerelease" ; ++ } ++ else if $(version) = 14.3 + { + limit = "-version \"[17.0,18.0)\" -prerelease" ; + } +@@ -2174,7 +2190,7 @@ for local arch in [ MATCH "^\\.cpus-on-(.*)" : [ VARNAMES $(__name__) ] ] + armv7 armv7s ; + + # Known toolset versions, in order of preference. +-.known-versions = 14.3 14.2 14.1 14.0 12.0 11.0 10.0 10.0express 9.0 9.0express 8.0 8.0express 7.1 ++.known-versions = 14.4 14.3 14.2 14.1 14.0 12.0 11.0 10.0 10.0express 9.0 9.0express 8.0 8.0express 7.1 + 7.1toolkit 7.0 6.0 ; + + # Version aliases. +@@ -2226,6 +2242,11 @@ for local arch in [ MATCH "^\\.cpus-on-(.*)" : [ VARNAMES $(__name__) ] ] + "Microsoft Visual Studio/2022/*/VC/Tools/MSVC/*/bin/Host*/*" + ; + .version-14.3-env = VS170COMNTOOLS ProgramFiles ProgramFiles(x86) ; ++.version-14.4-path = ++ "../../VC/Tools/MSVC/*/bin/Host*/*" ++ "Microsoft Visual Studio/2022/*/VC/Tools/MSVC/*/bin/Host*/*" ++ ; ++.version-14.4-env = VS170COMNTOOLS ProgramFiles ProgramFiles(x86) ; + + # Auto-detect all the available msvc installations on the system. + auto-detect-toolset-versions ; +""" + + @versioned def build_and_install_boost( version: str, @@ -564,6 +661,25 @@ def build_and_install_boost( b2 = "b2" if target_os == "windows" else "./b2" runtime_link = "static" if target_os == "windows" else "shared" + # Windows かつ Boost 1.85.0 の場合はパッチを当てる + if target_os == "windows" and version == "1.85.0": + directory = cmdcap(["git", "rev-parse", "--show-prefix"]) + cmd( + [ + "git", + "apply", + "-p1", + "--ignore-space-change", + "--ignore-whitespace", + "--whitespace=nowarn", + f"--directory={directory}", + "-", + ], + input=BOOST_PATCH_SUPPORT_14_4, + text=True, + encoding="utf-8", + ) + cmd([bootstrap]) if target_os == "iphone": @@ -749,15 +865,23 @@ def install_sora_and_deps(platform: str, source_dir: str, install_dir: str): def build_sora( - platform: str, sora_dir: str, sora_args: List[str], debug: bool, webrtc_build_dir: Optional[str] + platform: str, + local_sora_cpp_sdk_dir: str, + local_sora_cpp_sdk_args: List[str], + debug: bool, + local_webrtc_build_dir: Optional[str], ): - if debug and "--debug" not in sora_args: - sora_args = ["--debug", *sora_args] - if webrtc_build_dir is not None: - sora_args = ["--webrtc-build-dir", webrtc_build_dir, *sora_args] + if debug and "--debug" not in local_sora_cpp_sdk_args: + local_sora_cpp_sdk_args = ["--debug", *local_sora_cpp_sdk_args] + if local_webrtc_build_dir is not None: + local_sora_cpp_sdk_args = [ + "--local-webrtc-build-dir", + local_webrtc_build_dir, + *local_sora_cpp_sdk_args, + ] - with cd(sora_dir): - cmd(["python3", "run.py", platform, *sora_args]) + with cd(local_sora_cpp_sdk_dir): + cmd(["python3", "run.py", platform, *local_sora_cpp_sdk_args]) class SoraInfo(NamedTuple): @@ -766,11 +890,11 @@ class SoraInfo(NamedTuple): def get_sora_info( - platform: str, sora_dir: Optional[str], install_dir: str, debug: bool + platform: str, local_sora_cpp_sdk_dir: Optional[str], install_dir: str, debug: bool ) -> SoraInfo: - if sora_dir is not None: + if local_sora_cpp_sdk_dir is not None: configuration = "debug" if debug else "release" - install_dir = os.path.join(sora_dir, "_install", platform, configuration) + install_dir = os.path.join(local_sora_cpp_sdk_dir, "_install", platform, configuration) return SoraInfo( sora_install_dir=os.path.join(install_dir, "sora"), @@ -814,6 +938,16 @@ def install_rootfs(version, install_dir, conf): if os.path.exists(file) and not os.path.exists(link): os.symlink(os.path.basename(file), link) + # JetPack 6 から tegra → nvidia になった + link = os.path.join( + rootfs_dir, "usr", "lib", "aarch64-linux-gnu", "nvidia", "libnvbuf_fdmap.so" + ) + file = os.path.join( + rootfs_dir, "usr", "lib", "aarch64-linux-gnu", "nvidia", "libnvbuf_fdmap.so.1.0.0" + ) + if os.path.exists(file) and not os.path.exists(link): + os.symlink(os.path.basename(file), link) + @versioned def install_android_ndk(version, install_dir, source_dir): @@ -930,8 +1064,6 @@ def install_sdl2( ] if platform == "windows": cmake_args += [ - "-G", - "Visual Studio 16 2019", "-DSDL_FORCE_STATIC_VCRT=ON", "-DHAVE_LIBC=ON", ] @@ -1497,32 +1629,32 @@ def get_webrtc_platform(platform: Platform) -> str: # `--sora-args '--test'` のようにスペースを使うと、ハイフンから始まるオプションが正しく解釈されない def add_sora_arguments(parser): parser.add_argument( - "--sora-dir", + "--local-sora-cpp-sdk-dir", type=os.path.abspath, default=None, help="Refer to local Sora C++ SDK. " "When this option is specified, Sora C++ SDK will also be built.", ) parser.add_argument( - "--sora-args", + "--local-sora-cpp-sdk-args", type=shlex.split, default=[], - help="Options for building local Sora C++ SDK when `--sora-dir` is specified.", + help="Options for building local Sora C++ SDK when `--local-sora-cpp-sdk-dir` is specified.", ) # add_sora_arguments と同様の注意点があるので注意すること def add_webrtc_build_arguments(parser): parser.add_argument( - "--webrtc-build-dir", + "--local-webrtc-build-dir", type=os.path.abspath, default=None, help="Refer to local webrtc-build. " "When this option is specified, webrtc-build will also be built.", ) parser.add_argument( - "--webrtc-build-args", + "--local-webrtc-build-args", type=shlex.split, default=[], - help="Options for building local webrtc-build when `--webrtc-build-dir` is specified.", + help="Options for building local webrtc-build when `--local-webrtc-build-dir` is specified.", ) diff --git a/doc/build.md b/doc/build.md new file mode 100644 index 00000000..ada98127 --- /dev/null +++ b/doc/build.md @@ -0,0 +1,40 @@ +# ビルド + +様々なプラットフォームへ対応しているため、ビルドがとても複雑です。 + +ビルド関連の質問については環境問題がほとんどであり、環境問題を改善するコストがとても高いため基本的には解答しません。 + +github actions でビルドを行い確認していますので、まずは github actions の [build.yml](https://github.com/shiguredo/sora-cpp-sdk/blob/develop/.github/workflows/build.yml) を確認してみてください。 + +github actions のビルドが失敗していたり、 +ビルド済みバイナリがうまく動作しない場合は discord へご連絡ください。 + +## サンプルのビルド + +> [!important] +> ここでは macos arm64 でのビルドを想定しています + +サンプルをビルドする際、二つのパターンでビルドすることができます。 + +- github に公開されているバイナリの sdk を利用する +- ローカルでビルドした sdk を利用する + +ここでは一通りの機能を実装しているサンプルである sumomo を例にします。 +また、ビルド用の `run.py` はプラットフォーム毎に用意されています。 + +### github に公開されているバイナリの sdk を利用する + +```bash +python3 examples/sumomo/macos_arm64/run.py +``` + +### ローカルでビルドした sdk を利用する + +```bash +python3 examples/sumomo/macos_arm64/run.py --local-sora-cpp-sdk-dir . +``` + +ローカルでビルドした sdk のルートディレクトリを `--local-sora-cpp-sdk-dir` にて指定してください。 + +> [!note] +> `--local-sora-cpp-sdk-dir` を指定した際は `examples/VERSION` に定義されている `SORA_CPP_SDK_VERSION` と `BOOST_VERSION` の値は利用されません。 diff --git a/doc/faq.md b/doc/faq.md index b38a393c..82e0dad0 100644 --- a/doc/faq.md +++ b/doc/faq.md @@ -2,12 +2,14 @@ ## ドキュメント -Sora C++ SDK にドキュメントはありません。基本的にはサンプルを参考にしてください。 +Sora C++ SDK にドキュメントはありません。サンプルを参考にしてください。 -また、不明点などあれば Discord でお問い合わせください。 +不明点などあれば時雨堂の Discord の `#sora-sdk-faq` にてご相談ください。 ## ビルド +様々なプラットフォームへ対応しているため、ビルドがとても複雑です。 + ビルド関連の質問については環境問題がほとんどであり、環境問題を改善するコストがとても高いため基本的には解答しません。 GitHub Actions でビルドを行い確認していますので、まずは GitHub Actions の [build.yml](https://github.com/shiguredo/sora-cpp-sdk/blob/develop/.github/workflows/build.yml) を確認してみてください。 @@ -27,9 +29,10 @@ NVIDIA VIDEO CODEC SDK のハードウェアデコーダでは width height の Windows 環境でのみ Jetson の H.264 映像を受信すると映像の色が緑色になってしまうことを確認しています。こちらの事象は NVIDIA Jetson の JetPack を 5.1.1 にすることで解決します。 -## iOS または macOS から H.264 の FHD で配信するにはどうすればいいですか? +## iOS または macOS から H.264 の 1080p で配信するにはどうすればいいですか? -iOS または macOS から FHD で配信したい場合は Sora の H.264 のプロファイルレベル ID を 5.2 以上に設定してください。設定は以下の方法で行うことができます。 +iOS または macOS から 1080p で配信したい場合は Sora の H.264 のプロファイルレベル ID を 5.2 以上に設定してください。 +設定は以下の方法で行うことができます。 - `SoraSignalingConfig` に `video_h264_params` を設定する - Sora の設定を変更する @@ -38,19 +41,13 @@ Sora の設定については [Sora のドキュメント](https://sora-doc.shig ## 4K@30fps で映像を配信できません -Sora C++ SDK では、 `SoraVideoEncoderFactoryConfig` という構造体に `force_i420_conversion_for_simulcast_adapter` というフラグがあり、デフォルトで true になっています。 -このフラグを false にすることで、パフォーマンスが向上して 4K@30fps で映像を配信できる可能性がありますが、 JetsonBuffer を利用している環境などでサイマルキャストが正常に動作しなくなります。 - -このフラグが必要になった背景をコードの [コメント](https://github.com/shiguredo/sora-cpp-sdk/blob/8f6dba9218e0cda7cdefafe64a37c1af9d5e5c9e/include/sora/sora_video_encoder_factory.h#L57-L71) から以下に抜粋します。 - -``` -このフラグが必要になった背景は以下の通り -- サイマルキャスト時、 JetsonBuffer のような一部の kNative なバッファーの実装において、バッファーを複数回読み込めないという制限があるため、 I420 への変換が必要になる - - サイマルキャストは複数ストリームを送信するため、バッファーを複数回読みこむ必要がある -- サイマルキャスト時は use_simulcast_adapter = true にしてサイマルキャストアダプタを利用する必要があるが、 SoraClientContext の実装ではサイマルキャスト時でも非サイマルキャスト時でも常に use_simulcast_adapter = true として SoraVideoEncoderFactory を生成している - - Sora に type: connect で simulcast 有効/無効を指定して接続しても、Sora 接続後に受信する type: offer でサイマルキャストの有効/無効が上書きできるため、SoraClientContext 生成時に use_simulcast_adapter の値をどうするべきか決定できない。そのため常に use_simulcast_adapter = true にしておくのが安全となる - - 非サイマルキャスト時に use_simulcast_adapter = true とするのは、パフォーマンスの問題はあっても動作に影響は無い -- 上記の2つの問題によって、非サイマルキャスト時でも I420 への変換処理が走ることになって、解像度や性能によってはフレームレートが出ないことがある -- この I420 への変換は、 Sora の設定も含めて利用者が非サイマルキャストだと保証できる場合、あるいはサイマルキャストであっても複数回読める kNative なバッファーを利用している場合には不要な処理になる -- そのような場合に I420 への変換を無効にすることで十分なフレームレートを確保できるようにするために、このフラグが必要になった -``` \ No newline at end of file +Sora C++ SDK は 4K@30fps での映像の配信に対応していますが、 +一部の環境では 4K@30fps で映像を配信できない場合があります。 + +その場合、 `SoraVideoEncoderFactoryConfig` という構造体の `force_i420_conversion_for_simulcast_adapter` フラグを `false` にすることで、4K@30fps で映像を配信できる場合があります。 + +かなり内部的な話なので詳細については [コードのコメント](https://github.com/shiguredo/sora-cpp-sdk/blob/8f6dba9218e0cda7cdefafe64a37c1af9d5e5c9e/include/sora/sora_video_encoder_factory.h#L57-L71) をご確認ください。 + +## Intel VPL を使ってみたい + +Intel VPL の使い方については[Intel VPL](doc/vpl.md) をご確認ください。 diff --git a/doc/known_issues.md b/doc/known_issues.md index 2dbf7f75..285fe8d7 100644 --- a/doc/known_issues.md +++ b/doc/known_issues.md @@ -4,8 +4,11 @@ Intel VPL で H.264 の映像をデコードすると、以下の事象が発生することを確認しています。 -- 受信した映像に遅延が発生する - 受信した映像が止まる -- 長時間デコードを行うと、セグメンテーション・フォルトが発生してプロセスがクラッシュする + +調査の結果、IntelVPL に問題がある可能性が高く、現在問題報告を行なって対応をしております。 +https://community.intel.com/t5/Media-Intel-Video-Processing/Issue-with-Decoding-H-264-Video-Encoded-in-macOS-Safari/m-p/1587647 + +こちらの問題が解消すると改善する見込みです。 Intel VPL を利用して映像をデコードする場合は、 H.264 を避けて他のコーデックを利用してください。 \ No newline at end of file diff --git a/doc/maintenance_policy.md b/doc/maintenance_policy.md new file mode 100644 index 00000000..38ea10c1 --- /dev/null +++ b/doc/maintenance_policy.md @@ -0,0 +1,104 @@ +# メンテナンスポリシー + +Sora C++ SDK ではメンテナンスポリシーにはプライオリティがあります。 + +プライオリティは P1 から P4 まであります。 +P1 が一番プライオリティが高いです。 + +## プライオリティ + +- P1 と P2 は ``develop`` ブランチで開発します +- P1 と P2 ``main`` ブランチでリリースされます +- P1 と P2 ``main`` ブランチでタグを打ちます +- P1 と P2 は [Sora Unity SDK](https://github.com/shiguredo/sora-unity-sdk) で利用できます +- P1 と P2 は [Sora Python SDK](https://github.com/shiguredo/sora-python-sdk) で利用できます +- P1 は最新の OS や HWA への対応が C++ SDK のリリースのトリガーになる場合があります +- P2 は最新の OS や HWA への対応が C++ SDK のリリースのトリガーになりません +- P3 と P4 は C++ SDK メジャーリリースには含まれません +- P3 と P4 は ``support/`` ブランチでタグを打ちます +- P1 と P2 と P3 は Sora のメジャーリリースに追従します +- P4 は優先実装でのみ ``main`` ブランチへ追従します +- P4 は優先実装でのみアップデートを行います + +## 独自タグについて + +- P1 と P2 以外はそれぞれで独自タグを利用します +- タグの一番最後はリリース回数 +- リリース回数以外の文字列が変化したらリリース回数は 0 にもどす +- `{sora-cpp-sdk-version}-{platform-name}-{platform-version}.{release}` +- `2024.1.0-jetson-jetpack-5.1.3.0` +- `2024.1.0-jetson-jetpack-5.1.3.1` +- `2024.1.0-jetson-jetpack-5.1.3.2` +- `2024.1.0-jetson-jetpack-5.1.4.0` +- `2024.1.0-jetson-jetpack-5.1.4.1` +- `2024.1.0-jetson-jetpack-5.1.4.2` +- `2024.1.1-jetson-jetpack-5.1.4.0` +- `2024.1.1-jetson-jetpack-5.1.4.1` + +## P1 + +- Apple macOS / iOS / iPadOS +- Google Android + +### 方針 + +- `develop` ブランチで開発 +- `main` ブランチでリリース +- 最新の OS がリリースされたタイミングで対応 +- 最新のハードウェアアクセラレーターに対応 +- Sora メジャーリリースへ追従 +- リリースのタイミングで検証 + +## P2 + +- Intel VPL x86_64 + - Ubuntu 24.04 + - Ubuntu 22.04 + - Windows 11 +- NVIDIA Video Codec SDK x86_64 + - Ubuntu 24.04 + - Ubuntu 22.04 + - Windows 11 + +### 方針 + +- `develop` ブランチで開発 +- `main` ブランチでリリース +- 最新の OS がリリースされたタイミングでは対応しない +- 最新のハードウェアアクセラレーターに対応しない +- Sora メジャーリリースへの追従 +- リリースのタイミングでの検証は最低限 + +## P3 + +- NVIDIA Jetson JetPack 6 arm64 + - ブランチ: `support/jetson-jetpack-6` + - タグ: `{sora-cpp-sdk-version}-jetson-jetpack-{platform-version}.{release}` + - `2024.1.0-jetson-jetpack-6.0.0.0` +- Microsoft HoloLens 2 + - ブランチ: `support/hololens2` + - タグ: `{sora-cpp-sdk-version}-hololens2.{release}` + - `2024.1.0-hololens2.0` + +### 方針 + +- ``support`` ブランチで開発 +- 独自タグでリリースされる +- Sora メジャーリリースに追従する +- リリースのタイミングでの検証は最低限 + +## P4 + +- NVIDIA Jetson JetPack 5 arm64 + - ブランチ: `support/jetson-jetpack-5` + - タグ: `{sora-cpp-sdk-version}-jetson-jetpack-{platform-version}.{release}` + - `2024.1.0-jetson-jetpack-5.1.3.0` + +### 方針 + +- ``support`` ブランチで開発 +- 独自タグでリリースされる +- Sora メジャーリリースに追従しない +- リリースのタイミングでの検証は最低限 +- 優先実装でのみ Sora メジャーリリースへ追従 +- 優先実装でのみアップデートを行う diff --git a/doc/vpl.md b/doc/vpl.md index 0187703c..09676686 100644 --- a/doc/vpl.md +++ b/doc/vpl.md @@ -14,7 +14,7 @@ Intel VPL には Intel VPL (ライブラリの Intel VPL と区別するため Intel Media SDK は既に開発が終了しており、後継の Intel VPL ランタイムに開発が移行しているため、 これから VPL を利用する場合は、 Intel VPL ランタイムに対応したチップを利用することを推奨します。 -https://www.intel.com/content/www/us/en/developer/tools/vpl/overview.html#gs.73uoi4 の Specifications のセクションより、ランタイムと対応するチップの一覧を以下に引用します。 + の Specifications のセクションより、ランタイムと対応するチップの一覧を以下に引用します。 - [Intel VPL](https://github.com/oneapi-src/oneVPL-intel-gpu) - Intel® Iris® Xe graphics @@ -26,43 +26,47 @@ https://www.intel.com/content/www/us/en/developer/tools/vpl/overview.html#gs.73u - Intel® Server GPU - 5th to 11th generation Intel Core processors using integrated graphics -## NVIDIA の GPU が搭載された PC で Intel VPL を利用する方法 - -Sora C++ SDK の 実装 (SoraVideoEncoderFactory, SoraVideoDecoderFactory クラス) では、 -NVIDIA の GPU を利用するエンコーダー/デコーダーの優先度が Intel VPL を利用するものより高くなっています。 - -そのため、 NVIDIA の GPU が搭載された PC で Intel VPL を利用するには、以下のいずれかの対応が必要です。 - -- NVIDIA の GPU を利用するエンコーダー/デコーダーをビルド時に無効化する ... ビルド・スクリプトで `USE_NVCODEC_ENCODER` を指定している箇所を削除する -- NVIDIA の GPU のドライバーを削除する - -Sora C++ SDK をビルドしている場合は、前者の方法を推奨します。 -また、 GPU のドライバーを削除する場合は自己責任で行ってください。 - ## 環境構築 ### Windows -手動でドライバーのインストールなどを行う必要はありません。 +Intel VPL を利用するためには Intel のディスプレイドライバーがインストールされている必要があります。 +Windows では環境によってドライバーの有無が異なるため、以下のいずれかの手順でインストールされているドライバーを確認してください。 + +- デバイスマネージャー > ディスプレイアダプター > インストールされているドライバーを確認 +- Win + R キーを押下 > `ファイル名を指定して実行` のダイアログが開くので `dxdiag` と入力して `OK` ボタンを押下 > DirectX 診断ツールが起動するので、 `ディスプレイ` のタブからインストールされているドライバーを確認 + +Intel のドライバーが確認できない場合は、以下のページから適切なドライバーをインストールしてください。 + ### Ubuntu 22.04 -https://dgpu-docs.intel.com/driver/client/overview.html を参考に必要なドライバーとソフトウェアをインストールします。 +Ubuntu 22.04 で Intel VPL を利用するためには、ドライバーとライブラリをインストールする必要があります。 +公式ドキュメントの を参考に必要なドライバーとライブラリをインストールします。 + +#### Intel VPL ランタイムをインストールする -#### Intel VPL ランタイムを利用する手順 +##### Intel の apt リポジトリを追加 -Intel VPL ランタイムを利用する場合は libmfxgen1 ではなく libmfx-gen1.2 を使う必要があるため、ドキュメントのコマンドを一部読み替えて実行します。 +パッケージのインストールには Intel の apt リポジトリを追加する必要があります。 ```bash -# Intel の apt リポジトリを追加 -$ wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \ + +wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \ sudo gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg -$ echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | \ +echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | \ sudo tee /etc/apt/sources.list.d/intel-gpu-jammy.list -$ sudo apt updatっg +sudo apt update +``` + +##### パッケージのインストール -# パッケージのインストール -$ sudo apt install -y \ +公式ドキュメントでは libmfxgen1 をインストールする手順が記載されていますが、Intel VPL ランタイムを使用するには libmfx-gen1.2 が必要です。 + +以下の実行例のように、 libmfx-gen1.2 をインストールしてください。 + +```bash +sudo apt install -y \ intel-opencl-icd intel-level-zero-gpu level-zero \ intel-media-va-driver-non-free libmfx1 libmfx-gen1.2 libvpl2 \ libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \ @@ -70,20 +74,30 @@ $ sudo apt install -y \ mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo ``` +##### 再起動 + +パッケージのインストールが完了したら、再起動してください。 + #### Intel Media SDK を利用する手順 -ドキュメントの通りです。 +Intel のチップセットの世代によって、 Intel Media SDK を利用する必要がある場合があります。 + +以下の手順で Intel Media SDK をインストールしてください。 + +##### Intel の apt リポジトリを追加 ```bash -# Intel の apt リポジトリを追加 -$ wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \ +wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \ sudo gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg -$ echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | \ +echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | \ sudo tee /etc/apt/sources.list.d/intel-gpu-jammy.list -$ sudo apt update +sudo apt update +``` + +###### パッケージのインストール -# パッケージのインストール -$ sudo apt install -y \ +```bash +sudo apt install -y \ intel-opencl-icd intel-level-zero-gpu level-zero \ intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \ libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \ @@ -99,7 +113,7 @@ $ sudo apt install -y \ 以下は `vainfo` を実行した出力の例です。 対応しているプロファイルやエントリーポイントは環境によって異なります。 -``` +```console $ vainfo Trying display: wayland libva info: VA-API version 1.20.0 diff --git a/examples/OLD_CHANGES.md b/examples/OLD_CHANGES.md deleted file mode 100644 index 8da1b6a5..00000000 --- a/examples/OLD_CHANGES.md +++ /dev/null @@ -1,149 +0,0 @@ -**このファイルは古いバージョンの変更履歴です。最新の変更履歴は [CHANGES.md](../CHANGES.md) を参照してください。** - -# 変更履歴 - -- CHANGE - - 下位互換のない変更 -- UPDATE - - 下位互換がある変更 -- ADD - - 下位互換がある追加 -- FIX - - バグ修正 - -## develop - -- [CHANGE] Lyra を削除する - - @enm10k -- [ADD] 各 run.py に --sora-dir と --sora-args オプションを追加する - - @enm10k -- [ADD] 各 run.py に --webrtc-build-dir と --webrtc-build-args オプションを追加する - - @melpon -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2024.3.1 にあげる - - @enm10k -- [FIX] sumomo がデバッグ・ビルドで動作しない問題を修正する - - @enm10k - -## sora-cpp-sdk-2024.2.0 - -- [CHANGE] momo_sample を sumomo にリネームする - - @enm10k -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2024.2.0 にあげる - - WEBRTC_BUILD_VERSION を m121.6167.3.0 にあげる - - BOOST_VERSION を 1.84.0 にあげる - - LYRA_VERSION を 1.3.2 にあげる - - CMAKE_VERSION を 3.28.1 にあげる - - SDL2_VERSION を 2.30.0 にあげる - - CLI11_VERSION を v2.4.1 にあげる - - @enm10k -- [FIX] sdl_sample で `--audio-codec-lyra-bitrate` が未指定の時に不定な値が送信されるのを修正する - - SDLSampleConfig.audio_codec_lyra_bitrate の値が初期化されておらず、未指定の時に不定な値が送信されるようになっていた - - SDLSampleConfig.audio_codec_lyra_bitrate の初期値を 0 に設定する - - 値が 0 の時、Sora に値は送信されない - - @miosakuma - -## sora-cpp-sdk-2024.1.0 - -- [CHANGE] JetPack のバージョンを 5.1.2 にあげる - - JetPack 5.1.1, 5.1.2 で動作を確認 - - JetPack 5.1 では、互換性の問題で JetsonJpegDecoder がエラーになることを確認 - - @enm10k -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2024.1.0 にあげる - - @miosakuma - -## sora-cpp-sdk-2023.17.0 - -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2023.17.0 にあげる - - WEBRTC_BUILD_VERSION を m120.6099.1.2 にあげる - - @miosakuma -- [UPDATE] - - momo_sample と sdl_sample のビデオコーデックに H265 を指定可能にする - - @miosakuma - -## sora-cpp-sdk-2023.15.0 - -- [UPDATE] - - momo_sample に --hw-mjpeg-decoder オプションを追加する - - @enm10k -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2023.15.1 にあげる - - WEBRTC_BUILD_VERSION を m119.6045.2.1 にあげる - - CMake を 3.27.7 に上げる - - @miosakuma - -## sora-cpp-sdk-2023.14.0 - -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2023.14.0 にあげる - - WEBRTC_BUILD_VERSION を m117.5938.2.0 にあげる - - @torikizi - -## sora-cpp-sdk-2023.13.0 - -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2023.13.0 にあげる - - WEBRTC_BUILD_VERSION を m116.5845.6.1 にあげる - - @torikizi - -## sora-cpp-sdk-2023.9.0 - -- [UPDATE] Sora C++ SDK を 2023.9.0 にあげる - - @voluntas @miosakuma -- [UPDATE] WEBRTC_BUILD_VERSION を m115.5790.7.0 にあげる - - @miosakuma -- [UPDATE] SDL2_VERSION を 2.28.1 にあげる - - @voluntas - -## sora-cpp-sdk-2023.7.0 - -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2023.7.0 にあげる - - CMAKE_VERSION を 3.26.4 にあげる - - @torikizi - -## sora-cpp-sdk-2023.6.0 - -- [UPDATE] Sora C++ SDK を 2023.6.0 にあげる - - @miosakuma -- [UPDATE] WEBRTC_BUILD_VERSION を m114.5735.2.0 にあげる - - @miosakuma -- [ADD] Windows の momo_sample に --relwithdebinfo オプションを追加する - - @melpon - -## sora-cpp-sdk-2023.5.1 - -- [UPDATE] Sora C++ SDK を 2023.5.1 にあげる - - @miosakuma - -## sora-cpp-sdk-2023.5.0 - -- [CHANGE] SoraDefaultClient を削除して SoraClientContext を追加する - - @melpon -- [UPDATE] JetPack 5.1 に対応する - - @melpon -- [UPDATE] VERSION のライブラリをアップデートする - - SORA_CPP_SDK_VERSION を 2023.5.0 にあげる - - WEBRTC_BUILD_VERSION を m114.5735.0.1 にあげる - - BOOST_VERSION を 1.82.0 にあげる - - CMAKE_VERSION を 3.25.0 にあげる - - SDL2_VERSION を 2.26.5 にあげる - - CLI11_VERSION を 2.3.2 にあげる - - @miosakuma -- [UPDATE] CXX_STANDARD を 20 にあげる - - @miosakuma -- [ADD] デバイス一覧を取得する機能追加する - - @melpon - -## sora-cpp-sdk-2023.2.0 - -- [UPDATE] Sora C++ SDK を 2023.2.0 にあげる - - @miosakuma - -## sora-cpp-sdk-2023.1.0 - -- [UPDATE] Sora C++ SDK を 2023.1.0 にあげる - - @torikizi diff --git a/examples/README.md b/examples/README.md index b4ab8123..20044bf0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,14 +7,14 @@ examples 以下のディレクトリには、 Sora C++ SDK を利用したサン ### SDL サンプル WebRTC SFU Sora と映像の送受信を行い、[SDL (Simple DirectMedia Layer)](https://www.libsdl.org/) を利用して受信映像を表示するサンプルです。 -使い方は [SDL サンプルを使ってみる](./doc/USE_SDL_SAMPLE.md) をお読みください。 +使い方は [SDL サンプルを使ってみる](./sdl_sample/README.md) をお読みください。 -### Sumomo +### Sumomo [WebRTC Native Client Momo](https://github.com/shiguredo/momo) の sora モードを模したサンプルです。 -使い方は [Sumomo を使ってみる](./doc/USE_SUMOMO.md) をお読みください。 +使い方は [Sumomo を使ってみる](./sumomo/README.md) をお読みください。 ### メッセージング受信サンプル WebRTC SFU Sora の [メッセージング機能](https://sora-doc.shiguredo.jp/MESSAGING) を使って送信されたメッセージを受信するサンプルです。 -使い方は [メッセージング受信サンプルを使ってみる](./doc/USE_MESSAGING_RECVONLY_SAMPLE.md) をお読みください。 +使い方は [メッセージング受信サンプルを使ってみる](./messaging_recvonly_sample/README.md) をお読みください。 diff --git a/examples/VERSION b/examples/VERSION index 6a7dc4d9..520a1b4a 100644 --- a/examples/VERSION +++ b/examples/VERSION @@ -1,6 +1,6 @@ -SORA_CPP_SDK_VERSION=2024.6.1 -WEBRTC_BUILD_VERSION=m122.6261.1.0 -BOOST_VERSION=1.84.0 -CMAKE_VERSION=3.28.1 -SDL2_VERSION=2.30.0 -CLI11_VERSION=v2.4.1 +SORA_CPP_SDK_VERSION=2024.7.0 +WEBRTC_BUILD_VERSION=m127.6533.1.1 +BOOST_VERSION=1.85.0 +CMAKE_VERSION=3.29.6 +SDL2_VERSION=2.30.5 +CLI11_VERSION=v2.4.2 diff --git a/examples/doc/USE_MESSAGING_RECVONLY_SAMPLE.md b/examples/messaging_recvonly_sample/README.md similarity index 65% rename from examples/doc/USE_MESSAGING_RECVONLY_SAMPLE.md rename to examples/messaging_recvonly_sample/README.md index 01b8dc8a..5a8ae897 100644 --- a/examples/doc/USE_MESSAGING_RECVONLY_SAMPLE.md +++ b/examples/messaging_recvonly_sample/README.md @@ -16,14 +16,17 @@ **ビルドに関しての問い合わせは受け付けておりません。うまくいかない場合は [GitHub Actions](https://github.com/shiguredo/sora-cpp-sdk/blob/develop/.github/workflows/build.yml) の内容をご確認ください。** - ### リポジトリをクローンする [sora-cpp-sdk](https://github.com/shiguredo/sora-cpp-sdk) をクローンして、examples 以下のディレクトリを利用してください。 +develop ブランチは開発ブランチであり、ビルドが失敗することがあるため、 `main` またはリリースタグを指定するようにしてください。 + +以下は main ブランチを指定する例です。 + ```shell -$ git clone https://github.com/shiguredo/sora-cpp-sdk.git -$ cd sora-cpp-sdk/examples +git clone -b main https://github.com/shiguredo/sora-cpp-sdk.git +cd sora-cpp-sdk/examples ``` ### メッセージング受信サンプルをビルドする @@ -35,7 +38,7 @@ $ cd sora-cpp-sdk/examples 以下のツールを準備してください。 - [Visual Studio 2019](https://visualstudio.microsoft.com/ja/downloads/) - - C++ をビルドするためのコンポーネントを入れてください。 + - C++ をビルドするためのコンポーネントを入れてください。 - Python 3.10.5 ##### ビルド @@ -62,7 +65,7 @@ $ cd sora-cpp-sdk/examples ##### ビルド ```shell -$ python3 messaging_recvonly_sample/macos_arm64/run.py +python3 messaging_recvonly_sample/macos_arm64/run.py ``` 成功した場合、`_build/macos_arm64/release/messaging_recvonly_sample` に `messaging_recvonly_sample` が作成されます。 @@ -79,17 +82,17 @@ _build/macos_arm64/release/messaging_recvonly_sample 必要なパッケージをインストールしてください。 ```shell -$ sudo apt install libx11-dev -$ sudo apt install libdrm-dev -$ sudo apt install libva-dev -$ sudo apt install pkg-config -$ sudo apt install python3 +sudo apt install libx11-dev +sudo apt install libdrm-dev +sudo apt install libva-dev +sudo apt install pkg-config +sudo apt install python3 ``` ##### ビルド ```shell -$ python3 messaging_recvonly_sample/ubuntu-20.04_x86_64/run.py +python3 messaging_recvonly_sample/ubuntu-20.04_x86_64/run.py ``` 成功した場合、`_build/ubuntu-20.04_x86_64/release/messaging_recvonly_sample` に `messaging_recvonly_sample` が作成されます。 @@ -106,17 +109,17 @@ _build/ubuntu-20.04_x86_64/release/messaging_recvonly_sample/ 必要なパッケージをインストールしてください。 ```shell -$ sudo apt install libx11-dev -$ sudo apt install libdrm-dev -$ sudo apt install libva-dev -$ sudo apt install pkg-config -$ sudo apt install python3 +sudo apt install libx11-dev +sudo apt install libdrm-dev +sudo apt install libva-dev +sudo apt install pkg-config +sudo apt install python3 ``` ##### ビルド ```shell -$ python3 messaging_recvonly_sample/ubuntu-22.04_x86_64/run.py +python3 messaging_recvonly_sample/ubuntu-22.04_x86_64/run.py ``` 成功した場合、以下のファイルが作成されます。`_build/ubuntu-22.04_x86_64/release/messaging_recvonly_sample` に `messaging_recvonly_sample` が作成されます。 @@ -126,39 +129,6 @@ _build/ubuntu-22.04_x86_64/release/messaging_recvonly_sample/ └── messaging_recvonly_sample ``` -#### Ubuntu 20.04 x86_64 で Ubuntu 20.04 armv8 Jetson 向けのビルドをする - -**NVIDIA Jetson 上ではビルドできません。Ubuntu 20.04 x86_64 上でクロスコンパイルしたバイナリを利用するようにしてください。** - -##### 事前準備 - -必要なパッケージをインストールしてください。 - -```shell -$ sudo apt install multistrap -$ sudo apt install binutils-aarch64-linux-gnu -$ sudo apt install python3 -``` - -multistrap に insecure なリポジトリからの取得を許可する設定を行います。 - -```shell -$ sudo sed -e 's/Apt::Get::AllowUnauthenticated=true/Apt::Get::AllowUnauthenticated=true";\n$config_str .= " -o Acquire::AllowInsecureRepositories=true/' -i /usr/sbin/multistrap -``` - -##### ビルド - -```shell -$ python3 messaging_recvonly_sample/ubuntu-20.04_armv8_jetson/run.py -``` - -成功した場合、以下のファイルが作成されます。`_build/ubuntu-20.04_armv8_jetson/release/messaging_recvonly_sample` に `messaging_recvonly_sample` が作成されます。 - -``` -_build/ubuntu-20.04_armv8_jetson/release/messaging_recvonly_sample/ -└── messaging_recvonly_sample -``` - ## 実行する ### コマンドラインから必要なオプションを指定して実行します @@ -168,13 +138,15 @@ _build/ubuntu-20.04_armv8_jetson/release/messaging_recvonly_sample/ デフォルトでは `#sora-devtools` ラベルのメッセージが受信対象となります。 Windows の場合 + ```powershell > .\messaging_recvonly_sample.exe --signaling-url wss://sora.example.com/signaling --channel-id sora ``` Windows 以外の場合 + ```shell -$ ./messaging_recvonly_sample --signaling-url wss://sora.example.com/signaling --channel-id sora +./messaging_recvonly_sample --signaling-url wss://sora.example.com/signaling --channel-id sora ``` #### Sora に関するオプション @@ -183,12 +155,12 @@ $ ./messaging_recvonly_sample --signaling-url wss://sora.example.com/signaling - - `--signaling-url` : Sora サーバのシグナリング URL (必須) - `--channel-id` : channel_id (必須) - - 任意のチャンネル ID + - 任意のチャンネル ID - `--data-channels` : 受信対象のデータチャネルのリストを JSON 形式で指定します - - 未指定の場合は `[{"label":"#sora-devtools", "direction":"recvonly"}]` が設定されます - - 指定可能な内容については Sora のドキュメントの ["type": "connect" 時の "data_channels"](https://sora-doc.shiguredo.jp/MESSAGING#8e04a8) を参照してください + - 未指定の場合は `[{"label":"#sora-devtools", "direction":"recvonly"}]` が設定されます + - 指定可能な内容については Sora のドキュメントの ["type": "connect" 時の "data_channels"](https://sora-doc.shiguredo.jp/MESSAGING#8e04a8) を参照してください #### その他のオプション - `--help` - - ヘルプを表示します + - ヘルプを表示します diff --git a/examples/messaging_recvonly_sample/macos_arm64/run.py b/examples/messaging_recvonly_sample/macos_arm64/run.py index dd152c5d..334c086f 100644 --- a/examples/messaging_recvonly_sample/macos_arm64/run.py +++ b/examples/messaging_recvonly_sample/macos_arm64/run.py @@ -36,16 +36,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -57,17 +57,23 @@ def install_deps( else: build_webrtc_args = { "platform": "macos_arm64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("macos_arm64", source_dir, install_dir) else: - build_sora("macos_arm64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "macos_arm64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -112,10 +118,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -123,8 +129,10 @@ def main(): sample_build_dir = os.path.join(build_dir, "messaging_recvonly_sample") mkdir_p(sample_build_dir) with cd(sample_build_dir): - webrtc_info = get_webrtc_info("macos_arm64", args.webrtc_build_dir, install_dir, args.debug) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + webrtc_info = get_webrtc_info( + "macos_arm64", args.local_webrtc_build_dir, install_dir, args.debug + ) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/messaging_recvonly_sample/ubuntu-20.04_x86_64/run.py b/examples/messaging_recvonly_sample/ubuntu-20.04_x86_64/run.py index bb880a03..7d8d248a 100644 --- a/examples/messaging_recvonly_sample/ubuntu-20.04_x86_64/run.py +++ b/examples/messaging_recvonly_sample/ubuntu-20.04_x86_64/run.py @@ -36,16 +36,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -57,15 +57,17 @@ def install_deps( else: build_webrtc_args = { "platform": "ubuntu-20.04_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-20.04_x86_64", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-20.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -90,10 +92,16 @@ def install_deps( } install_llvm(**install_llvm_args) - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("ubuntu-20.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-20.04_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-20.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -138,10 +146,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -150,9 +158,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-20.04_x86_64", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-20.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/messaging_recvonly_sample/ubuntu-22.04_x86_64/run.py b/examples/messaging_recvonly_sample/ubuntu-22.04_x86_64/run.py index 8b2b5776..48c7134d 100644 --- a/examples/messaging_recvonly_sample/ubuntu-22.04_x86_64/run.py +++ b/examples/messaging_recvonly_sample/ubuntu-22.04_x86_64/run.py @@ -36,16 +36,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -57,15 +57,17 @@ def install_deps( else: build_webrtc_args = { "platform": "ubuntu-22.04_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-22.04_x86_64", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-22.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -91,10 +93,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("ubuntu-22.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-22.04_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-22.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -139,10 +147,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -151,9 +159,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-22.04_x86_64", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-22.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/messaging_recvonly_sample/ubuntu-20.04_armv8_jetson/CMakeLists.txt b/examples/messaging_recvonly_sample/ubuntu-24.04_x86_64/CMakeLists.txt similarity index 94% rename from examples/messaging_recvonly_sample/ubuntu-20.04_armv8_jetson/CMakeLists.txt rename to examples/messaging_recvonly_sample/ubuntu-24.04_x86_64/CMakeLists.txt index af278586..48fbba9f 100644 --- a/examples/messaging_recvonly_sample/ubuntu-20.04_armv8_jetson/CMakeLists.txt +++ b/examples/messaging_recvonly_sample/ubuntu-24.04_x86_64/CMakeLists.txt @@ -23,6 +23,8 @@ find_package(Boost REQUIRED COMPONENTS json filesystem) find_package(WebRTC REQUIRED) find_package(Sora REQUIRED) find_package(Threads REQUIRED) +find_package(Libva REQUIRED) +find_package(Libdrm REQUIRED) add_executable(messaging_recvonly_sample) set_target_properties(messaging_recvonly_sample PROPERTIES CXX_STANDARD 20 C_STANDARD 20) @@ -36,5 +38,4 @@ target_compile_options(messaging_recvonly_sample ) target_include_directories(messaging_recvonly_sample PRIVATE ${CLI11_DIR}/include) target_link_libraries(messaging_recvonly_sample PRIVATE Sora::sora) -target_link_directories(messaging_recvonly_sample PRIVATE ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/tegra) target_compile_definitions(messaging_recvonly_sample PRIVATE CLI11_HAS_FILESYSTEM=0) diff --git a/examples/messaging_recvonly_sample/ubuntu-20.04_armv8_jetson/run.py b/examples/messaging_recvonly_sample/ubuntu-24.04_x86_64/run.py similarity index 70% rename from examples/messaging_recvonly_sample/ubuntu-20.04_armv8_jetson/run.py rename to examples/messaging_recvonly_sample/ubuntu-24.04_x86_64/run.py index c24df6d5..a9b1f861 100644 --- a/examples/messaging_recvonly_sample/ubuntu-20.04_armv8_jetson/run.py +++ b/examples/messaging_recvonly_sample/ubuntu-24.04_x86_64/run.py @@ -1,5 +1,4 @@ import argparse -import hashlib import multiprocessing import os import sys @@ -25,7 +24,6 @@ install_cli11, install_cmake, install_llvm, - install_rootfs, install_sora_and_deps, install_webrtc, mkdir_p, @@ -38,48 +36,38 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") - # multistrap を使った sysroot の構築 - conf = os.path.join(BASE_DIR, "multistrap", "ubuntu-20.04_armv8_jetson.conf") - # conf ファイルのハッシュ値をバージョンとする - version_md5 = hashlib.md5(open(conf, "rb").read()).hexdigest() - install_rootfs_args = { - "version": version_md5, - "version_file": os.path.join(install_dir, "rootfs.version"), - "install_dir": install_dir, - "conf": conf, - } - install_rootfs(**install_rootfs_args) - # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), "source_dir": source_dir, "install_dir": install_dir, - "platform": "ubuntu-20.04_armv8", + "platform": "ubuntu-24.04_x86_64", } install_webrtc(**install_webrtc_args) else: build_webrtc_args = { - "platform": "ubuntu-20.04_armv8", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "platform": "ubuntu-24.04_x86_64", + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-20.04_armv8", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-24.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -105,10 +93,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: - install_sora_and_deps("ubuntu-20.04_armv8_jetson", source_dir, install_dir) + if local_sora_cpp_sdk_dir is None: + install_sora_and_deps("ubuntu-24.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-20.04_armv8_jetson", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-24.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -140,7 +134,7 @@ def main(): args = parser.parse_args() configuration_dir = "debug" if args.debug else "release" - platform = "ubuntu-20.04_armv8_jetson" + platform = "ubuntu-24.04_x86_64" source_dir = os.path.join(BASE_DIR, "_source", platform, configuration_dir) build_dir = os.path.join(BASE_DIR, "_build", platform, configuration_dir) install_dir = os.path.join(BASE_DIR, "_install", platform, configuration_dir) @@ -153,10 +147,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -165,9 +159,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-20.04_armv8", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-24.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") @@ -179,20 +173,9 @@ def main(): # クロスコンパイルの設定。 # 本来は toolchain ファイルに書く内容 - sysroot = os.path.join(install_dir, "rootfs") cmake_args += [ - "-DCMAKE_SYSTEM_NAME=Linux", - "-DCMAKE_SYSTEM_PROCESSOR=aarch64", f"-DCMAKE_C_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang')}", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", f"-DCMAKE_CXX_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang++')}", - "-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu", - f"-DCMAKE_FIND_ROOT_PATH={sysroot}", - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH", - f"-DCMAKE_SYSROOT={sysroot}", f"-DLIBCXX_INCLUDE_DIR={cmake_path(os.path.join(webrtc_info.libcxx_dir, 'include'))}", ] diff --git a/examples/messaging_recvonly_sample/windows_x86_64/run.py b/examples/messaging_recvonly_sample/windows_x86_64/run.py index b44b3cc4..41608d36 100644 --- a/examples/messaging_recvonly_sample/windows_x86_64/run.py +++ b/examples/messaging_recvonly_sample/windows_x86_64/run.py @@ -35,16 +35,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -56,17 +56,23 @@ def install_deps( else: build_webrtc_args = { "platform": "windows_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("windows_x86_64", source_dir, install_dir) else: - build_sora("windows_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "windows_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -112,10 +118,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -124,9 +130,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "windows_x86_64", args.webrtc_build_dir, install_dir, args.debug + "windows_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/multistrap/ubuntu-20.04_armv8_jetson.conf b/examples/multistrap/ubuntu-20.04_armv8_jetson.conf deleted file mode 100644 index 7e1f1975..00000000 --- a/examples/multistrap/ubuntu-20.04_armv8_jetson.conf +++ /dev/null @@ -1,23 +0,0 @@ -[General] -noauth=true -unpack=true -bootstrap=Ports Jetson T194 -aptsources=Ports Jetson T194 - -[Ports] -packages=libstdc++-10-dev libc6-dev libxext-dev libdbus-1-dev -source=http://ports.ubuntu.com -suite=focal -components=main universe - -[Jetson] -packages= -source=https://repo.download.nvidia.com/jetson/common -suite=r35.4 -components=main - -[T194] -packages=nvidia-l4t-camera nvidia-l4t-jetson-multimedia-api -source=https://repo.download.nvidia.com/jetson/t194 -suite=r35.4 -components=main diff --git a/examples/doc/USE_SDL_SAMPLE.md b/examples/sdl_sample/README.md similarity index 58% rename from examples/doc/USE_SDL_SAMPLE.md rename to examples/sdl_sample/README.md index 3d7145e0..5cf140f1 100644 --- a/examples/doc/USE_SDL_SAMPLE.md +++ b/examples/sdl_sample/README.md @@ -4,10 +4,6 @@ [WebRTC SFU Sora](https://sora.shiguredo.jp/) と音声や映像の送受信を行い、[SDL (Simple DirectMedia Layer)](https://www.libsdl.org/) を利用して映像を表示するサンプルです。 -このサンプルに [Sora Labo](https://sora-labo.shiguredo.app/) / [Sora Cloud](https://sora-cloud.shiguredo.jp/) に接続する機能を用意する予定は現在ありません。[Sumomo を使ってみる](./doc/USE_SUMOMO.md) か独自に実装していただく必要があります。 - -- 参考記事 : [sora-cpp-sdk-samples にmomoのオプションを移植した](https://zenn.dev/tetsu_koba/articles/06e11dd4870796) - ## 動作環境 [動作環境](../../README.md#動作環境) をご確認ください。 @@ -20,14 +16,17 @@ **ビルドに関しての問い合わせは受け付けておりません。うまくいかない場合は [GitHub Actions](https://github.com/shiguredo/sora-cpp-sdk/blob/develop/.github/workflows/build.yml) の内容をご確認ください。** - ### リポジトリをクローンする [sora-cpp-sdk](https://github.com/shiguredo/sora-cpp-sdk) をクローンして、examples 以下のディレクトリを利用してください。 +develop ブランチは開発ブランチであり、ビルドが失敗することがあるため、 `main` またはリリースタグを指定するようにしてください。 + +以下は main ブランチを指定する例です。 + ```shell -$ git clone https://github.com/shiguredo/sora-cpp-sdk.git -$ cd sora-cpp-sdk/examples +git clone -b main https://github.com/shiguredo/sora-cpp-sdk.git +cd sora-cpp-sdk/examples ``` ### SDL サンプルをビルドする @@ -39,7 +38,7 @@ $ cd sora-cpp-sdk/examples 以下のツールを準備してください。 - [Visual Studio 2019](https://visualstudio.microsoft.com/ja/downloads/) - - C++ をビルドするためのコンポーネントを入れてください。 + - C++ をビルドするためのコンポーネントを入れてください。 - Python 3.10.5 ##### ビルド @@ -66,7 +65,7 @@ $ cd sora-cpp-sdk/examples ##### ビルド ```shell -$ python3 sdl_sample/macos_arm64/run.py +python3 sdl_sample/macos_arm64/run.py ``` 成功した場合、`_build/macos_arm64/release/sdl_sample` に `sdl_sample` が作成されます。 @@ -83,18 +82,18 @@ _build/macos_arm64/release/sdl_sample 必要なパッケージをインストールしてください。 ```shell -$ sudo apt install libxext-dev -$ sudo apt install libx11-dev -$ sudo apt install libdrm-dev -$ sudo apt install libva-dev -$ sudo apt install pkg-config -$ sudo apt install python3 +sudo apt install libxext-dev +sudo apt install libx11-dev +sudo apt install libdrm-dev +sudo apt install libva-dev +sudo apt install pkg-config +sudo apt install python3 ``` ##### ビルド ```shell -$ python3 sdl_sample/ubuntu-20.04_x86_64/run.py +python3 sdl_sample/ubuntu-20.04_x86_64/run.py ``` 成功した場合、`_build/ubuntu-20.04_x86_64/release/sdl_sample` に `sdl_sample` が作成されます。 @@ -111,18 +110,18 @@ _build/ubuntu-20.04_x86_64/release/sdl_sample/ 必要なパッケージをインストールしてください。 ```shell -$ sudo apt install libxext-dev -$ sudo apt install libx11-dev -$ sudo apt install libdrm-dev -$ sudo apt install libva-dev -$ sudo apt install pkg-config -$ sudo apt install python3 +sudo apt install libxext-dev +sudo apt install libx11-dev +sudo apt install libdrm-dev +sudo apt install libva-dev +sudo apt install pkg-config +sudo apt install python3 ``` ##### ビルド ```shell -$ python3 sdl_sample/ubuntu-22.04_x86_64/run.py +python3 sdl_sample/ubuntu-22.04_x86_64/run.py ``` 成功した場合、以下のファイルが作成されます。`_build/ubuntu-22.04_x86_64/release/sdl_sample` に `sdl_sample` が作成されます。 @@ -132,39 +131,6 @@ _build/ubuntu-22.04_x86_64/release/sdl_sample/ └── sdl_sample ``` -#### Ubuntu 20.04 x86_64 で Ubuntu 20.04 armv8 Jetson 向けのビルドをする - -**NVIDIA Jetson 上ではビルドできません。Ubuntu 20.04 x86_64 上でクロスコンパイルしたバイナリを利用するようにしてください。** - -##### 事前準備 - -必要なパッケージをインストールしてください。 - -```shell -$ sudo apt install multistrap -$ sudo apt install binutils-aarch64-linux-gnu -$ sudo apt install python3 -``` - -multistrap に insecure なリポジトリからの取得を許可する設定を行います。 - -```shell -$ sudo sed -e 's/Apt::Get::AllowUnauthenticated=true/Apt::Get::AllowUnauthenticated=true";\n$config_str .= " -o Acquire::AllowInsecureRepositories=true/' -i /usr/sbin/multistrap -``` - -##### ビルド - -```shell -$ python3 sdl_sample/ubuntu-20.04_armv8_jetson/run.py -``` - -成功した場合、以下のファイルが作成されます。`_build/ubuntu-20.04_armv8_jetson/release/sdl_sample` に `sdl_sample` が作成されます。 - -``` -_build/ubuntu-20.04_armv8_jetson/release/sdl_sample/ -└── sdl_sample -``` - ## 実行する ### コマンドラインから必要なオプションを指定して実行します @@ -173,13 +139,15 @@ _build/ubuntu-20.04_armv8_jetson/release/sdl_sample/ 以下は Sora サーバのシグナリング URL `wss://sora.example.com/signaling` の `sora` チャンネルに `sendrecv` で接続する例です。 Windows の場合 + ```powershell > .\sdl_sample.exe --signaling-url wss://sora.example.com/signaling --role sendrecv --channel-id sora --multistream true ``` Windows 以外の場合 + ```shell -$ ./sdl_sample --signaling-url wss://sora.example.com/signaling --role sendrecv --channel-id sora --multistream true +./sdl_sample --signaling-url wss://sora.example.com/signaling --role sendrecv --channel-id sora --multistream true ``` #### 必須オプション @@ -187,41 +155,41 @@ $ ./sdl_sample --signaling-url wss://sora.example.com/signaling --role sendrecv - `--signaling-url` : Sora サーバのシグナリング URL (必須) - `--channel-id` : [channel_id](https://sora-doc.shiguredo.jp/SIGNALING#ee30e9) (必須) - `--role` : [role](https://sora-doc.shiguredo.jp/SIGNALING#6d21b9) (必須) - - sendrecv / sendonly / recvonly のいずれかを指定 + - sendrecv / sendonly / recvonly のいずれかを指定 #### SDL サンプル実行に関するオプション - `--log-level` : 実行時にターミナルに出力するログのレベル - - `verbose->0,info->1,warning->2,error->3,none->4` の値が指定可能です + - `verbose->0,info->1,warning->2,error->3,none->4` の値が指定可能です #### Sora に関するオプション 設定内容については [Sora のドキュメント](https://sora-doc.shiguredo.jp/SIGNALING) も参考にしてください。 - `--video-codec-type` : [ビデオコーデック指定](https://sora-doc.shiguredo.jp/SIGNALING#d47f4d) - - VP8 / VP9 / AV1 / H264 / H265 が指定可能ですが利用可能なコーデックはプラットフォームに依存します - - 未指定の場合は Sora のデフォルトである VP9 が利用されます + - VP8 / VP9 / AV1 / H264 / H265 が指定可能ですが利用可能なコーデックはプラットフォームに依存します + - 未指定の場合は Sora のデフォルトである VP9 が利用されます - `--multistream` : [マルチストリーム](https://sora-doc.shiguredo.jp/SIGNALING#808bc2) 機能の利用 (true/false) - - 未指定の場合は Sora の設定 (デフォルト: true) が設定されます + - 未指定の場合は Sora の設定 (デフォルト: true) が設定されます - `--video` : 映像の利用 (true/false) - - 未指定の場合は true が設定されます + - 未指定の場合は true が設定されます - `--audio` : 音声の利用 (true/false) - - 未指定の場合は true が設定されます + - 未指定の場合は true が設定されます - `--metadata` : [メタデータ](https://sora-doc.shiguredo.jp/SIGNALING#414142) - - JSON 形式の文字列を指定してください + - JSON 形式の文字列を指定してください #### SDL に関するオプション - `--width` - - 映像を表示するウインドウの横幅を指定します + - 映像を表示するウインドウの横幅を指定します - `--height` - - 映像を表示するウインドウの縦幅を指定します + - 映像を表示するウインドウの縦幅を指定します - `--fullscreen` - - 映像を表示するウインドウをフルスクリーンにします + - 映像を表示するウインドウをフルスクリーンにします - `--show-me` - - 送信している自分の映像を表示します + - 送信している自分の映像を表示します #### その他のオプション - `--help` - - ヘルプを表示します + - ヘルプを表示します diff --git a/examples/sdl_sample/macos_arm64/run.py b/examples/sdl_sample/macos_arm64/run.py index 509db920..54ff806b 100644 --- a/examples/sdl_sample/macos_arm64/run.py +++ b/examples/sdl_sample/macos_arm64/run.py @@ -37,16 +37,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -58,8 +58,8 @@ def install_deps( else: build_webrtc_args = { "platform": "macos_arm64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) @@ -67,10 +67,16 @@ def install_deps( sysroot = cmdcap(["xcrun", "--sdk", "macosx", "--show-sdk-path"]) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("macos_arm64", source_dir, install_dir) else: - build_sora("macos_arm64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "macos_arm64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -136,10 +142,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -147,8 +153,10 @@ def main(): sample_build_dir = os.path.join(build_dir, "sdl_sample") mkdir_p(sample_build_dir) with cd(sample_build_dir): - webrtc_info = get_webrtc_info("macos_arm64", args.webrtc_build_dir, install_dir, args.debug) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + webrtc_info = get_webrtc_info( + "macos_arm64", args.local_webrtc_build_dir, install_dir, args.debug + ) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/sdl_sample/ubuntu-20.04_x86_64/run.py b/examples/sdl_sample/ubuntu-20.04_x86_64/run.py index 38314283..cea8092f 100644 --- a/examples/sdl_sample/ubuntu-20.04_x86_64/run.py +++ b/examples/sdl_sample/ubuntu-20.04_x86_64/run.py @@ -37,16 +37,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -58,15 +58,17 @@ def install_deps( else: build_webrtc_args = { "platform": "ubuntu-20.04_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-20.04_x86_64", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-20.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -92,10 +94,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("ubuntu-20.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-20.04_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-20.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -156,10 +164,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -168,9 +176,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-20.04_x86_64", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-20.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/sdl_sample/ubuntu-22.04_x86_64/run.py b/examples/sdl_sample/ubuntu-22.04_x86_64/run.py index de00f06b..5217a82a 100644 --- a/examples/sdl_sample/ubuntu-22.04_x86_64/run.py +++ b/examples/sdl_sample/ubuntu-22.04_x86_64/run.py @@ -37,16 +37,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -58,15 +58,17 @@ def install_deps( else: build_webrtc_args = { "platform": "ubuntu-22.04_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-22.04_x86_64", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-22.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -92,10 +94,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("ubuntu-22.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-22.04_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-22.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -156,10 +164,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -168,9 +176,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-22.04_x86_64", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-22.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/sdl_sample/ubuntu-20.04_armv8_jetson/CMakeLists.txt b/examples/sdl_sample/ubuntu-24.04_x86_64/CMakeLists.txt similarity index 94% rename from examples/sdl_sample/ubuntu-20.04_armv8_jetson/CMakeLists.txt rename to examples/sdl_sample/ubuntu-24.04_x86_64/CMakeLists.txt index 2863ed1b..7d57c4a5 100644 --- a/examples/sdl_sample/ubuntu-20.04_armv8_jetson/CMakeLists.txt +++ b/examples/sdl_sample/ubuntu-24.04_x86_64/CMakeLists.txt @@ -25,6 +25,8 @@ find_package(WebRTC REQUIRED) find_package(Sora REQUIRED) find_package(SDL2 REQUIRED) find_package(Threads REQUIRED) +find_package(Libva REQUIRED) +find_package(Libdrm REQUIRED) add_executable(sdl_sample) set_target_properties(sdl_sample PROPERTIES CXX_STANDARD 20 C_STANDARD 20) @@ -38,5 +40,4 @@ target_compile_options(sdl_sample ) target_include_directories(sdl_sample PRIVATE ${CLI11_DIR}/include) target_link_libraries(sdl_sample PRIVATE Sora::sora SDL2::SDL2 SDL2::SDL2main) -target_link_directories(sdl_sample PRIVATE ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/tegra) target_compile_definitions(sdl_sample PRIVATE CLI11_HAS_FILESYSTEM=0) diff --git a/examples/sdl_sample/ubuntu-20.04_armv8_jetson/run.py b/examples/sdl_sample/ubuntu-24.04_x86_64/run.py similarity index 68% rename from examples/sdl_sample/ubuntu-20.04_armv8_jetson/run.py rename to examples/sdl_sample/ubuntu-24.04_x86_64/run.py index f61e4c27..899e5e15 100644 --- a/examples/sdl_sample/ubuntu-20.04_armv8_jetson/run.py +++ b/examples/sdl_sample/ubuntu-24.04_x86_64/run.py @@ -1,5 +1,4 @@ import argparse -import hashlib import multiprocessing import os import sys @@ -25,7 +24,6 @@ install_cli11, install_cmake, install_llvm, - install_rootfs, install_sdl2, install_sora_and_deps, install_webrtc, @@ -39,49 +37,38 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") - # multistrap を使った sysroot の構築 - conf = os.path.join(BASE_DIR, "multistrap", "ubuntu-20.04_armv8_jetson.conf") - # conf ファイルのハッシュ値をバージョンとする - version_md5 = hashlib.md5(open(conf, "rb").read()).hexdigest() - install_rootfs_args = { - "version": version_md5, - "version_file": os.path.join(install_dir, "rootfs.version"), - "install_dir": install_dir, - "conf": conf, - } - install_rootfs(**install_rootfs_args) - sysroot = os.path.join(install_dir, "rootfs") - # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), "source_dir": source_dir, "install_dir": install_dir, - "platform": "ubuntu-20.04_armv8", + "platform": "ubuntu-24.04_x86_64", } install_webrtc(**install_webrtc_args) else: build_webrtc_args = { - "platform": "ubuntu-20.04_armv8", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "platform": "ubuntu-24.04_x86_64", + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-20.04_armv8", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-24.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -107,10 +94,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: - install_sora_and_deps("ubuntu-20.04_armv8_jetson", source_dir, install_dir) + if local_sora_cpp_sdk_dir is None: + install_sora_and_deps("ubuntu-24.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-20.04_armv8_jetson", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-24.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -134,18 +127,8 @@ def install_deps( "debug": debug, "platform": "linux", "cmake_args": [ - "-DCMAKE_SYSTEM_NAME=Linux", - "-DCMAKE_SYSTEM_PROCESSOR=aarch64", f"-DCMAKE_C_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang')}", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", f"-DCMAKE_CXX_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang++')}", - "-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu", - f"-DCMAKE_FIND_ROOT_PATH={sysroot}", - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH", - f"-DCMAKE_SYSROOT={sysroot}", ], } install_sdl2(**install_sdl2_args) @@ -168,7 +151,7 @@ def main(): args = parser.parse_args() configuration_dir = "debug" if args.debug else "release" - platform = "ubuntu-20.04_armv8_jetson" + platform = "ubuntu-24.04_x86_64" source_dir = os.path.join(BASE_DIR, "_source", platform, configuration_dir) build_dir = os.path.join(BASE_DIR, "_build", platform, configuration_dir) install_dir = os.path.join(BASE_DIR, "_install", platform, configuration_dir) @@ -181,10 +164,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -193,9 +176,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-20.04_armv8", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-24.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") @@ -208,20 +191,9 @@ def main(): # クロスコンパイルの設定。 # 本来は toolchain ファイルに書く内容 - sysroot = os.path.join(install_dir, "rootfs") cmake_args += [ - "-DCMAKE_SYSTEM_NAME=Linux", - "-DCMAKE_SYSTEM_PROCESSOR=aarch64", f"-DCMAKE_C_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang')}", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", f"-DCMAKE_CXX_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang++')}", - "-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu", - f"-DCMAKE_FIND_ROOT_PATH={sysroot}", - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH", - f"-DCMAKE_SYSROOT={sysroot}", f"-DLIBCXX_INCLUDE_DIR={cmake_path(os.path.join(webrtc_info.libcxx_dir, 'include'))}", ] diff --git a/examples/sdl_sample/windows_x86_64/run.py b/examples/sdl_sample/windows_x86_64/run.py index 359940a7..4a7ea8b3 100644 --- a/examples/sdl_sample/windows_x86_64/run.py +++ b/examples/sdl_sample/windows_x86_64/run.py @@ -36,16 +36,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -57,17 +57,23 @@ def install_deps( else: build_webrtc_args = { "platform": "windows_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("windows_x86_64", source_dir, install_dir) else: - build_sora("windows_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "windows_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -126,10 +132,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -138,9 +144,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "windows_x86_64", args.webrtc_build_dir, install_dir, args.debug + "windows_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/doc/USE_SUMOMO.md b/examples/sumomo/README.md similarity index 60% rename from examples/doc/USE_SUMOMO.md rename to examples/sumomo/README.md index c6a9c047..8b28aacb 100644 --- a/examples/doc/USE_SUMOMO.md +++ b/examples/sumomo/README.md @@ -20,9 +20,13 @@ [sora-cpp-sdk](https://github.com/shiguredo/sora-cpp-sdk) をクローンして、examples 以下のディレクトリを利用してください。 +develop ブランチは開発ブランチであり、ビルドが失敗することがあるため、 `main` またはリリースタグを指定するようにしてください。 + +以下は main ブランチを指定する例です。 + ```shell -$ git clone https://github.com/shiguredo/sora-cpp-sdk.git -$ cd sora-cpp-sdk/examples +git clone -b main https://github.com/shiguredo/sora-cpp-sdk.git +cd sora-cpp-sdk/examples ``` ### Sumomo をビルドする @@ -61,7 +65,7 @@ $ cd sora-cpp-sdk/examples ##### ビルド ```shell -$ python3 sumomo/macos_arm64/run.py +python3 sumomo/macos_arm64/run.py ``` 成功した場合、`_build/macos_arm64/release/sumomo` に `sumomo` が作成されます。 @@ -78,18 +82,18 @@ _build/macos_arm64/release/sumomo 必要なパッケージをインストールしてください。 ```shell -$ sudo apt install libxext-dev -$ sudo apt install libx11-dev -$ sudo apt install libdrm-dev -$ sudo apt install libva-dev -$ sudo apt install pkg-config -$ sudo apt install python3 +sudo apt install libxext-dev +sudo apt install libx11-dev +sudo apt install libdrm-dev +sudo apt install libva-dev +sudo apt install pkg-config +sudo apt install python3 ``` ##### ビルド ```shell -$ python3 sumomo/ubuntu-20.04_x86_64/run.py +python3 sumomo/ubuntu-20.04_x86_64/run.py ``` 成功した場合、`_build/ubuntu-20.04_x86_64/release/sumomo` に `sumomo` が作成されます。 @@ -106,18 +110,18 @@ _build/ubuntu-20.04_x86_64/release/sumomo/ 必要なパッケージをインストールしてください。 ```shell -$ sudo apt install libxext-dev -$ sudo apt install libx11-dev -$ sudo apt install libdrm-dev -$ sudo apt install libva-dev -$ sudo apt install pkg-config -$ sudo apt install python3 +sudo apt install libxext-dev +sudo apt install libx11-dev +sudo apt install libdrm-dev +sudo apt install libva-dev +sudo apt install pkg-config +sudo apt install python3 ``` ##### ビルド ```shell -$ python3 sumomo/ubuntu-22.04_x86_64/run.py +python3 sumomo/ubuntu-22.04_x86_64/run.py ``` 成功した場合、以下のファイルが作成されます。`_build/ubuntu-22.04_x86_64/release/sumomo` に `sumomo` が作成されます。 @@ -127,39 +131,6 @@ _build/ubuntu-22.04_x86_64/release/sumomo/ └── sumomo ``` -#### Ubuntu 20.04 x86_64 で Ubuntu 20.04 armv8 Jetson 向けのビルドをする - -**NVIDIA Jetson 上ではビルドできません。Ubuntu 20.04 x86_64 上でクロスコンパイルしたバイナリを利用するようにしてください。** - -##### 事前準備 - -必要なパッケージをインストールしてください。 - -```shell -$ sudo apt install multistrap -$ sudo apt install binutils-aarch64-linux-gnu -$ sudo apt install python3 -``` - -multistrap に insecure なリポジトリからの取得を許可する設定を行います。 - -```shell -$ sudo sed -e 's/Apt::Get::AllowUnauthenticated=true/Apt::Get::AllowUnauthenticated=true";\n$config_str .= " -o Acquire::AllowInsecureRepositories=true/' -i /usr/sbin/multistrap -``` - -##### ビルド - -```shell -$ python3 sumomo/ubuntu-20.04_armv8_jetson/run.py -``` - -成功した場合、以下のファイルが作成されます。`_build/ubuntu-20.04_armv8_jetson/release/sumomo` に `sumomo` が作成されます。 - -``` -_build/ubuntu-20.04_armv8_jetson/release/sumomo/ -└── sumomo -``` - ## 実行する ### コマンドラインから必要なオプションを指定して実行します @@ -168,13 +139,15 @@ _build/ubuntu-20.04_armv8_jetson/release/sumomo/ 以下は Sora サーバのシグナリング URL `wss://sora.example.com/signaling` の `sora` チャンネルに `sendrecv` で接続する例です。 Windows の場合 + ```powershell > .\sumomo.exe --signaling-url wss://sora.example.com/signaling --role sendrecv --channel-id sora --multistream true --use-sdl ``` Windows 以外の場合 + ```shell -$ ./sumomo --signaling-url wss://sora.example.com/signaling --role sendrecv --channel-id sora --multistream true --use-sdl +./sumomo --signaling-url wss://sora.example.com/signaling --role sendrecv --channel-id sora --multistream true --use-sdl ``` #### 必須オプション @@ -182,77 +155,77 @@ $ ./sumomo --signaling-url wss://sora.example.com/signaling --role sendrecv --ch - `--signaling-url` : Sora サーバのシグナリング URL (必須) - `--channel-id` : [channel_id](https://sora-doc.shiguredo.jp/SIGNALING#ee30e9) (必須) - `--role` : [role](https://sora-doc.shiguredo.jp/SIGNALING#6d21b9) (必須) - - sendrecv / sendonly / recvonly のいずれかを指定 + - sendrecv / sendonly / recvonly のいずれかを指定 #### Momo Sample 実行に関するオプション - `--log-level` : 実行時にターミナルに出力するログのレベル - - `verbose->0,info->1,warning->2,error->3,none->4` の値が指定可能です + - `verbose->0,info->1,warning->2,error->3,none->4` の値が指定可能です - `--resolution` : 映像配信する際の解像度 - - 解像度は `QVGA, VGA, HD, FHD, 4K, or [WIDTH]x[HEIGHT]` の値が指定可能です - - 未指定の場合は `VGA` が設定されます + - 解像度は `QVGA, VGA, HD, FHD, 4K, or [WIDTH]x[HEIGHT]` の値が指定可能です + - 未指定の場合は `VGA` が設定されます - `--hw-mjpeg-decoder` : HW MJPEG デコーダーの利用 (true/false) - - 未指定の場合は false が設定されます - - NVIDIA Jetson のみで利用できます + - 未指定の場合は false が設定されます + - NVIDIA Jetson のみで利用できます #### Sora に関するオプション 設定内容については [Sora のドキュメント](https://sora-doc.shiguredo.jp/SIGNALING) も参考にしてください。 -- `--client-id` : [client_id](https://sora-doc.shiguredo.jp/SIGNALING#d00933) +- `--client-id` : [client_id](https://sora-doc.shiguredo.jp/SIGNALING#d00933) - `--video` : 映像の利用 (true/false) - - 未指定の場合は true が設定されます + - 未指定の場合は true が設定されます - `--audio` : 音声の利用 (true/false) - - 未指定の場合は true が設定されます + - 未指定の場合は true が設定されます - `--video-codec-type` : [ビデオコーデック指定](https://sora-doc.shiguredo.jp/SIGNALING#d47f4d) - - VP8 / VP9 / AV1 / H264 / H265 が指定可能ですが利用可能なコーデックはプラットフォームに依存します - - 未指定の場合は Sora のデフォルトである VP9 が利用されます + - VP8 / VP9 / AV1 / H264 / H265 が指定可能ですが利用可能なコーデックはプラットフォームに依存します + - 未指定の場合は Sora のデフォルトである VP9 が利用されます - `--audio-codec-type` : [オーディオコーデック指定](https://sora-doc.shiguredo.jp/SIGNALING#0fcf4e) - - OPUS が指定可能です - - 未指定の場合は Sora のデフォルトである OPUS が利用されます + - OPUS が指定可能です + - 未指定の場合は Sora のデフォルトである OPUS が利用されます - `--video-bit-rate` : [ビデオビットレート指定](https://sora-doc.shiguredo.jp/SIGNALING#5667cf) - - 0 - 30000 の値が指定可能です - - 0 は未指定と見なされます + - 0 - 30000 の値が指定可能です + - 0 は未指定と見なされます - `--audio-bit-rate` : [オーディオビットレート指定](https://sora-doc.shiguredo.jp/SIGNALING#414142) - - 0 - 510 の値が指定可能です - - 0 は未指定と見なされます + - 0 - 510 の値が指定可能です + - 0 は未指定と見なされます - `--metadata` : [メタデータ](https://sora-doc.shiguredo.jp/SIGNALING#414142) - - JSON 形式の文字列を指定してください + - JSON 形式の文字列を指定してください - `--multistream` : [マルチストリーム](https://sora-doc.shiguredo.jp/SIGNALING#808bc2) 機能の利用 (true/false) - - 未指定の場合は Sora の設定 (デフォルト: true) が設定されます + - 未指定の場合は Sora の設定 (デフォルト: true) が設定されます - `--spotlight` : [スポットライト](https://sora-doc.shiguredo.jp/SIGNALING#8f6c79) 機能の利用 (true/false) - - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます + - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます - `--spotlight-number` : [spotlight_number](https://sora-doc.shiguredo.jp/SPOTLIGHT#c66032) - - 0 - 8 の値が指定可能です - - 0 は未指定と見なされます + - 0 - 8 の値が指定可能です + - 0 は未指定と見なされます - `--simulcast` : [サイマルキャスト](https://sora-doc.shiguredo.jp/SIGNALING#584185) 機能の利用 (true/false) - - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます + - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます - `--data-channel-signaling` : [DataChannel 経由のシグナリング](https://sora-doc.shiguredo.jp/DATA_CHANNEL_SIGNALING) を行います (true/false) - - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます + - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます - `--ignore-disconnect-websocket` - - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます + - 未指定の場合は Sora の設定 (デフォルト: false) が設定されます #### proxy に関するオプション - `--proxy-url` : プロキシの url - - 例) http://proxy.example.com:3128 + - 例) - `--proxy-username` : プロキシ認証に使用するユーザー名 - `--proxy-password` : プロキシ認証に使用するパスワード #### SDL に関するオプション - `--use-sdl` - - [SDL (Simple DirectMedia Layer)](https://www.libsdl.org/) を利用して映像を表示します + - [SDL (Simple DirectMedia Layer)](https://www.libsdl.org/) を利用して映像を表示します - `--window-width` - - 映像を表示するウインドウの横幅を指定します + - 映像を表示するウインドウの横幅を指定します - `--window-height` - - 映像を表示するウインドウの縦幅を指定します + - 映像を表示するウインドウの縦幅を指定します - `--fullscreen` - - 映像を表示するウインドウをフルスクリーンにします + - 映像を表示するウインドウをフルスクリーンにします - `--show-me` - - 送信している自分の映像を表示します + - 送信している自分の映像を表示します #### その他のオプション - `--help` - - ヘルプを表示します + - ヘルプを表示します diff --git a/examples/sumomo/macos_arm64/run.py b/examples/sumomo/macos_arm64/run.py index d6077daa..a21b161c 100644 --- a/examples/sumomo/macos_arm64/run.py +++ b/examples/sumomo/macos_arm64/run.py @@ -37,16 +37,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -58,8 +58,8 @@ def install_deps( else: build_webrtc_args = { "platform": "macos_arm64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) @@ -67,10 +67,16 @@ def install_deps( sysroot = cmdcap(["xcrun", "--sdk", "macosx", "--show-sdk-path"]) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("macos_arm64", source_dir, install_dir) else: - build_sora("macos_arm64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "macos_arm64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -136,10 +142,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -147,8 +153,10 @@ def main(): sample_build_dir = os.path.join(build_dir, "sumomo") mkdir_p(sample_build_dir) with cd(sample_build_dir): - webrtc_info = get_webrtc_info("macos_arm64", args.webrtc_build_dir, install_dir, args.debug) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + webrtc_info = get_webrtc_info( + "macos_arm64", args.local_webrtc_build_dir, install_dir, args.debug + ) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/sumomo/src/sumomo.cpp b/examples/sumomo/src/sumomo.cpp index 6f519e7d..702dbacc 100644 --- a/examples/sumomo/src/sumomo.cpp +++ b/examples/sumomo/src/sumomo.cpp @@ -19,7 +19,7 @@ #include #endif -struct MomoSampleConfig { +struct SumomoConfig { std::string signaling_url; std::string channel_id; std::string role; @@ -32,6 +32,8 @@ struct MomoSampleConfig { bool hw_mjpeg_decoder = false; int video_bit_rate = 0; int audio_bit_rate = 0; + boost::json::value video_h264_params; + boost::json::value video_h265_params; boost::json::value metadata; boost::optional multistream; boost::optional spotlight; @@ -81,11 +83,10 @@ struct MomoSampleConfig { } }; -class MomoSample : public std::enable_shared_from_this, - public sora::SoraSignalingObserver { +class Sumomo : public std::enable_shared_from_this, + public sora::SoraSignalingObserver { public: - MomoSample(std::shared_ptr context, - MomoSampleConfig config) + Sumomo(std::shared_ptr context, SumomoConfig config) : context_(context), config_(config) {} void Run() { @@ -137,6 +138,8 @@ class MomoSample : public std::enable_shared_from_this, config.audio_codec_type = config_.audio_codec_type; config.video_bit_rate = config_.video_bit_rate; config.audio_bit_rate = config_.audio_bit_rate; + config.video_h264_params = config_.video_h264_params; + config.video_h265_params = config_.video_h265_params; config.metadata = config_.metadata; config.multistream = config_.multistream; config.spotlight = config_.spotlight; @@ -148,12 +151,14 @@ class MomoSample : public std::enable_shared_from_this, config.proxy_url = config_.proxy_url; config.proxy_username = config_.proxy_username; config.proxy_password = config_.proxy_password; - config.network_manager = context_->signaling_thread()->BlockingCall([this]() { - return context_->connection_context()->default_network_manager(); - }); - config.socket_factory = context_->signaling_thread()->BlockingCall([this]() { - return context_->connection_context()->default_socket_factory(); - }); + config.network_manager = + context_->signaling_thread()->BlockingCall([this]() { + return context_->connection_context()->default_network_manager(); + }); + config.socket_factory = + context_->signaling_thread()->BlockingCall([this]() { + return context_->connection_context()->default_socket_factory(); + }); conn_ = sora::SoraSignaling::Create(config); boost::asio::executor_work_guard @@ -257,7 +262,7 @@ class MomoSample : public std::enable_shared_from_this, private: std::shared_ptr context_; - MomoSampleConfig config_; + SumomoConfig config_; rtc::scoped_refptr audio_track_; rtc::scoped_refptr video_track_; std::shared_ptr conn_; @@ -296,7 +301,7 @@ int main(int argc, char* argv[]) { } #endif - MomoSampleConfig config; + SumomoConfig config; auto is_valid_resolution = CLI::Validator( [](std::string input) -> std::string { @@ -362,6 +367,12 @@ int main(int argc, char* argv[]) { ->check(CLI::Range(0, 30000)); app.add_option("--audio-bit-rate", config.audio_bit_rate, "Audio bit rate") ->check(CLI::Range(0, 510)); + std::string video_h264_params; + app.add_option("--video-h264-params", video_h264_params, + "Parameters for H.264 video codec"); + std::string video_h265_params; + app.add_option("--video-h265-params", video_h265_params, + "Parameters for H.265 video codec"); std::string metadata; app.add_option("--metadata", metadata, "Signaling metadata used in connect message") @@ -399,12 +410,27 @@ int main(int argc, char* argv[]) { app.add_option("--srtp-key-log-file", config.srtp_key_log_file, "SRTP keying material output file"); + // SoraClientContextConfig に関するオプション + boost::optional use_hardware_encoder; + add_optional_bool(app, "--use-hardware-encoder", use_hardware_encoder, + "Use hardware encoder"); + std::string openh264; + app.add_option("--openh264", openh264, "Path to OpenH264 library"); + try { app.parse(argc, argv); } catch (const CLI::ParseError& e) { exit(app.exit(e)); } + if (!video_h264_params.empty()) { + config.video_h264_params = boost::json::parse(video_h264_params); + } + + if (!video_h265_params.empty()) { + config.video_h265_params = boost::json::parse(video_h265_params); + } + // メタデータのパース if (!metadata.empty()) { config.metadata = boost::json::parse(metadata); @@ -416,10 +442,14 @@ int main(int argc, char* argv[]) { rtc::LogMessage::LogThreads(); } - auto context = - sora::SoraClientContext::Create(sora::SoraClientContextConfig()); - auto momosample = std::make_shared(context, config); - momosample->Run(); + auto context_config = sora::SoraClientContextConfig(); + if (use_hardware_encoder != boost::none) { + context_config.use_hardware_encoder = *use_hardware_encoder; + } + context_config.openh264 = openh264; + auto context = sora::SoraClientContext::Create(context_config); + auto sumomo = std::make_shared(context, config); + sumomo->Run(); return 0; } diff --git a/examples/sumomo/ubuntu-20.04_x86_64/run.py b/examples/sumomo/ubuntu-20.04_x86_64/run.py index 100ddb5b..5584d416 100644 --- a/examples/sumomo/ubuntu-20.04_x86_64/run.py +++ b/examples/sumomo/ubuntu-20.04_x86_64/run.py @@ -37,16 +37,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -58,15 +58,17 @@ def install_deps( else: build_webrtc_args = { "platform": "ubuntu-20.04_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-20.04_x86_64", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-20.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -92,10 +94,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("ubuntu-20.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-20.04_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-20.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -156,10 +164,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -168,9 +176,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-20.04_x86_64", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-20.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/sumomo/ubuntu-22.04_x86_64/run.py b/examples/sumomo/ubuntu-22.04_x86_64/run.py index ad19c2da..814e313b 100644 --- a/examples/sumomo/ubuntu-22.04_x86_64/run.py +++ b/examples/sumomo/ubuntu-22.04_x86_64/run.py @@ -37,16 +37,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -58,15 +58,17 @@ def install_deps( else: build_webrtc_args = { "platform": "ubuntu-22.04_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-22.04_x86_64", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-22.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -92,10 +94,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("ubuntu-22.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-22.04_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-22.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -156,10 +164,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -168,9 +176,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-22.04_x86_64", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-22.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/examples/sumomo/ubuntu-20.04_armv8_jetson/CMakeLists.txt b/examples/sumomo/ubuntu-24.04_x86_64/CMakeLists.txt similarity index 95% rename from examples/sumomo/ubuntu-20.04_armv8_jetson/CMakeLists.txt rename to examples/sumomo/ubuntu-24.04_x86_64/CMakeLists.txt index 88f39424..c41f6cbc 100644 --- a/examples/sumomo/ubuntu-20.04_armv8_jetson/CMakeLists.txt +++ b/examples/sumomo/ubuntu-24.04_x86_64/CMakeLists.txt @@ -25,6 +25,8 @@ find_package(WebRTC REQUIRED) find_package(Sora REQUIRED) find_package(SDL2 REQUIRED) find_package(Threads REQUIRED) +find_package(Libva REQUIRED) +find_package(Libdrm REQUIRED) add_executable(sumomo) set_target_properties(sumomo PROPERTIES CXX_STANDARD 20 C_STANDARD 20) @@ -38,5 +40,4 @@ target_compile_options(sumomo ) target_include_directories(sumomo PRIVATE ${CLI11_DIR}/include) target_link_libraries(sumomo PRIVATE Sora::sora SDL2::SDL2 SDL2::SDL2main) -target_link_directories(sumomo PRIVATE ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/tegra) target_compile_definitions(sumomo PRIVATE CLI11_HAS_FILESYSTEM=0) diff --git a/examples/sumomo/ubuntu-20.04_armv8_jetson/run.py b/examples/sumomo/ubuntu-24.04_x86_64/run.py similarity index 68% rename from examples/sumomo/ubuntu-20.04_armv8_jetson/run.py rename to examples/sumomo/ubuntu-24.04_x86_64/run.py index b516b0c4..993abd72 100644 --- a/examples/sumomo/ubuntu-20.04_armv8_jetson/run.py +++ b/examples/sumomo/ubuntu-24.04_x86_64/run.py @@ -1,5 +1,4 @@ import argparse -import hashlib import multiprocessing import os import sys @@ -25,7 +24,6 @@ install_cli11, install_cmake, install_llvm, - install_rootfs, install_sdl2, install_sora_and_deps, install_webrtc, @@ -39,49 +37,38 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") - # multistrap を使った sysroot の構築 - conf = os.path.join(BASE_DIR, "multistrap", "ubuntu-20.04_armv8_jetson.conf") - # conf ファイルのハッシュ値をバージョンとする - version_md5 = hashlib.md5(open(conf, "rb").read()).hexdigest() - install_rootfs_args = { - "version": version_md5, - "version_file": os.path.join(install_dir, "rootfs.version"), - "install_dir": install_dir, - "conf": conf, - } - install_rootfs(**install_rootfs_args) - sysroot = os.path.join(install_dir, "rootfs") - # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), "source_dir": source_dir, "install_dir": install_dir, - "platform": "ubuntu-20.04_armv8", + "platform": "ubuntu-24.04_x86_64", } install_webrtc(**install_webrtc_args) else: build_webrtc_args = { - "platform": "ubuntu-20.04_armv8", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "platform": "ubuntu-24.04_x86_64", + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info("ubuntu-20.04_armv8", webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info( + "ubuntu-24.04_x86_64", local_webrtc_build_dir, install_dir, debug + ) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: webrtc_version = read_version_file(webrtc_info.version_file) # LLVM @@ -107,10 +94,16 @@ def install_deps( install_llvm(**install_llvm_args) # Sora C++ SDK, Boost - if sora_dir is None: - install_sora_and_deps("ubuntu-20.04_armv8_jetson", source_dir, install_dir) + if local_sora_cpp_sdk_dir is None: + install_sora_and_deps("ubuntu-24.04_x86_64", source_dir, install_dir) else: - build_sora("ubuntu-20.04_armv8_jetson", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "ubuntu-24.04_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -134,18 +127,8 @@ def install_deps( "debug": debug, "platform": "linux", "cmake_args": [ - "-DCMAKE_SYSTEM_NAME=Linux", - "-DCMAKE_SYSTEM_PROCESSOR=aarch64", f"-DCMAKE_C_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang')}", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", f"-DCMAKE_CXX_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang++')}", - "-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu", - f"-DCMAKE_FIND_ROOT_PATH={sysroot}", - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH", - f"-DCMAKE_SYSROOT={sysroot}", ], } install_sdl2(**install_sdl2_args) @@ -168,7 +151,7 @@ def main(): args = parser.parse_args() configuration_dir = "debug" if args.debug else "release" - platform = "ubuntu-20.04_armv8_jetson" + platform = "ubuntu-24.04_x86_64" source_dir = os.path.join(BASE_DIR, "_source", platform, configuration_dir) build_dir = os.path.join(BASE_DIR, "_build", platform, configuration_dir) install_dir = os.path.join(BASE_DIR, "_install", platform, configuration_dir) @@ -181,10 +164,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Debug" if args.debug else "Release" @@ -193,9 +176,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "ubuntu-20.04_armv8", args.webrtc_build_dir, install_dir, args.debug + "ubuntu-24.04_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") @@ -208,20 +191,9 @@ def main(): # クロスコンパイルの設定。 # 本来は toolchain ファイルに書く内容 - sysroot = os.path.join(install_dir, "rootfs") cmake_args += [ - "-DCMAKE_SYSTEM_NAME=Linux", - "-DCMAKE_SYSTEM_PROCESSOR=aarch64", f"-DCMAKE_C_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang')}", - "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu", f"-DCMAKE_CXX_COMPILER={os.path.join(webrtc_info.clang_dir, 'bin', 'clang++')}", - "-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu", - f"-DCMAKE_FIND_ROOT_PATH={sysroot}", - "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", - "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH", - "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH", - f"-DCMAKE_SYSROOT={sysroot}", f"-DLIBCXX_INCLUDE_DIR={cmake_path(os.path.join(webrtc_info.libcxx_dir, 'include'))}", ] diff --git a/examples/sumomo/windows_x86_64/run.py b/examples/sumomo/windows_x86_64/run.py index 8c351c2f..9ca037ea 100644 --- a/examples/sumomo/windows_x86_64/run.py +++ b/examples/sumomo/windows_x86_64/run.py @@ -36,16 +36,16 @@ def install_deps( build_dir, install_dir, debug, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], - sora_dir: Optional[str], - sora_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], + local_sora_cpp_sdk_dir: Optional[str], + local_sora_cpp_sdk_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") # WebRTC - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -57,17 +57,23 @@ def install_deps( else: build_webrtc_args = { "platform": "windows_x86_64", - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) # Sora C++ SDK, Boost - if sora_dir is None: + if local_sora_cpp_sdk_dir is None: install_sora_and_deps("windows_x86_64", source_dir, install_dir) else: - build_sora("windows_x86_64", sora_dir, sora_args, debug, webrtc_build_dir) + build_sora( + "windows_x86_64", + local_sora_cpp_sdk_dir, + local_sora_cpp_sdk_args, + debug, + local_webrtc_build_dir, + ) # CMake install_cmake_args = { @@ -126,10 +132,10 @@ def main(): build_dir, install_dir, args.debug, - args.webrtc_build_dir, - args.webrtc_build_args, - args.sora_dir, - args.sora_args, + args.local_webrtc_build_dir, + args.local_webrtc_build_args, + args.local_sora_cpp_sdk_dir, + args.local_sora_cpp_sdk_args, ) configuration = "Release" @@ -142,9 +148,9 @@ def main(): mkdir_p(sample_build_dir) with cd(sample_build_dir): webrtc_info = get_webrtc_info( - "windows_x86_64", args.webrtc_build_dir, install_dir, args.debug + "windows_x86_64", args.local_webrtc_build_dir, install_dir, args.debug ) - sora_info = get_sora_info(platform, args.sora_dir, install_dir, args.debug) + sora_info = get_sora_info(platform, args.local_sora_cpp_sdk_dir, install_dir, args.debug) cmake_args = [] cmake_args.append(f"-DCMAKE_BUILD_TYPE={configuration}") diff --git a/include/sora/cuda_context.h b/include/sora/cuda_context.h index 9cc82fab..0fe455ab 100644 --- a/include/sora/cuda_context.h +++ b/include/sora/cuda_context.h @@ -19,8 +19,10 @@ class CudaContext { enum class CudaVideoCodec { H264, + H265, VP8, VP9, + AV1, JPEG, }; diff --git a/include/sora/hwenc_jetson/jetson_buffer.h b/include/sora/hwenc_jetson/jetson_buffer.h deleted file mode 100644 index 3b41ecc8..00000000 --- a/include/sora/hwenc_jetson/jetson_buffer.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef SORA_HWENC_JETSON_JETSON_BUFFER_H_ -#define SORA_HWENC_JETSON_JETSON_BUFFER_H_ - -#include -#include - -// WebRTC -#include -#include -#include -#include - -#include "jetson_jpeg_decoder.h" - -namespace sora { - -class JetsonBuffer : public webrtc::VideoFrameBuffer { - public: - static rtc::scoped_refptr Create( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height, - int fd, - uint32_t pixfmt, - std::shared_ptr decoder); - - static rtc::scoped_refptr Create(webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height); - - Type type() const override; - webrtc::VideoType VideoType() const; - int width() const override; - int height() const override; - rtc::scoped_refptr ToI420() override; - - int RawWidth() const; - int RawHeight() const; - int DecodedFd() const; - uint32_t V4L2PixelFormat() const; - std::shared_ptr JpegDecoder() const; - uint8_t* Data() const; - void SetLength(size_t size); - size_t Length() const; - - protected: - JetsonBuffer(webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height, - int fd, - uint32_t pixfmt, - std::shared_ptr decoder); - - JetsonBuffer(webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height); - - private: - webrtc::VideoType video_type_; - const int raw_width_; - const int raw_height_; - const int scaled_width_; - const int scaled_height_; - const int fd_; - const uint32_t pixfmt_; - const std::shared_ptr decoder_; - const std::unique_ptr data_; - size_t length_; -}; - -} // namespace sora - -#endif diff --git a/include/sora/hwenc_jetson/jetson_jpeg_decoder.h b/include/sora/hwenc_jetson/jetson_jpeg_decoder.h deleted file mode 100644 index 7c2c9702..00000000 --- a/include/sora/hwenc_jetson/jetson_jpeg_decoder.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef SORA_HWENC_JETSON_JETSON_JPEG_DECODER_H_ -#define SORA_HWENC_JETSON_JETSON_JPEG_DECODER_H_ - -#include - -#include "jetson_jpeg_decoder_pool.h" - -class NvJPEGDecoder; - -namespace sora { - -class JetsonJpegDecoderPool; - -class JetsonJpegDecoder { - public: - JetsonJpegDecoder(std::shared_ptr pool, - std::shared_ptr decoder); - ~JetsonJpegDecoder(); - - int DecodeToFd(int& fd, - unsigned char* in_buf, - unsigned long in_buf_size, - uint32_t& pixfmt, - uint32_t& width, - uint32_t& height); - - private: - std::shared_ptr pool_; - std::shared_ptr decoder_; -}; - -} // namespace sora - -#endif \ No newline at end of file diff --git a/include/sora/hwenc_jetson/jetson_jpeg_decoder_pool.h b/include/sora/hwenc_jetson/jetson_jpeg_decoder_pool.h deleted file mode 100644 index 328e6f1d..00000000 --- a/include/sora/hwenc_jetson/jetson_jpeg_decoder_pool.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef SORA_HWENC_JETSON_JETSON_JPEG_DECODER_POOL_H_ -#define SORA_HWENC_JETSON_JETSON_JPEG_DECODER_POOL_H_ - -#include -#include -#include - -#include "jetson_jpeg_decoder.h" - -class NvJPEGDecoder; - -namespace sora { - -class JetsonJpegDecoder; - -class JetsonJpegDecoderPool - : public std::enable_shared_from_this { - public: - std::shared_ptr Pop(); - void Push(std::shared_ptr decoder); - - private: - std::mutex mtx_; - std::queue> decoder_queue_; -}; - -} // namespace sora - -#endif diff --git a/include/sora/hwenc_jetson/jetson_v4l2_capturer.h b/include/sora/hwenc_jetson/jetson_v4l2_capturer.h deleted file mode 100644 index fc01594d..00000000 --- a/include/sora/hwenc_jetson/jetson_v4l2_capturer.h +++ /dev/null @@ -1,83 +0,0 @@ -#ifndef SORA_HWENC_JETSON_JETSON_V4L2_CAPTURER_H_ -#define SORA_HWENC_JETSON_JETSON_V4L2_CAPTURER_H_ - -#include -#include - -#include - -// Linux -#include - -// WebRTC -#include -#include -#include -#include - -#include "jetson_jpeg_decoder_pool.h" -#include "sora/scalable_track_source.h" -#include "sora/v4l2/v4l2_video_capturer.h" - -namespace sora { - -class JetsonV4L2Capturer : public ScalableVideoTrackSource { - public: - static rtc::scoped_refptr Create( - const V4L2VideoCapturerConfig& config); - JetsonV4L2Capturer(const V4L2VideoCapturerConfig& config); - ~JetsonV4L2Capturer(); - - private: - static void LogDeviceList( - webrtc::VideoCaptureModule::DeviceInfo* device_info); - - int32_t Init(const char* deviceUniqueId, - const std::string& specifiedVideoDevice); - int32_t StartCapture(const V4L2VideoCapturerConfig& config); - - int32_t StopCapture(); - bool AllocateVideoBuffers(); - bool DeAllocateVideoBuffers(); - void OnCaptured(v4l2_buffer* buf); - - int32_t _deviceFd; - int32_t _currentWidth; - int32_t _currentHeight; - int32_t _currentPixelFormat; - int32_t _currentFrameRate; - webrtc::VideoType _captureVideoType; - struct Buffer { - void* start; - size_t length; - int fd; - }; - Buffer* _pool; - - private: - static rtc::scoped_refptr Create( - webrtc::VideoCaptureModule::DeviceInfo* device_info, - const V4L2VideoCapturerConfig& config, - size_t capture_device_index); - bool FindDevice(const char* deviceUniqueIdUTF8, const std::string& device); - - enum { kNoOfV4L2Bufffers = 4 }; - - static void CaptureThread(void*); - bool CaptureProcess(); - - rtc::PlatformThread _captureThread; - webrtc::Mutex capture_lock_; - bool quit_ RTC_GUARDED_BY(capture_lock_); - std::string _videoDevice; - - int32_t _buffersAllocatedByDevice; - bool _useNative; - bool _captureStarted; - - std::shared_ptr jpeg_decoder_pool_; -}; - -} // namespace sora - -#endif diff --git a/include/sora/hwenc_jetson/jetson_video_decoder.h b/include/sora/hwenc_jetson/jetson_video_decoder.h deleted file mode 100644 index b8cc9566..00000000 --- a/include/sora/hwenc_jetson/jetson_video_decoder.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#ifndef SORA_HWENC_JETSON_JETSON_VIDEO_DECODER_H_ -#define SORA_HWENC_JETSON_JETSON_VIDEO_DECODER_H_ - -// WebRTC -#include -#include -#include - -struct v4l2_crop; -class NvV4l2Element; -class NvVideoDecoder; - -namespace sora { - -class JetsonVideoDecoder : public webrtc::VideoDecoder { - public: - JetsonVideoDecoder(webrtc::VideoCodecType codec); - ~JetsonVideoDecoder() override; - - static bool IsSupportedVP8(); - static bool IsSupportedAV1(); - - bool Configure(const Settings& settings) override; - - int32_t Decode(const webrtc::EncodedImage& input_image, - bool missing_frames, - int64_t render_time_ms) override; - - int32_t RegisterDecodeCompleteCallback( - webrtc::DecodedImageCallback* callback) override; - - int32_t Release() override; - - const char* ImplementationName() const override; - - private: - int32_t JetsonConfigure(); - bool JetsonRelease(); - void SendEOS(NvV4l2Element* element); - static void CaptureLoopFunction(void* obj); - void CaptureLoop(); - int32_t SetCapture(); - - uint32_t input_format_; - NvVideoDecoder* decoder_; - webrtc::DecodedImageCallback* decode_complete_callback_; - webrtc::VideoFrameBufferPool buffer_pool_; - rtc::PlatformThread capture_loop_; - std::atomic eos_; - std::atomic got_error_; - int dst_dma_fd_; - std::shared_ptr capture_crop_; -}; - -} // namespace sora - -#endif diff --git a/include/sora/hwenc_jetson/jetson_video_encoder.h b/include/sora/hwenc_jetson/jetson_video_encoder.h deleted file mode 100644 index 4e577ebc..00000000 --- a/include/sora/hwenc_jetson/jetson_video_encoder.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#ifndef SORA_HWENC_JETSON_JETSON_VIDEO_ENCODER_H_ -#define SORA_HWENC_JETSON_JETSON_VIDEO_ENCODER_H_ - -#include -#include -#include - -// Linux -#include - -// WebRTC -#include -#include -#include -#include -#include -#include - -#include "jetson_jpeg_decoder.h" - -#define CONVERTER_CAPTURE_NUM 2 - -class NvBuffer; -class NvV4l2Element; -class NvVideoEncoder; -struct v4l2_ctrl_videoenc_outputbuf_metadata_; - -namespace sora { - -class JetsonVideoEncoder : public webrtc::VideoEncoder { - public: - explicit JetsonVideoEncoder(const cricket::VideoCodec& codec); - ~JetsonVideoEncoder() override; - - static bool IsSupportedVP8(); - static bool IsSupportedVP9(); - static bool IsSupportedAV1(); - - int32_t InitEncode(const webrtc::VideoCodec* codec_settings, - int32_t number_of_cores, - size_t max_payload_size) override; - int32_t RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback* callback) override; - int32_t Release() override; - int32_t Encode( - const webrtc::VideoFrame& frame, - const std::vector* frame_types) override; - void SetRates(const RateControlParameters& parameters) override; - webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; - - private: - struct FrameParams { - FrameParams(int32_t w, - int32_t h, - int64_t rtms, - int64_t ntpms, - int64_t tsus, - int64_t rtpts, - webrtc::VideoRotation r, - absl::optional c, - std::shared_ptr d) - : width(w), - height(h), - render_time_ms(rtms), - ntp_time_ms(ntpms), - timestamp_us(tsus), - timestamp_rtp(rtpts), - rotation(r), - color_space(c), - decoder_(d) {} - - int32_t width; - int32_t height; - int64_t render_time_ms; - int64_t ntp_time_ms; - int64_t timestamp_us; - int64_t timestamp_rtp; - webrtc::VideoRotation rotation; - absl::optional color_space; - std::shared_ptr decoder_; - }; - - int32_t JetsonConfigure(); - void JetsonRelease(); - void SendEOS(); - static bool EncodeFinishedCallbackFunction(struct v4l2_buffer* v4l2_buf, - NvBuffer* buffer, - NvBuffer* shared_buffer, - void* data); - bool EncodeFinishedCallback(struct v4l2_buffer* v4l2_buf, - NvBuffer* buffer, - NvBuffer* shared_buffer); - void SetFramerate(uint32_t framerate); - void SetBitrateBps(uint32_t bitrate_bps); - int32_t SendFrame(unsigned char* buffer, - size_t size, - std::unique_ptr params, - v4l2_ctrl_videoenc_outputbuf_metadata_* enc_metadata); - - webrtc::VideoCodec codec_; - webrtc::EncodedImageCallback* callback_; - NvVideoEncoder* encoder_; - std::unique_ptr bitrate_adjuster_; - uint32_t framerate_; - int32_t configured_framerate_; - uint32_t target_bitrate_bps_; - uint32_t configured_bitrate_bps_; - int key_frame_interval_; - uint32_t decode_pixfmt_; - uint32_t raw_width_; - uint32_t raw_height_; - int32_t width_; - int32_t height_; - bool use_native_; - bool use_dmabuff_; - int dmabuff_fd_[CONVERTER_CAPTURE_NUM]; - - webrtc::GofInfoVP9 gof_; - size_t gof_idx_; - - std::unique_ptr svc_controller_; - - webrtc::Mutex frame_params_lock_; - std::queue> frame_params_; - std::mutex enc0_buffer_mtx_; - std::condition_variable enc0_buffer_cond_; - std::queue* enc0_buffer_queue_; - int output_plane_fd_[32]; - webrtc::EncodedImage encoded_image_; - webrtc::ScalabilityMode scalability_mode_; - std::vector obu_seq_header_; -}; - -} // namespace sora - -#endif diff --git a/include/sora/hwenc_nvcodec/nvcodec_h264_encoder.h b/include/sora/hwenc_nvcodec/nvcodec_h264_encoder.h deleted file mode 100644 index 3bce4edb..00000000 --- a/include/sora/hwenc_nvcodec/nvcodec_h264_encoder.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef SORA_HWENC_NVCODEC_NVCODEC_H264_ENCODER_H_ -#define SORA_HWENC_NVCODEC_NVCODEC_H264_ENCODER_H_ - -#include - -// WebRTC -#include -#include - -#include "sora/cuda_context.h" - -namespace sora { - -class NvCodecH264Encoder : public webrtc::VideoEncoder { - public: - static bool IsSupported(std::shared_ptr cuda_context); - static std::unique_ptr Create( - const cricket::VideoCodec& codec, - std::shared_ptr cuda_context); -}; - -} // namespace sora - -#endif diff --git a/include/sora/hwenc_nvcodec/nvcodec_video_encoder.h b/include/sora/hwenc_nvcodec/nvcodec_video_encoder.h new file mode 100644 index 00000000..75b72abe --- /dev/null +++ b/include/sora/hwenc_nvcodec/nvcodec_video_encoder.h @@ -0,0 +1,25 @@ +#ifndef SORA_HWENC_NVCODEC_NVCODEC_VIDEO_ENCODER_H_ +#define SORA_HWENC_NVCODEC_NVCODEC_VIDEO_ENCODER_H_ + +#include + +// WebRTC +#include +#include + +#include "sora/cuda_context.h" + +namespace sora { + +class NvCodecVideoEncoder : public webrtc::VideoEncoder { + public: + static bool IsSupported(std::shared_ptr cuda_context, + CudaVideoCodec codec); + static std::unique_ptr Create( + std::shared_ptr cuda_context, + CudaVideoCodec codec); +}; + +} // namespace sora + +#endif diff --git a/include/sora/open_h264_video_encoder.h b/include/sora/open_h264_video_encoder.h index 8b1b3621..c30b8b4a 100644 --- a/include/sora/open_h264_video_encoder.h +++ b/include/sora/open_h264_video_encoder.h @@ -11,7 +11,7 @@ namespace sora { std::unique_ptr CreateOpenH264VideoEncoder( - const cricket::VideoCodec& codec, + const webrtc::SdpVideoFormat& format, std::string openh264); } diff --git a/include/sora/sora_signaling.h b/include/sora/sora_signaling.h index 89383e00..82c303e0 100644 --- a/include/sora/sora_signaling.h +++ b/include/sora/sora_signaling.h @@ -15,8 +15,9 @@ #include #include -#include "data_channel.h" -#include "websocket.h" +#include "sora/data_channel.h" +#include "sora/version.h" +#include "sora/websocket.h" namespace sora { @@ -79,6 +80,7 @@ struct SoraSignalingConfig { boost::json::value video_vp9_params; boost::json::value video_av1_params; boost::json::value video_h264_params; + boost::json::value video_h265_params; std::string audio_streaming_language_code; boost::json::value metadata; boost::json::value signaling_notify_metadata; @@ -133,6 +135,8 @@ struct SoraSignalingConfig { rtc::PacketSocketFactory* socket_factory = nullptr; bool disable_signaling_url_randomization = false; + + boost::optional user_agent; }; class SoraSignaling : public std::enable_shared_from_this, diff --git a/include/sora/sora_video_decoder_factory.h b/include/sora/sora_video_decoder_factory.h index 7983ae33..2ba98fec 100644 --- a/include/sora/sora_video_decoder_factory.h +++ b/include/sora/sora_video_decoder_factory.h @@ -5,6 +5,7 @@ #include // WebRTC +#include #include #include @@ -52,7 +53,8 @@ class SoraVideoDecoderFactory : public webrtc::VideoDecoderFactory { std::vector GetSupportedFormats() const override; - std::unique_ptr CreateVideoDecoder( + std::unique_ptr Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) override; private: diff --git a/include/sora/sora_video_encoder_factory.h b/include/sora/sora_video_encoder_factory.h index 8b6c3391..c866ef9a 100644 --- a/include/sora/sora_video_encoder_factory.h +++ b/include/sora/sora_video_encoder_factory.h @@ -56,7 +56,9 @@ struct SoraVideoEncoderFactoryConfig { /* use_simulcast_adapter = true の際に、エンコーダー内でビデオ・フレームのバッファーを I420 に変換するかどうか - false に設定すると CPU への負荷が下がり、エンコードの性能が向上するが、バッファーの実装によってはサイマルキャストが利用できなくなる + + force_i420_conversion_for_simulcast_adapter = false を設定することで I420 への変換を実行しなくなるため、 CPU への負荷が下がり、エンコードの性能が向上する + ただし、使用するバッファーの実装によっては、バッファーを複数回読みこむことができないため、サイマルキャストが利用できなくなる このフラグが必要になった背景は以下の通り - サイマルキャスト時、 JetsonBuffer のような一部の kNative なバッファーの実装において、バッファーを複数回読み込めないという制限があるため、 I420 への変換が必要になる @@ -81,12 +83,14 @@ class SoraVideoEncoderFactory : public webrtc::VideoEncoderFactory { std::vector GetSupportedFormats() const override; - std::unique_ptr CreateVideoEncoder( + std::unique_ptr Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) override; private: // 一番内側のエンコーダを作る std::unique_ptr CreateInternalVideoEncoder( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format, int& alignment); diff --git a/include/sora/version.h b/include/sora/version.h index 29a5884b..6e556b50 100644 --- a/include/sora/version.h +++ b/include/sora/version.h @@ -3,13 +3,19 @@ #include +// Boost +#include + namespace sora { +typedef boost::optional http_header_value; + class Version { public: static std::string GetClientName(); static std::string GetLibwebrtcName(); static std::string GetEnvironmentName(); + static http_header_value GetDefaultUserAgent(); }; } // namespace sora diff --git a/include/sora/websocket.h b/include/sora/websocket.h index 50ce6ed3..4208de62 100644 --- a/include/sora/websocket.h +++ b/include/sora/websocket.h @@ -16,6 +16,7 @@ #include #include "url_parts.h" +#include "version.h" namespace sora { @@ -67,6 +68,8 @@ class Websocket { Websocket(boost::asio::ip::tcp::socket socket); ~Websocket(); + void SetUserAgent(http_header_value user_agent); + // WebSocket クライアントの接続確立 void Connect(const std::string& url, connect_callback_t on_connect); @@ -144,6 +147,8 @@ class Websocket { boost::asio::deadline_timer close_timeout_timer_; + http_header_value user_agent_; + bool https_proxy_ = false; std::string proxy_url_; std::string proxy_username_; diff --git a/multistrap/ubuntu-20.04_armv8_jetson.conf b/multistrap/ubuntu-20.04_armv8_jetson.conf deleted file mode 100644 index ef5e19a3..00000000 --- a/multistrap/ubuntu-20.04_armv8_jetson.conf +++ /dev/null @@ -1,23 +0,0 @@ -[General] -noauth=true -unpack=true -bootstrap=Ports Jetson T194 -aptsources=Ports Jetson T194 - -[Ports] -packages=libstdc++-10-dev libc6-dev -source=http://ports.ubuntu.com -suite=focal -components=main universe - -[Jetson] -packages= -source=https://repo.download.nvidia.com/jetson/common -suite=r35.4 -components=main - -[T194] -packages=nvidia-l4t-camera nvidia-l4t-jetson-multimedia-api -source=https://repo.download.nvidia.com/jetson/t194 -suite=r35.4 -components=main \ No newline at end of file diff --git a/run.py b/run.py index 8f33147d..587d6fd8 100644 --- a/run.py +++ b/run.py @@ -1,5 +1,4 @@ import argparse -import hashlib import logging import multiprocessing import os @@ -32,7 +31,6 @@ install_cuda_windows, install_llvm, install_openh264, - install_rootfs, install_vpl, install_webrtc, mkdir_p, @@ -74,7 +72,11 @@ def get_common_cmake_args( args.append(f"-DCMAKE_OBJCXX_COMPILER_TARGET={target}") args.append(f"-DCMAKE_SYSROOT={sysroot}") if platform.target.os == "ubuntu": - if platform.target.package_name in ("ubuntu-20.04_x86_64", "ubuntu-22.04_x86_64"): + if platform.target.package_name in ( + "ubuntu-20.04_x86_64", + "ubuntu-22.04_x86_64", + "ubuntu-24.04_x86_64", + ): args.append("-DCMAKE_C_COMPILER=clang-18") args.append("-DCMAKE_CXX_COMPILER=clang++-18") else: @@ -88,28 +90,6 @@ def get_common_cmake_args( args.append(f"-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES={path}") cxxflags = ["-nostdinc++", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE"] args.append(f"-DCMAKE_CXX_FLAGS={' '.join(cxxflags)}") - if platform.target.os == "jetson": - sysroot = os.path.join(install_dir, "rootfs") - args.append("-DCMAKE_SYSTEM_NAME=Linux") - args.append("-DCMAKE_SYSTEM_PROCESSOR=aarch64") - args.append(f"-DCMAKE_SYSROOT={sysroot}") - args.append("-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu") - args.append("-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu") - args.append( - f"-DCMAKE_C_COMPILER={cmake_path(os.path.join(webrtc_info.clang_dir, 'bin', 'clang'))}" - ) - args.append( - f"-DCMAKE_CXX_COMPILER={cmake_path(os.path.join(webrtc_info.clang_dir, 'bin', 'clang++'))}" - ) - args.append(f"-DCMAKE_FIND_ROOT_PATH={sysroot}") - args.append("-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER") - args.append("-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH") - args.append("-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH") - args.append("-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH") - path = cmake_path(os.path.join(webrtc_info.libcxx_dir, "include")) - args.append(f"-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES={path}") - cxxflags = ["-nostdinc++", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE"] - args.append(f"-DCMAKE_CXX_FLAGS={' '.join(cxxflags)}") if platform.target.os == "ios": args += ["-G", "Xcode"] args.append("-DCMAKE_SYSTEM_NAME=iOS") @@ -146,25 +126,12 @@ def install_deps( build_dir: str, install_dir: str, debug: bool, - webrtc_build_dir: Optional[str], - webrtc_build_args: List[str], + local_webrtc_build_dir: Optional[str], + local_webrtc_build_args: List[str], ): with cd(BASE_DIR): version = read_version_file("VERSION") - # multistrap を使った sysroot の構築 - if platform.target.os == "jetson": - conf = os.path.join(BASE_DIR, "multistrap", f"{platform.target.package_name}.conf") - # conf ファイルのハッシュ値をバージョンとする - version_md5 = hashlib.md5(open(conf, "rb").read()).hexdigest() - install_rootfs_args = { - "version": version_md5, - "version_file": os.path.join(install_dir, "rootfs.version"), - "install_dir": install_dir, - "conf": conf, - } - install_rootfs(**install_rootfs_args) - # Android NDK if platform.target.os == "android": install_android_ndk_args = { @@ -198,7 +165,7 @@ def install_deps( # WebRTC webrtc_platform = get_webrtc_platform(platform) - if webrtc_build_dir is None: + if local_webrtc_build_dir is None: install_webrtc_args = { "version": version["WEBRTC_BUILD_VERSION"], "version_file": os.path.join(install_dir, "webrtc.version"), @@ -211,20 +178,20 @@ def install_deps( else: build_webrtc_args = { "platform": webrtc_platform, - "webrtc_build_dir": webrtc_build_dir, - "webrtc_build_args": webrtc_build_args, + "local_webrtc_build_dir": local_webrtc_build_dir, + "local_webrtc_build_args": local_webrtc_build_args, "debug": debug, } build_webrtc(**build_webrtc_args) - webrtc_info = get_webrtc_info(webrtc_platform, webrtc_build_dir, install_dir, debug) + webrtc_info = get_webrtc_info(webrtc_platform, local_webrtc_build_dir, install_dir, debug) webrtc_version = read_version_file(webrtc_info.version_file) webrtc_deps = read_version_file(webrtc_info.deps_file) # Windows は MSVC を使うので不要 # macOS と iOS は Apple Clang を使うので不要 - if platform.target.os not in ("windows", "macos", "ios") and webrtc_build_dir is None: + if platform.target.os not in ("windows", "macos", "ios") and local_webrtc_build_dir is None: # LLVM tools_url = webrtc_version["WEBRTC_SRC_TOOLS_URL"] tools_commit = webrtc_version["WEBRTC_SRC_TOOLS_COMMIT"] @@ -324,35 +291,6 @@ def install_deps( install_boost_args["toolset"] = "clang" install_boost_args["android_ndk"] = os.path.join(install_dir, "android-ndk") install_boost_args["native_api_level"] = version["ANDROID_NATIVE_API_LEVEL"] - elif platform.target.os == "jetson": - sysroot = os.path.join(install_dir, "rootfs") - install_boost_args["target_os"] = "linux" - install_boost_args["cxx"] = os.path.join(webrtc_info.clang_dir, "bin", "clang++") - install_boost_args["cflags"] = [ - "-fPIC", - f"--sysroot={sysroot}", - "--target=aarch64-linux-gnu", - f"-I{os.path.join(sysroot, 'usr', 'include', 'aarch64-linux-gnu')}", - ] - install_boost_args["cxxflags"] = [ - "-fPIC", - "--target=aarch64-linux-gnu", - f"--sysroot={sysroot}", - f"-I{os.path.join(sysroot, 'usr', 'include', 'aarch64-linux-gnu')}", - "-D_LIBCPP_ABI_NAMESPACE=Cr", - "-D_LIBCPP_ABI_VERSION=2", - "-D_LIBCPP_DISABLE_AVAILABILITY", - "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE", - "-nostdinc++", - "-std=gnu++17", - f"-isystem{os.path.join(webrtc_info.libcxx_dir, 'include')}", - ] - install_boost_args["linkflags"] = [ - f"-L{os.path.join(sysroot, 'usr', 'lib', 'aarch64-linux-gnu')}", - f"-B{os.path.join(sysroot, 'usr', 'lib', 'aarch64-linux-gnu')}", - ] - install_boost_args["toolset"] = "clang" - install_boost_args["architecture"] = "arm" else: install_boost_args["target_os"] = "linux" install_boost_args["cxx"] = os.path.join(webrtc_info.clang_dir, "bin", "clang++") @@ -530,7 +468,7 @@ def install_deps( "macos_arm64", "ubuntu-20.04_x86_64", "ubuntu-22.04_x86_64", - "ubuntu-20.04_armv8_jetson", + "ubuntu-24.04_x86_64", "ios", "android", ] @@ -558,8 +496,8 @@ def main(): platform = Platform("ubuntu", "20.04", "x86_64") elif args.target == "ubuntu-22.04_x86_64": platform = Platform("ubuntu", "22.04", "x86_64") - elif args.target == "ubuntu-20.04_armv8_jetson": - platform = Platform("jetson", None, "armv8") + elif args.target == "ubuntu-24.04_x86_64": + platform = Platform("ubuntu", "24.04", "x86_64") elif args.target == "ios": platform = Platform("ios", None, None) elif args.target == "android": @@ -586,8 +524,8 @@ def main(): build_dir, install_dir, args.debug, - webrtc_build_dir=args.webrtc_build_dir, - webrtc_build_args=args.webrtc_build_args, + local_webrtc_build_dir=args.local_webrtc_build_dir, + local_webrtc_build_args=args.local_webrtc_build_args, ) configuration = "Release" @@ -605,7 +543,7 @@ def main(): cmake_args.append(f"-DBOOST_ROOT={cmake_path(os.path.join(install_dir, 'boost'))}") webrtc_platform = get_webrtc_platform(platform) webrtc_info = get_webrtc_info( - webrtc_platform, args.webrtc_build_dir, install_dir, args.debug + webrtc_platform, args.local_webrtc_build_dir, install_dir, args.debug ) webrtc_version = read_version_file(webrtc_info.version_file) webrtc_deps = read_version_file(webrtc_info.deps_file) @@ -626,7 +564,11 @@ def main(): if platform.target.os == "windows": cmake_args.append(f"-DCMAKE_SYSTEM_VERSION={WINDOWS_SDK_VERSION}") if platform.target.os == "ubuntu": - if platform.target.package_name in ("ubuntu-20.04_x86_64", "ubuntu-22.04_x86_64"): + if platform.target.package_name in ( + "ubuntu-20.04_x86_64", + "ubuntu-22.04_x86_64", + "ubuntu-24.04_x86_64", + ): cmake_args.append("-DCMAKE_C_COMPILER=clang-18") cmake_args.append("-DCMAKE_CXX_COMPILER=clang++-18") else: @@ -685,25 +627,6 @@ def main(): cmake_args.append( f"-DSORA_WEBRTC_LDFLAGS={os.path.join(install_dir, 'webrtc.ldflags')}" ) - if platform.target.os == "jetson": - sysroot = os.path.join(install_dir, "rootfs") - cmake_args.append("-DCMAKE_SYSTEM_NAME=Linux") - cmake_args.append("-DCMAKE_SYSTEM_PROCESSOR=aarch64") - cmake_args.append(f"-DCMAKE_SYSROOT={sysroot}") - cmake_args.append("-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu") - cmake_args.append("-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu") - cmake_args.append(f"-DCMAKE_FIND_ROOT_PATH={sysroot}") - cmake_args.append("-DUSE_LIBCXX=ON") - cmake_args.append( - f"-DLIBCXX_INCLUDE_DIR={cmake_path(os.path.join(webrtc_info.libcxx_dir, 'include'))}" - ) - cmake_args.append( - f"-DCMAKE_C_COMPILER={cmake_path(os.path.join(webrtc_info.clang_dir, 'bin', 'clang'))}" - ) - cmake_args.append( - f"-DCMAKE_CXX_COMPILER={cmake_path(os.path.join(webrtc_info.clang_dir, 'bin', 'clang++'))}" - ) - cmake_args.append("-DUSE_JETSON_ENCODER=ON") # NvCodec if platform.target.os in ("windows", "ubuntu") and platform.target.arch == "x86_64": @@ -848,6 +771,7 @@ def main(): if platform.target.package_name in ( "ubuntu-20.04_x86_64", "ubuntu-22.04_x86_64", + "ubuntu-24.04_x86_64", ): cmake_args.append("-DCMAKE_C_COMPILER=clang-18") cmake_args.append("-DCMAKE_CXX_COMPILER=clang++-18") @@ -862,29 +786,6 @@ def main(): cmake_args.append( f"-DLIBCXX_INCLUDE_DIR={cmake_path(os.path.join(webrtc_info.libcxx_dir, 'include'))}" ) - if platform.target.os == "jetson": - sysroot = os.path.join(install_dir, "rootfs") - cmake_args.append("-DJETSON=ON") - cmake_args.append("-DCMAKE_SYSTEM_NAME=Linux") - cmake_args.append("-DCMAKE_SYSTEM_PROCESSOR=aarch64") - cmake_args.append(f"-DCMAKE_SYSROOT={sysroot}") - cmake_args.append("-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu") - cmake_args.append("-DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu") - cmake_args.append(f"-DCMAKE_FIND_ROOT_PATH={sysroot}") - cmake_args.append("-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER") - cmake_args.append("-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH") - cmake_args.append("-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH") - cmake_args.append("-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH") - cmake_args.append("-DUSE_LIBCXX=ON") - cmake_args.append( - f"-DLIBCXX_INCLUDE_DIR={cmake_path(os.path.join(webrtc_info.libcxx_dir, 'include'))}" - ) - cmake_args.append( - f"-DCMAKE_C_COMPILER={cmake_path(os.path.join(webrtc_info.clang_dir, 'bin', 'clang'))}" - ) - cmake_args.append( - f"-DCMAKE_CXX_COMPILER={cmake_path(os.path.join(webrtc_info.clang_dir, 'bin', 'clang++'))}" - ) if platform.target.os in ("windows", "macos", "ubuntu"): cmake_args.append("-DTEST_CONNECT_DISCONNECT=ON") diff --git a/src/camera_device_capturer.cpp b/src/camera_device_capturer.cpp index 2e0f60d6..e1571a07 100644 --- a/src/camera_device_capturer.cpp +++ b/src/camera_device_capturer.cpp @@ -4,8 +4,6 @@ #include "sora/mac/mac_capturer.h" #elif defined(SORA_CPP_SDK_ANDROID) #include "sora/android/android_capturer.h" -#elif defined(SORA_CPP_SDK_JETSON) && defined(USE_JETSON_ENCODER) -#include "sora/hwenc_jetson/jetson_v4l2_capturer.h" #elif (defined(SORA_CPP_SDK_UBUNTU_2004) || \ defined(SORA_CPP_SDK_UBUNTU_2204)) && \ defined(USE_NVCODEC_ENCODER) @@ -33,20 +31,6 @@ CreateCameraDeviceCapturer(const CameraDeviceCapturerConfig& config) { (JNIEnv*)config.jni_env, (jobject)config.application_context, config.signaling_thread, config.width, config.height, config.fps, config.device_name); -#elif defined(SORA_CPP_SDK_JETSON) && defined(USE_JETSON_ENCODER) - sora::V4L2VideoCapturerConfig v4l2_config; - v4l2_config.on_frame = config.on_frame; - v4l2_config.video_device = config.device_name; - v4l2_config.width = config.width; - v4l2_config.height = config.height; - v4l2_config.framerate = config.fps; - v4l2_config.force_i420 = config.force_i420; - v4l2_config.use_native = config.use_native; - if (config.use_native) { - return sora::JetsonV4L2Capturer::Create(v4l2_config); - } else { - return sora::V4L2VideoCapturer::Create(v4l2_config); - } #elif (defined(SORA_CPP_SDK_UBUNTU_2004) || \ defined(SORA_CPP_SDK_UBUNTU_2204)) && \ defined(USE_NVCODEC_ENCODER) diff --git a/src/default_video_formats.cpp b/src/default_video_formats.cpp index 2421e508..3da6a29a 100644 --- a/src/default_video_formats.cpp +++ b/src/default_video_formats.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/src/hwenc_jetson/jetson_buffer.cpp b/src/hwenc_jetson/jetson_buffer.cpp deleted file mode 100644 index d1e41417..00000000 --- a/src/hwenc_jetson/jetson_buffer.cpp +++ /dev/null @@ -1,243 +0,0 @@ -#include "sora/hwenc_jetson/jetson_buffer.h" - -// Linux -#include - -// WebRTC -#include -#include -#include - -// Jetson Linux Multimedia API -#include -#include - -namespace sora { - -static const int kBufferAlignment = 64; - -rtc::scoped_refptr JetsonBuffer::Create( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height, - int fd, - uint32_t pixfmt, - std::shared_ptr decoder) { - return rtc::make_ref_counted(video_type, raw_width, raw_height, - scaled_width, scaled_height, fd, - pixfmt, decoder); -} - -rtc::scoped_refptr JetsonBuffer::Create( - webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height) { - return rtc::make_ref_counted(video_type, raw_width, raw_height, - scaled_width, scaled_height); -} - -webrtc::VideoFrameBuffer::Type JetsonBuffer::type() const { - return Type::kNative; -} - -webrtc::VideoType JetsonBuffer::VideoType() const { - return video_type_; -} - -int JetsonBuffer::width() const { - return scaled_width_; -} - -int JetsonBuffer::height() const { - return scaled_height_; -} - -rtc::scoped_refptr JetsonBuffer::ToI420() { - if (video_type_ == webrtc::VideoType::kMJPEG) { - rtc::scoped_refptr scaled_buffer = - webrtc::I420Buffer::Create(scaled_width_, scaled_height_); - int32_t buffer_width = ((scaled_width_ + 15) / 16) * 16; - int32_t buffer_height = ((scaled_height_ + 15) / 16) * 16; - - NvBufSurfaceAllocateParams input_params = {0}; - input_params.params.width = buffer_width; - input_params.params.height = buffer_height; - input_params.params.layout = NVBUF_LAYOUT_PITCH; - input_params.params.colorFormat = NVBUF_COLOR_FORMAT_YUV420; - input_params.params.memType = NVBUF_MEM_SURFACE_ARRAY; - input_params.memtag = NvBufSurfaceTag_NONE; - - NvBufSurface* dst_surf = 0; - - if (NvBufSurfaceAllocate( - &dst_surf, - 1, /* NvUtils では複数のバッファーを同時に初期化できるため、バッファーの数を指定する */ - &input_params) == -1) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufSurfaceAllocate"; - return scaled_buffer; - } - NvBufSurfaceParams params = dst_surf->surfaceList[0]; - - NvBufSurfTransformRect src_rect, dest_rect; - src_rect.top = 0; - src_rect.left = 0; - src_rect.width = params.width; - src_rect.height = params.height; - dest_rect.top = 0; - dest_rect.left = 0; - dest_rect.width = buffer_width; - dest_rect.height = buffer_height; - - NvBufSurfTransformParams trans_params; - memset(&trans_params, 0, sizeof(trans_params)); - trans_params.transform_flag = NVBUFSURF_TRANSFORM_FILTER; - trans_params.transform_flip = NvBufSurfTransform_None; - trans_params.transform_filter = NvBufSurfTransformInter_Algo3; - trans_params.src_rect = &src_rect; - trans_params.dst_rect = &dest_rect; - - NvBufSurface* src_surf = 0; - if (NvBufSurfaceFromFd(fd_, (void**)(&src_surf)) == -1) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufSurfaceFromFd"; - return scaled_buffer; - } - - if (NvBufSurfTransform(src_surf, dst_surf, &trans_params) != - NvBufSurfTransformError_Success) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufSurfTransform"; - return scaled_buffer; - } - - int ret; - void* data_addr; - uint8_t* dest_addr; - int num_planes = dst_surf->surfaceList->planeParams.num_planes; - int index = 0; - for (int plane = 0; plane < num_planes; plane++) { - ret = NvBufSurfaceMap(dst_surf, index, plane, NVBUF_MAP_READ); - if (ret == 0) { - NvBufSurfaceSyncForCpu(dst_surf, index, plane); - data_addr = dst_surf->surfaceList->mappedAddr.addr[plane]; - int height, width; - if (plane == 0) { - dest_addr = scaled_buffer.get()->MutableDataY(); - width = scaled_width_; - height = scaled_height_; - } else if (plane == 1) { - dest_addr = scaled_buffer.get()->MutableDataU(); - width = (scaled_width_ + 1) >> 1; - height = (scaled_height_ + 1) >> 1; - } else if (plane == 2) { - dest_addr = scaled_buffer.get()->MutableDataV(); - width = (scaled_width_ + 1) >> 1; - height = (scaled_height_ + 1) >> 1; - } - for (int i = 0; i < height; i++) { - memcpy(dest_addr + width * i, - (uint8_t*)data_addr + - dst_surf->surfaceList->planeParams.pitch[plane] * i, - width); - } - } - NvBufSurfaceUnMap(dst_surf, index, plane); - if (ret == -1) { - RTC_LOG(LS_ERROR) << __FUNCTION__ - << " Failed to NvBufSurfaceMap plane=" << plane; - return scaled_buffer; - } - } - - NvBufSurfaceDestroy(dst_surf); - - return scaled_buffer; - } else { - rtc::scoped_refptr i420_buffer = - webrtc::I420Buffer::Create(raw_width_, raw_height_); - const int conversionResult = libyuv::ConvertToI420( - data_.get(), length_, i420_buffer.get()->MutableDataY(), - i420_buffer.get()->StrideY(), i420_buffer.get()->MutableDataU(), - i420_buffer.get()->StrideU(), i420_buffer.get()->MutableDataV(), - i420_buffer.get()->StrideV(), 0, 0, raw_width_, raw_height_, raw_width_, - raw_height_, libyuv::kRotate0, ConvertVideoType(video_type_)); - if (raw_width_ == scaled_width_ && raw_height_ == scaled_height_) { - return i420_buffer; - } - rtc::scoped_refptr scaled_buffer = - webrtc::I420Buffer::Create(scaled_width_, scaled_height_); - scaled_buffer->ScaleFrom(*i420_buffer->ToI420()); - return scaled_buffer; - } -} - -int JetsonBuffer::RawWidth() const { - return raw_width_; -} - -int JetsonBuffer::RawHeight() const { - return raw_height_; -} - -int JetsonBuffer::DecodedFd() const { - return fd_; -} - -uint32_t JetsonBuffer::V4L2PixelFormat() const { - return pixfmt_; -} - -std::shared_ptr JetsonBuffer::JpegDecoder() const { - return decoder_; -} - -uint8_t* JetsonBuffer::Data() const { - return data_.get(); -} - -void JetsonBuffer::SetLength(size_t length) { - length_ = length; -} - -size_t JetsonBuffer::Length() const { - return length_; -} - -JetsonBuffer::JetsonBuffer(webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height, - int fd, - uint32_t pixfmt, - std::shared_ptr decoder) - : video_type_(video_type), - raw_width_(raw_width), - raw_height_(raw_height), - scaled_width_(scaled_width), - scaled_height_(scaled_height), - fd_(fd), - pixfmt_(pixfmt), - decoder_(decoder), - data_(nullptr) {} - -JetsonBuffer::JetsonBuffer(webrtc::VideoType video_type, - int raw_width, - int raw_height, - int scaled_width, - int scaled_height) - : video_type_(video_type), - raw_width_(raw_width), - raw_height_(raw_height), - scaled_width_(scaled_width), - scaled_height_(scaled_height), - fd_(-1), - pixfmt_(0), - decoder_(nullptr), - data_(static_cast(webrtc::AlignedMalloc( - webrtc::CalcBufferSize(video_type, raw_width, raw_height), - kBufferAlignment))) {} - -} // namespace sora diff --git a/src/hwenc_jetson/jetson_jpeg_decoder.cpp b/src/hwenc_jetson/jetson_jpeg_decoder.cpp deleted file mode 100644 index bfd048bd..00000000 --- a/src/hwenc_jetson/jetson_jpeg_decoder.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "sora/hwenc_jetson/jetson_jpeg_decoder.h" - -// Jetson Linux Multimedia API -#include - -namespace sora { - -JetsonJpegDecoder::JetsonJpegDecoder( - std::shared_ptr pool, - std::shared_ptr decoder) - : pool_(pool), decoder_(std::move(decoder)) {} - -JetsonJpegDecoder::~JetsonJpegDecoder() { - pool_->Push(std::move(decoder_)); -} - -int JetsonJpegDecoder::DecodeToFd(int& fd, - unsigned char* in_buf, - unsigned long in_buf_size, - uint32_t& pixfmt, - uint32_t& width, - uint32_t& height) { - return decoder_->decodeToFd(fd, in_buf, in_buf_size, pixfmt, width, height); -} - -} // namespace sora diff --git a/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp b/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp deleted file mode 100644 index f0b33176..00000000 --- a/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "sora/hwenc_jetson/jetson_jpeg_decoder_pool.h" - -// WebRTC -#include - -// Jetson Linux Multimedia API -#include - -namespace sora { - -std::shared_ptr JetsonJpegDecoderPool::Pop() { - std::shared_ptr nv_decoder; - - // JetPack 5.1.2 で同じフレームが送信され続ける問題が発生したため、キューを無効化した - // JetPack 5.1.1 では正常に動作していた - // momo で同様の問題に対応した際の PR: https://github.com/shiguredo/momo/pull/297/ - // { - // std::lock_guard lock(mtx_); - // if (decoder_queue_.size() == 0) { - // nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec")); - // } else { - // nv_decoder = std::move(decoder_queue_.front()); - // decoder_queue_.pop(); - // } - // } - nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec")); - - std::shared_ptr decoder( - new JetsonJpegDecoder(shared_from_this(), std::move(nv_decoder))); - return decoder; -} - -void JetsonJpegDecoderPool::Push(std::shared_ptr decoder) { - std::lock_guard lock(mtx_); - // decoder_queue_.push(std::move(decoder)); -} - -} // namespace sora diff --git a/src/hwenc_jetson/jetson_v4l2_capturer.cpp b/src/hwenc_jetson/jetson_v4l2_capturer.cpp deleted file mode 100644 index 3aaf687d..00000000 --- a/src/hwenc_jetson/jetson_v4l2_capturer.cpp +++ /dev/null @@ -1,614 +0,0 @@ -#include "sora/hwenc_jetson/jetson_v4l2_capturer.h" - -// C -#include -#include -#include - -// C++ -#include -#include - -// Linux -#include -#include -#include -#include -#include -#include -#include - -// WebRTC -#include -#include -#include -#include -#include -#include -#include -#include - -// L4T Multimedia API -#include - -#include "sora/hwenc_jetson/jetson_buffer.h" - -#define MJPEG_EOS_SEARCH_SIZE 4096 - -namespace sora { - -rtc::scoped_refptr JetsonV4L2Capturer::Create( - const V4L2VideoCapturerConfig& config) { - rtc::scoped_refptr capturer; - std::unique_ptr device_info( - webrtc::VideoCaptureFactory::CreateDeviceInfo()); - if (!device_info) { - RTC_LOG(LS_ERROR) << "Failed to CreateDeviceInfo"; - return nullptr; - } - - LogDeviceList(device_info.get()); - - for (int i = 0; i < device_info->NumberOfDevices(); ++i) { - capturer = Create(device_info.get(), config, i); - if (capturer) { - RTC_LOG(LS_INFO) << "Get Capture"; - return capturer; - } - } - RTC_LOG(LS_ERROR) << "Failed to create JetsonV4L2Capturer"; - return nullptr; -} - -void JetsonV4L2Capturer::LogDeviceList( - webrtc::VideoCaptureModule::DeviceInfo* device_info) { - for (int i = 0; i < device_info->NumberOfDevices(); ++i) { - char device_name[256]; - char unique_name[256]; - if (device_info->GetDeviceName(static_cast(i), device_name, - sizeof(device_name), unique_name, - sizeof(unique_name)) != 0) { - RTC_LOG(LS_WARNING) << "Failed to GetDeviceName(" << i << ")"; - continue; - } - RTC_LOG(LS_INFO) << "GetDeviceName(" << i - << "): device_name=" << device_name - << ", unique_name=" << unique_name; - } -} - -rtc::scoped_refptr JetsonV4L2Capturer::Create( - webrtc::VideoCaptureModule::DeviceInfo* device_info, - const V4L2VideoCapturerConfig& config, - size_t capture_device_index) { - char device_name[256]; - char unique_name[256]; - if (device_info->GetDeviceName(static_cast(capture_device_index), - device_name, sizeof(device_name), unique_name, - sizeof(unique_name)) != 0) { - RTC_LOG(LS_WARNING) << "Failed to GetDeviceName"; - return nullptr; - } - rtc::scoped_refptr v4l2_capturer = - rtc::make_ref_counted(config); - if (v4l2_capturer->Init((const char*)&unique_name, config.video_device) < 0) { - RTC_LOG(LS_WARNING) << "Failed to create JetsonV4L2Capturer(" << unique_name - << ")"; - return nullptr; - } - if (v4l2_capturer->StartCapture(config) < 0) { - RTC_LOG(LS_WARNING) << "Failed to start JetsonV4L2Capturer(w = " - << config.width << ", h = " << config.height - << ", fps = " << config.framerate << ")"; - return nullptr; - } - return v4l2_capturer; -} - -JetsonV4L2Capturer::JetsonV4L2Capturer(const V4L2VideoCapturerConfig& config) - : ScalableVideoTrackSource(config), - _deviceFd(-1), - _buffersAllocatedByDevice(-1), - _currentWidth(-1), - _currentHeight(-1), - _currentFrameRate(-1), - _captureStarted(false), - _captureVideoType(webrtc::VideoType::kI420), - _pool(NULL) {} - -bool JetsonV4L2Capturer::FindDevice(const char* deviceUniqueIdUTF8, - const std::string& device) { - int fd; - if ((fd = open(device.c_str(), O_RDONLY)) != -1) { - // query device capabilities - struct v4l2_capability cap; - if (ioctl(fd, VIDIOC_QUERYCAP, &cap) == 0) { - if (cap.bus_info[0] != 0) { - if (strncmp((const char*)cap.bus_info, (const char*)deviceUniqueIdUTF8, - strlen((const char*)deviceUniqueIdUTF8)) == - 0) // match with device id - { - close(fd); - return true; - } - } - } - close(fd); // close since this is not the matching device - } - return false; -} - -int32_t JetsonV4L2Capturer::Init(const char* deviceUniqueIdUTF8, - const std::string& specifiedVideoDevice) { - int fd; - bool found = false; - - if (!specifiedVideoDevice.empty()) { - // specifiedVideoDevice が指定されてる場合はそれだけ調べる - if (FindDevice(deviceUniqueIdUTF8, specifiedVideoDevice)) { - found = true; - _videoDevice = specifiedVideoDevice; - } - } else { - // specifiedVideoDevice が指定されてない場合は頑張って探す - /* detect /dev/video [0-63] entries */ - char device[32]; - int n; - for (n = 0; n < 64; n++) { - sprintf(device, "/dev/video%d", n); - if (FindDevice(deviceUniqueIdUTF8, device)) { - found = true; - _videoDevice = device; // store the video device - break; - } - } - } - - if (!found) { - RTC_LOG(LS_INFO) << "no matching device found"; - return -1; - } - return 0; -} - -JetsonV4L2Capturer::~JetsonV4L2Capturer() { - StopCapture(); - if (_deviceFd != -1) - close(_deviceFd); -} - -int32_t JetsonV4L2Capturer::StartCapture( - const V4L2VideoCapturerConfig& config) { - if (_captureStarted) { - if (config.width == _currentWidth && config.height == _currentHeight) { - return 0; - } else { - StopCapture(); - } - } - - webrtc::MutexLock lock(&capture_lock_); - // first open /dev/video device - if ((_deviceFd = open(_videoDevice.c_str(), O_RDWR | O_NONBLOCK, 0)) < 0) { - RTC_LOG(LS_INFO) << "error in opening " << _videoDevice - << " errono = " << errno; - return -1; - } - - // Supported video formats in preferred order. - // If the requested resolution is larger than VGA, we prefer MJPEG. Go for - // I420 otherwise. - const int nFormats = 6; - unsigned int fmts[nFormats] = {}; - if (config.use_native) { - fmts[0] = V4L2_PIX_FMT_MJPEG; - fmts[1] = V4L2_PIX_FMT_JPEG; - } else if (!config.force_i420 && - (config.width > 640 || config.height > 480)) { - fmts[0] = V4L2_PIX_FMT_MJPEG; - fmts[1] = V4L2_PIX_FMT_YUV420; - fmts[2] = V4L2_PIX_FMT_YVU420; - fmts[3] = V4L2_PIX_FMT_YUYV; - fmts[4] = V4L2_PIX_FMT_UYVY; - fmts[5] = V4L2_PIX_FMT_JPEG; - } else { - fmts[0] = V4L2_PIX_FMT_YUV420; - fmts[1] = V4L2_PIX_FMT_YVU420; - fmts[2] = V4L2_PIX_FMT_YUYV; - fmts[3] = V4L2_PIX_FMT_UYVY; - fmts[4] = V4L2_PIX_FMT_MJPEG; - fmts[5] = V4L2_PIX_FMT_JPEG; - } - - // Enumerate image formats. - struct v4l2_fmtdesc fmt; - int fmtsIdx = nFormats; - memset(&fmt, 0, sizeof(fmt)); - fmt.index = 0; - fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - RTC_LOG(LS_INFO) << "Video Capture enumerats supported image formats:"; - while (ioctl(_deviceFd, VIDIOC_ENUM_FMT, &fmt) == 0) { - RTC_LOG(LS_INFO) << " { pixelformat = " - << cricket::GetFourccName(fmt.pixelformat) - << ", description = '" << fmt.description << "' }"; - // Match the preferred order. - for (int i = 0; i < nFormats; i++) { - if (fmt.pixelformat == fmts[i] && i < fmtsIdx) - fmtsIdx = i; - } - // Keep enumerating. - fmt.index++; - } - - if (fmtsIdx == nFormats) { - RTC_LOG(LS_INFO) << "no supporting video formats found"; - return -1; - } else { - RTC_LOG(LS_INFO) << "We prefer format " - << cricket::GetFourccName(fmts[fmtsIdx]); - } - - struct v4l2_format video_fmt; - memset(&video_fmt, 0, sizeof(struct v4l2_format)); - video_fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - video_fmt.fmt.pix.sizeimage = 0; - video_fmt.fmt.pix.width = config.width; - video_fmt.fmt.pix.height = config.height; - video_fmt.fmt.pix.pixelformat = fmts[fmtsIdx]; - - // set format and frame size now - if (ioctl(_deviceFd, VIDIOC_S_FMT, &video_fmt) < 0) { - RTC_LOG(LS_INFO) << "error in VIDIOC_S_FMT, errno = " << errno; - return -1; - } - - // initialize current width and height - _currentWidth = video_fmt.fmt.pix.width; - _currentHeight = video_fmt.fmt.pix.height; - _currentPixelFormat = video_fmt.fmt.pix.pixelformat; - - if (video_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_YUYV) - _captureVideoType = webrtc::VideoType::kYUY2; - else if (video_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_YUV420) - _captureVideoType = webrtc::VideoType::kI420; - else if (video_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_YVU420) - _captureVideoType = webrtc::VideoType::kYV12; - else if (video_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_UYVY) - _captureVideoType = webrtc::VideoType::kUYVY; - else if (video_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG || - video_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_JPEG) - _captureVideoType = webrtc::VideoType::kMJPEG; - - // Trying to set frame rate, before check driver capability. - bool driver_framerate_support = true; - struct v4l2_streamparm streamparms; - memset(&streamparms, 0, sizeof(streamparms)); - streamparms.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (ioctl(_deviceFd, VIDIOC_G_PARM, &streamparms) < 0) { - RTC_LOG(LS_INFO) << "error in VIDIOC_G_PARM errno = " << errno; - driver_framerate_support = false; - // continue - } else { - // check the capability flag is set to V4L2_CAP_TIMEPERFRAME. - if (streamparms.parm.capture.capability & V4L2_CAP_TIMEPERFRAME) { - // driver supports the feature. Set required framerate. - memset(&streamparms, 0, sizeof(streamparms)); - streamparms.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - streamparms.parm.capture.timeperframe.numerator = 1; - streamparms.parm.capture.timeperframe.denominator = config.framerate; - if (ioctl(_deviceFd, VIDIOC_S_PARM, &streamparms) < 0) { - RTC_LOG(LS_INFO) << "Failed to set the framerate. errno=" << errno; - driver_framerate_support = false; - } else { - _currentFrameRate = config.framerate; - } - } - } - // If driver doesn't support framerate control, need to hardcode. - // Hardcoding the value based on the frame size. - if (!driver_framerate_support) { - if (!config.use_native && _currentWidth >= 800 && - _captureVideoType != webrtc::VideoType::kMJPEG) { - _currentFrameRate = 15; - } else { - _currentFrameRate = 30; - } - } - - if (!AllocateVideoBuffers()) { - RTC_LOG(LS_INFO) << "failed to allocate video capture buffers"; - return -1; - } - - // start capture thread; - if (_captureThread.empty()) { - quit_ = false; - _captureThread = rtc::PlatformThread::SpawnJoinable( - std::bind(JetsonV4L2Capturer::CaptureThread, this), "CaptureThread", - rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kHigh)); - } - - // Needed to start UVC camera - from the uvcview application - enum v4l2_buf_type type; - type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (ioctl(_deviceFd, VIDIOC_STREAMON, &type) == -1) { - RTC_LOG(LS_INFO) << "Failed to turn on stream"; - return -1; - } - - _captureStarted = true; - return 0; -} - -int32_t JetsonV4L2Capturer::StopCapture() { - if (!_captureThread.empty()) { - { - webrtc::MutexLock lock(&capture_lock_); - quit_ = true; - } - _captureThread.Finalize(); - } - - webrtc::MutexLock lock(&capture_lock_); - if (_captureStarted) { - _captureStarted = false; - - DeAllocateVideoBuffers(); - close(_deviceFd); - _deviceFd = -1; - } - - return 0; -} - -// critical section protected by the caller - -bool JetsonV4L2Capturer::AllocateVideoBuffers() { - struct v4l2_requestbuffers rbuffer; - memset(&rbuffer, 0, sizeof(v4l2_requestbuffers)); - - rbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - rbuffer.memory = V4L2_MEMORY_MMAP; - rbuffer.count = kNoOfV4L2Bufffers; - - if (ioctl(_deviceFd, VIDIOC_REQBUFS, &rbuffer) < 0) { - RTC_LOG(LS_INFO) << "Could not get buffers from device. errno = " << errno; - return false; - } - - if (rbuffer.count > kNoOfV4L2Bufffers) - rbuffer.count = kNoOfV4L2Bufffers; - - _buffersAllocatedByDevice = rbuffer.count; - - std::unique_ptr fds; - if (_captureVideoType != webrtc::VideoType::kMJPEG) { - fds.reset(new int[rbuffer.count]); - NvBufSurf::NvCommonAllocateParams params = {0}; - - params.memType = NVBUF_MEM_SURFACE_ARRAY; - params.width = _currentWidth; - params.height = _currentHeight; - params.layout = NVBUF_LAYOUT_PITCH; - if (_captureVideoType == webrtc::VideoType::kYUY2) - params.colorFormat = NVBUF_COLOR_FORMAT_YUYV; - else if (_captureVideoType == webrtc::VideoType::kI420) - params.colorFormat = NVBUF_COLOR_FORMAT_YUV420; - else if (_captureVideoType == webrtc::VideoType::kYV12) - params.colorFormat = NVBUF_COLOR_FORMAT_YVU420; - else if (_captureVideoType == webrtc::VideoType::kUYVY) - params.colorFormat = NVBUF_COLOR_FORMAT_UYVY; - params.memtag = NvBufSurfaceTag_CAMERA; - if (NvBufSurf::NvAllocate(¶ms, rbuffer.count, fds.get())) { - return false; - } - } - - // Map the buffers - _pool = new Buffer[rbuffer.count]; - - for (unsigned int i = 0; i < rbuffer.count; i++) { - struct v4l2_buffer buffer; - memset(&buffer, 0, sizeof(v4l2_buffer)); - if (fds == nullptr) { - buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buffer.memory = V4L2_MEMORY_MMAP; - buffer.index = i; - - if (ioctl(_deviceFd, VIDIOC_QUERYBUF, &buffer) < 0) { - return false; - } - - _pool[i].start = mmap(NULL, buffer.length, PROT_READ | PROT_WRITE, - MAP_SHARED, _deviceFd, buffer.m.offset); - - if (MAP_FAILED == _pool[i].start) { - for (unsigned int j = 0; j < i; j++) - munmap(_pool[j].start, _pool[j].length); - return false; - } - - _pool[i].length = buffer.length; - } else { - buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buffer.memory = V4L2_MEMORY_DMABUF; - buffer.index = i; - - if (ioctl(_deviceFd, VIDIOC_QUERYBUF, &buffer) < 0) { - return false; - } - - _pool[i].fd = fds[i]; - _pool[i].length = buffer.length; - } - - if (ioctl(_deviceFd, VIDIOC_QBUF, &buffer) < 0) { - return false; - } - } - - if (_captureVideoType == webrtc::VideoType::kMJPEG) { - jpeg_decoder_pool_.reset(new JetsonJpegDecoderPool()); - } - - return true; -} - -bool JetsonV4L2Capturer::DeAllocateVideoBuffers() { - if (_captureVideoType == webrtc::VideoType::kMJPEG) { - jpeg_decoder_pool_ = nullptr; - - // unmap buffers - for (int i = 0; i < _buffersAllocatedByDevice; i++) { - munmap(_pool[i].start, _pool[i].length); - } - delete[] _pool; - - } else { - for (int i = 0; i < _buffersAllocatedByDevice; i++) { - NvBufSurf::NvDestroy(_pool[i].fd); - } - delete[] _pool; - } - - // turn off stream - enum v4l2_buf_type type; - type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (ioctl(_deviceFd, VIDIOC_STREAMOFF, &type) < 0) { - RTC_LOG(LS_INFO) << "VIDIOC_STREAMOFF error. errno: " << errno; - } - - return true; -} - -void JetsonV4L2Capturer::CaptureThread(void* obj) { - JetsonV4L2Capturer* capturer = static_cast(obj); - while (capturer->CaptureProcess()) { - } -} - -bool JetsonV4L2Capturer::CaptureProcess() { - int retVal = 0; - fd_set rSet; - struct timeval timeout; - - FD_ZERO(&rSet); - FD_SET(_deviceFd, &rSet); - timeout.tv_sec = 1; - timeout.tv_usec = 0; - - // _deviceFd written only in StartCapture, when this thread isn't running. - retVal = select(_deviceFd + 1, &rSet, NULL, NULL, &timeout); - { - webrtc::MutexLock lock(&capture_lock_); - - if (quit_) { - return false; - } else if (retVal < 0 && errno != EINTR /* continue if interrupted */) { - // select failed - return false; - } else if (retVal == 0) { - // select timed out - return true; - } else if (!FD_ISSET(_deviceFd, &rSet)) { - // not event on camera handle - return true; - } - - if (_captureStarted) { - struct v4l2_buffer buf; - memset(&buf, 0, sizeof(struct v4l2_buffer)); - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (_captureVideoType == webrtc::VideoType::kMJPEG) { - buf.memory = V4L2_MEMORY_MMAP; - } else { - buf.memory = V4L2_MEMORY_DMABUF; - } - // dequeue a buffer - repeat until dequeued properly! - while (ioctl(_deviceFd, VIDIOC_DQBUF, &buf) < 0) { - if (errno != EINTR) { - RTC_LOG(LS_INFO) << "could not sync on a buffer on device " - << strerror(errno); - return true; - } - } - - OnCaptured(&buf); - - // enqueue the buffer again - if (ioctl(_deviceFd, VIDIOC_QBUF, &buf) == -1) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Failed to enqueue capture buffer"; - } - } - } - usleep(0); - return true; -} - -void JetsonV4L2Capturer::OnCaptured(v4l2_buffer* buf) { - const int64_t timestamp_us = rtc::TimeMicros(); - int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; - if (!AdaptFrame(_currentWidth, _currentHeight, timestamp_us, &adapted_width, - &adapted_height, &crop_width, &crop_height, &crop_x, - &crop_y)) { - return; - } - - if (_captureVideoType == webrtc::VideoType::kMJPEG) { - uint8_t* data = (uint8_t*)_pool[buf->index].start; - uint32_t bytesused = buf->bytesused; - // 一部のカメラ (DELL WB7022) は不正なデータを送ってくることがある。 - // これをハードウェアJPEGデコーダーに送ると Momo ごとクラッシュしてしまう。 - // JPEG の先頭は SOI マーカー 0xffd8 で始まるのでチェックして落ちないようにする。 - if (bytesused < 2 || data[0] != 0xff || data[1] != 0xd8) { - RTC_LOG(LS_WARNING) << __FUNCTION__ - << " Invalid JPEG buffer frame skipped"; - return; - } - - unsigned int eosSearchSize = MJPEG_EOS_SEARCH_SIZE; - uint8_t* p; - /* v4l2_buf.bytesused may have padding bytes for alignment - Search for EOF to get exact size */ - if (eosSearchSize > bytesused) - eosSearchSize = bytesused; - for (unsigned int i = 0; i < eosSearchSize; i++) { - p = data + bytesused; - if ((*(p - 2) == 0xff) && (*(p - 1) == 0xd9)) { - break; - } - bytesused--; - } - - std::shared_ptr decoder = jpeg_decoder_pool_->Pop(); - int fd = 0; - uint32_t width, height, pixfmt; - if (decoder->DecodeToFd(fd, data, bytesused, pixfmt, width, height) < 0) { - RTC_LOG(LS_ERROR) << "decodeToFd Failed"; - return; - } - rtc::scoped_refptr jetson_buffer( - JetsonBuffer::Create(_captureVideoType, width, height, adapted_width, - adapted_height, fd, pixfmt, std::move(decoder))); - OnFrame(webrtc::VideoFrame::Builder() - .set_video_frame_buffer(jetson_buffer) - .set_timestamp_rtp(0) - .set_timestamp_ms(rtc::TimeMillis()) - .set_timestamp_us(rtc::TimeMicros()) - .set_rotation(webrtc::kVideoRotation_0) - .build()); - } else { - rtc::scoped_refptr jetson_buffer(JetsonBuffer::Create( - _captureVideoType, _currentWidth, _currentHeight, adapted_width, - adapted_height, _pool[buf->index].fd, _currentPixelFormat, nullptr)); - OnFrame(webrtc::VideoFrame::Builder() - .set_video_frame_buffer(jetson_buffer) - .set_timestamp_rtp(0) - .set_timestamp_ms(rtc::TimeMillis()) - .set_timestamp_us(rtc::TimeMicros()) - .set_rotation(webrtc::kVideoRotation_0) - .build()); - } -} - -} // namespace sora \ No newline at end of file diff --git a/src/hwenc_jetson/jetson_video_decoder.cpp b/src/hwenc_jetson/jetson_video_decoder.cpp deleted file mode 100644 index 8b2b5762..00000000 --- a/src/hwenc_jetson/jetson_video_decoder.cpp +++ /dev/null @@ -1,466 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#include "sora/hwenc_jetson/jetson_video_decoder.h" - -#include - -// WebRTC -#include -#include -#include -#include -#include -#include - -// L4T Multimedia API -#include -#include - -// Jetson Linux Multimedia API -#include -#include - -#define INIT_ERROR(cond, desc) \ - if (cond) { \ - RTC_LOG(LS_ERROR) << __FUNCTION__ << desc; \ - Release(); \ - return WEBRTC_VIDEO_CODEC_ERROR; \ - } -#define CHUNK_SIZE 4000000 - -namespace sora { - -JetsonVideoDecoder::JetsonVideoDecoder(webrtc::VideoCodecType codec) - : input_format_(codec == webrtc::kVideoCodecVP8 ? V4L2_PIX_FMT_VP8 - : codec == webrtc::kVideoCodecVP9 ? V4L2_PIX_FMT_VP9 - : codec == webrtc::kVideoCodecH264 ? V4L2_PIX_FMT_H264 - : codec == webrtc::kVideoCodecAV1 ? V4L2_PIX_FMT_AV1 - : 0), - decoder_(nullptr), - decode_complete_callback_(nullptr), - buffer_pool_(false, 300 /* max_number_of_buffers*/), - eos_(false), - got_error_(false), - dst_dma_fd_(-1) {} - -JetsonVideoDecoder::~JetsonVideoDecoder() { - Release(); -} - -bool JetsonVideoDecoder::IsSupportedVP8() { - //SuppressErrors sup; - - auto decoder = NvVideoDecoder::createVideoDecoder("dec0"); - auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_VP8, CHUNK_SIZE); - delete decoder; - return ret >= 0; -} - -bool JetsonVideoDecoder::IsSupportedAV1() { - //SuppressErrors sup; - - auto decoder = NvVideoDecoder::createVideoDecoder("dec0"); - auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_AV1, CHUNK_SIZE); - delete decoder; - return ret >= 0; -} - -bool JetsonVideoDecoder::Configure(const Settings& settings) { - if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to JetsonConfigure"; - return false; - } - return true; -} - -int32_t JetsonVideoDecoder::Decode(const webrtc::EncodedImage& input_image, - bool missing_frames, - int64_t render_time_ms) { - if (decoder_ == nullptr) { - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - if (decode_complete_callback_ == NULL) { - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - if (input_image.data() == NULL && input_image.size() > 0) { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - NvBuffer* buffer; - - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, sizeof(planes)); - v4l2_buf.m.planes = planes; - - RTC_LOG(LS_INFO) << __FUNCTION__ << " output_plane.getNumBuffers: " - << decoder_->output_plane.getNumBuffers() - << " output_plane.getNumQueuedBuffers: " - << decoder_->output_plane.getNumQueuedBuffers(); - - if (decoder_->output_plane.getNumQueuedBuffers() == - decoder_->output_plane.getNumBuffers()) { - if (decoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, -1) < 0) { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at decoder output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - } else { - buffer = decoder_->output_plane.getNthBuffer( - decoder_->output_plane.getNumQueuedBuffers()); - v4l2_buf.index = decoder_->output_plane.getNumQueuedBuffers(); - } - - memcpy(buffer->planes[0].data, input_image.data(), input_image.size()); - buffer->planes[0].bytesused = input_image.size(); - - v4l2_buf.m.planes[0].bytesused = buffer->planes[0].bytesused; - - v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; - v4l2_buf.timestamp.tv_sec = - input_image.RtpTimestamp() / rtc::kNumMicrosecsPerSec; - v4l2_buf.timestamp.tv_usec = - input_image.RtpTimestamp() % rtc::kNumMicrosecsPerSec; - - if (decoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { - RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - RTC_LOG(LS_INFO) << __FUNCTION__ << " timestamp:" << input_image.RtpTimestamp() - << " bytesused:" << buffer->planes[0].bytesused; - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t JetsonVideoDecoder::RegisterDecodeCompleteCallback( - webrtc::DecodedImageCallback* callback) { - decode_complete_callback_ = callback; - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t JetsonVideoDecoder::Release() { - JetsonRelease(); - buffer_pool_.Release(); - return WEBRTC_VIDEO_CODEC_OK; -} - -const char* JetsonVideoDecoder::ImplementationName() const { - return "Jetson Video"; -} - -int32_t JetsonVideoDecoder::JetsonConfigure() { - int ret = 0; - - decoder_ = NvVideoDecoder::createVideoDecoder("dec0"); - INIT_ERROR(!decoder_, "Failed to createVideoDecoder"); - - ret = decoder_->subscribeEvent(V4L2_EVENT_RESOLUTION_CHANGE, 0, 0); - INIT_ERROR(ret < 0, - "Failed to decoder subscribeEvent V4L2_EVENT_RESOLUTION_CHANGE"); - - ret = decoder_->setOutputPlaneFormat(input_format_, CHUNK_SIZE); - INIT_ERROR(ret < 0, "Failed to decoder setOutputPlaneFormat"); - - // 入力されるのがフレーム単位でない場合は 1 を設定する - ret = decoder_->setFrameInputMode(1); - INIT_ERROR(ret < 0, "Failed to decoder setFrameInputMode"); - - ret = decoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); - INIT_ERROR(ret < 0, "Failed to setupPlane at decoder output_plane"); - - ret = decoder_->subscribeEvent(V4L2_EVENT_EOS, 0, 0); - INIT_ERROR(ret < 0, "Failed to subscribeEvent V4L2_EVENT_EOS"); - - ret = decoder_->output_plane.setStreamStatus(true); - INIT_ERROR(ret < 0, "Failed to setStreamStatus at decoder output_plane"); - - if (capture_loop_.empty()) { - eos_ = false; - capture_loop_ = rtc::PlatformThread::SpawnJoinable( - std::bind(JetsonVideoDecoder::CaptureLoopFunction, this), "CaptureLoop", - rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kHigh)); - } - - return WEBRTC_VIDEO_CODEC_OK; -} - -bool JetsonVideoDecoder::JetsonRelease() { - if (decoder_) { - if (!capture_loop_.empty()) { - eos_ = true; - SendEOS(decoder_); - while (decoder_->output_plane.getNumQueuedBuffers() > 0 && !got_error_ && - !decoder_->isInError()) { - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, sizeof(planes)); - - v4l2_buf.m.planes = planes; - if (decoder_->output_plane.dqBuffer(v4l2_buf, NULL, NULL, -1) < 0) { - RTC_LOG(LS_ERROR) - << __FUNCTION__ << " Failed to dqBuffer at decoder output_plane"; - got_error_ = true; - break; - } - } - capture_loop_.Finalize(); - } - delete decoder_; - decoder_ = nullptr; - } - if (dst_dma_fd_ != -1) { - NvBufSurf::NvDestroy(dst_dma_fd_); - dst_dma_fd_ = -1; - } - return true; -} - -void JetsonVideoDecoder::SendEOS(NvV4l2Element* element) { - if (element->output_plane.getStreamStatus()) { - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - NvBuffer* buffer; - - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); - v4l2_buf.m.planes = planes; - - while (element->output_plane.getNumQueuedBuffers() != 0) { - if (element->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; - } - } - planes[0].bytesused = 0; - if (element->output_plane.qBuffer(v4l2_buf, NULL) < 0) { - RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; - } - } -} - -void JetsonVideoDecoder::CaptureLoopFunction(void* obj) { - JetsonVideoDecoder* _this = static_cast(obj); - _this->CaptureLoop(); -} - -void JetsonVideoDecoder::CaptureLoop() { - struct v4l2_event event; - int ret; - do { - ret = decoder_->dqEvent(event, 10); - if (eos_) { - return; - } - if (ret < 0) { - if (errno == EAGAIN) { - continue; - } else { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to dqEvent at decoder"; - got_error_ = true; - break; - } - } - } while (event.type != V4L2_EVENT_RESOLUTION_CHANGE && !got_error_); - - if (!got_error_) { - SetCapture(); - } - - while (!(eos_ || got_error_ || decoder_->isInError())) { - ret = decoder_->dqEvent(event, false); - if (ret == 0 && event.type == V4L2_EVENT_RESOLUTION_CHANGE) { - SetCapture(); - continue; - } - - NvBuffer* buffer; - - while (1) { - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, sizeof(planes)); - v4l2_buf.m.planes = planes; - - // Dequeue a filled buffer - if (decoder_->capture_plane.dqBuffer(v4l2_buf, &buffer, NULL, 0)) { - if (errno == EAGAIN) { - usleep(1000); - } else { - RTC_LOG(LS_ERROR) - << __FUNCTION__ << " Failed to dqBuffer at decoder capture_plane"; - got_error_ = true; - } - break; - } - - uint64_t pts = v4l2_buf.timestamp.tv_sec * rtc::kNumMicrosecsPerSec + - v4l2_buf.timestamp.tv_usec; - - NvBufSurf::NvCommonTransformParams transform_params; - memset(&transform_params, 0, sizeof(transform_params)); - transform_params.src_top = capture_crop_->c.top; - transform_params.src_left = capture_crop_->c.left; - transform_params.src_width = capture_crop_->c.width; - transform_params.src_height = capture_crop_->c.height; - transform_params.dst_top = 0; - transform_params.dst_left = 0; - transform_params.dst_width = capture_crop_->c.width; - transform_params.dst_height = capture_crop_->c.height; - transform_params.flag = NVBUFSURF_TRANSFORM_FILTER; - transform_params.flip = NvBufSurfTransform_None; - transform_params.filter = NvBufSurfTransformInter_Algo3; - - // 何が来ても YUV420 に変換する - ret = NvBufSurf::NvTransform(&transform_params, buffer->planes[0].fd, - dst_dma_fd_); - if (ret == -1) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Transform failed"; - break; - } - - rtc::scoped_refptr i420_buffer = - buffer_pool_.CreateI420Buffer(capture_crop_->c.width, - capture_crop_->c.height); - if (!i420_buffer.get()) { - // Pool has too many pending frames. - RTC_HISTOGRAM_BOOLEAN( - "WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames", 1); - got_error_ = true; - break; - } - - void* src_data; - uint8_t* dst_data; - int dst_stride; - for (uint32_t i = 0; i < MAX_PLANES; i++) { - if (i == 0) { - dst_data = i420_buffer->MutableDataY(); - dst_stride = i420_buffer->StrideY(); - } else if (i == 1) { - dst_data = i420_buffer->MutableDataU(); - dst_stride = i420_buffer->StrideU(); - } else if (i == 2) { - dst_data = i420_buffer->MutableDataV(); - dst_stride = i420_buffer->StrideV(); - } else { - break; - } - - NvBufSurface* dst_surf = 0; - if (NvBufSurfaceFromFd(dst_dma_fd_, (void**)(&dst_surf)) == -1) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to NvBufSurfaceFromFd"; - break; - } - - ret = NvBufSurfaceMap(dst_surf, 0, i, NVBUF_MAP_READ); - NvBufSurfaceSyncForCpu(dst_surf, 0, i); - src_data = dst_surf->surfaceList[0].mappedAddr.addr[i]; - - NvBufSurfacePlaneParams params = dst_surf->surfaceList[0].planeParams; - for (uint32_t j = 0; j < params.height[i]; j++) { - memcpy(dst_data + j * dst_stride, - (char*)src_data + j * params.pitch[i], params.width[i]); - } - NvBufSurfaceUnMap(dst_surf, 0, i); - } - - webrtc::VideoFrame decoded_image = - webrtc::VideoFrame::Builder() - .set_video_frame_buffer(i420_buffer) - .set_timestamp_rtp(pts) - .build(); - decode_complete_callback_->Decoded(decoded_image, absl::nullopt, - absl::nullopt); - - if (decoder_->capture_plane.qBuffer(v4l2_buf, NULL) < 0) { - RTC_LOG(LS_ERROR) << __FUNCTION__ - << "Failed to qBuffer at capture_plane"; - got_error_ = true; - break; - } - } - } -} - -int JetsonVideoDecoder::SetCapture() { - int32_t ret; - - struct v4l2_format format; - ret = decoder_->capture_plane.getFormat(format); - INIT_ERROR(ret < 0, "Failed to getFormat at capture_plane"); - - capture_crop_.reset(new v4l2_crop()); - ret = decoder_->capture_plane.getCrop(*capture_crop_.get()); - INIT_ERROR(ret < 0, "Failed to getCrop at capture_plane"); - - RTC_LOG(LS_INFO) << __FUNCTION__ << " plane format " - << format.fmt.pix_mp.pixelformat << " " - << format.fmt.pix_mp.width << "x" - << format.fmt.pix_mp.height; - RTC_LOG(LS_INFO) << __FUNCTION__ << " crop " << capture_crop_->c.top << "x" - << capture_crop_->c.left << " " << capture_crop_->c.width - << "x" << format.fmt.pix_mp.height; - - if (dst_dma_fd_ != -1) { - NvBufSurf::NvDestroy(dst_dma_fd_); - dst_dma_fd_ = -1; - } - - NvBufSurf::NvCommonAllocateParams cParams; - cParams.width = capture_crop_->c.width; - cParams.height = capture_crop_->c.height; - cParams.layout = NVBUF_LAYOUT_PITCH; - cParams.colorFormat = NVBUF_COLOR_FORMAT_YUV420; - cParams.memtag = NvBufSurfaceTag_VIDEO_DEC; - cParams.memType = NVBUF_MEM_SURFACE_ARRAY; - - ret = NvBufSurf::NvAllocate(&cParams, 1, &dst_dma_fd_); - INIT_ERROR(ret == -1, "failed to NvBufSurfaceAllocate"); - - decoder_->capture_plane.deinitPlane(); - - ret = decoder_->setCapturePlaneFormat(format.fmt.pix_mp.pixelformat, - format.fmt.pix_mp.width, - format.fmt.pix_mp.height); - INIT_ERROR(ret < 0, "Failed to setCapturePlaneFormat at capture_plane"); - - int32_t min_capture_buffer_size; - ret = decoder_->getMinimumCapturePlaneBuffers(min_capture_buffer_size); - INIT_ERROR(ret < 0, "Failed to getMinimumCapturePlaneBuffers"); - - ret = decoder_->capture_plane.setupPlane( - V4L2_MEMORY_MMAP, min_capture_buffer_size + 5, false, false); - INIT_ERROR(ret < 0, "Failed to setupPlane at capture_plane"); - - ret = decoder_->capture_plane.setStreamStatus(true); - INIT_ERROR(ret < 0, "Failed to setStreamStatus at decoder capture_plane"); - - for (uint32_t i = 0; i < decoder_->capture_plane.getNumBuffers(); i++) { - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); - v4l2_buf.index = i; - v4l2_buf.m.planes = planes; - v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - v4l2_buf.memory = V4L2_MEMORY_MMAP; - ret = decoder_->capture_plane.qBuffer(v4l2_buf, NULL); - INIT_ERROR(ret < 0, "Failed to qBuffer at encoder capture_plane"); - } - - return WEBRTC_VIDEO_CODEC_OK; -} - -} // namespace sora diff --git a/src/hwenc_jetson/jetson_video_encoder.cpp b/src/hwenc_jetson/jetson_video_encoder.cpp deleted file mode 100644 index 6a594941..00000000 --- a/src/hwenc_jetson/jetson_video_encoder.cpp +++ /dev/null @@ -1,963 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#include "sora/hwenc_jetson/jetson_video_encoder.h" - -#include -#include - -// WebRTC -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// L4T Multimedia API -#include -#include -#include - -#include "sora/hwenc_jetson/jetson_buffer.h" - -#define H264HWENC_HEADER_DEBUG 0 -#define INIT_ERROR(cond, desc) \ - if (cond) { \ - RTC_LOG(LS_ERROR) << __FUNCTION__ << desc; \ - Release(); \ - return WEBRTC_VIDEO_CODEC_ERROR; \ - } - -static std::string hex_dump(const uint8_t* buf, size_t len) { - std::stringstream ss; - - for (size_t i = 0; i < len; ++i) { - // 行の先頭にオフセットを表示 - if (i % 16 == 0) { - ss << std::setw(8) << std::setfill('0') << std::hex << i << ": "; - } - - // 値を16進数で表示 - ss << std::setw(2) << std::setfill('0') << std::hex << (int)buf[i] << " "; - - // 16バイトごとに改行 - if ((i + 1) % 16 == 0 || i == len - 1) { - ss << "\n"; - } - } - - return ss.str(); -} - -static void save_to_file(const std::string& filename, - const uint8_t* buf, - size_t size) { - std::ofstream file(filename, std::ios::binary); - file.write((const char*)buf, size); -} - -namespace sora { - -JetsonVideoEncoder::JetsonVideoEncoder(const cricket::VideoCodec& codec) - : callback_(nullptr), - encoder_(nullptr), - configured_framerate_(30), - use_native_(false), - use_dmabuff_(false) {} - -JetsonVideoEncoder::~JetsonVideoEncoder() { - Release(); -} - -// 標準出力や標準エラーに出力されないようにいろいろする -//struct SuppressErrors { -// SuppressErrors() { -// old_stdout = stdout; -// old_stderr = stderr; -// old_log_level = log_level; -// stdout = fopen("/dev/null", "w"); -// stderr = fopen("/dev/null", "w"); -// log_level = -1; -// } -// ~SuppressErrors() { -// fclose(stdout); -// fclose(stderr); -// stdout = old_stdout; -// stderr = old_stderr; -// log_level = old_log_level; -// } -// FILE* old_stdout; -// FILE* old_stderr; -// int old_log_level; -//}; - -bool JetsonVideoEncoder::IsSupportedVP8() { - //SuppressErrors sup; - - auto encoder = NvVideoEncoder::createVideoEncoder("enc0"); - auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_VP8, 1024, 768, - 2 * 1024 * 1024); - delete encoder; - - return ret >= 0; -} - -bool JetsonVideoEncoder::IsSupportedVP9() { - //SuppressErrors sup; - - auto encoder = NvVideoEncoder::createVideoEncoder("enc0"); - auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_VP9, 1024, 768, - 2 * 1024 * 1024); - delete encoder; - - return ret >= 0; -} - -bool JetsonVideoEncoder::IsSupportedAV1() { - //SuppressErrors sup; - - auto encoder = NvVideoEncoder::createVideoEncoder("enc0"); - auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, 1024, 768, - 2 * 1024 * 1024); - delete encoder; - - return ret >= 0; -} - -int32_t JetsonVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, - int32_t number_of_cores, - size_t max_payload_size) { - RTC_DCHECK(codec_settings); - - int32_t release_ret = Release(); - if (release_ret != WEBRTC_VIDEO_CODEC_OK) { - return release_ret; - } - if (&codec_ != codec_settings) { - codec_ = *codec_settings; - } - - width_ = codec_settings->width; - height_ = codec_settings->height; - target_bitrate_bps_ = codec_settings->startBitrate * 1000; - if (codec_settings->codecType == webrtc::kVideoCodecH264) { - key_frame_interval_ = codec_settings->H264().keyFrameInterval; - } else if (codec_settings->codecType == webrtc::kVideoCodecVP8) { - key_frame_interval_ = codec_settings->VP8().keyFrameInterval; - } else if (codec_settings->codecType == webrtc::kVideoCodecVP9) { - key_frame_interval_ = codec_settings->VP9().keyFrameInterval; - RTC_LOG(LS_INFO) << "numberOfTemporalLayers: " - << codec_settings->VP9().numberOfTemporalLayers; - RTC_LOG(LS_INFO) << "denoisingOn: " << codec_settings->VP9().denoisingOn; - RTC_LOG(LS_INFO) << "keyFrameInterval: " - << codec_settings->VP9().keyFrameInterval; - RTC_LOG(LS_INFO) << "adaptiveQpMode: " - << codec_settings->VP9().adaptiveQpMode; - RTC_LOG(LS_INFO) << "automaticResizeOn: " - << codec_settings->VP9().automaticResizeOn; - RTC_LOG(LS_INFO) << "numberOfSpatialLayers: " - << codec_settings->VP9().numberOfSpatialLayers; - RTC_LOG(LS_INFO) << "interLayerPred: " - << codec_settings->VP9().interLayerPred; - } else if (codec_settings->codecType == webrtc::kVideoCodecAV1) { - auto scalability_mode = codec_settings->GetScalabilityMode(); - if (!scalability_mode) { - RTC_LOG(LS_WARNING) << "Scalability mode is not set, using 'L1T1'."; - scalability_mode = webrtc::ScalabilityMode::kL1T1; - } - RTC_LOG(LS_INFO) << "InitEncode scalability_mode:" - << (int)*scalability_mode; - svc_controller_ = webrtc::CreateScalabilityStructure(*scalability_mode); - scalability_mode_ = *scalability_mode; - - // codec_settings->AV1() にはキーフレームの設定が無いけれども、 - // key_frame_interval_ 自体は何らかの値で初期化しておかないと - // 不定値になってしまうので、適当な値を入れておく - key_frame_interval_ = 3000; - } - framerate_ = codec_settings->maxFramerate; - - RTC_LOG(LS_INFO) << "InitEncode " << framerate_ << "fps " - << target_bitrate_bps_ << "bit/sec " - << codec_settings->maxBitrate << "kbit/sec "; - - // Initialize encoded image. - encoded_image_.timing_.flags = - webrtc::VideoSendTiming::TimingFrameFlags::kInvalid; - encoded_image_.content_type_ = - (codec_settings->mode == webrtc::VideoCodecMode::kScreensharing) - ? webrtc::VideoContentType::SCREENSHARE - : webrtc::VideoContentType::UNSPECIFIED; - - gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); - gof_idx_ = 0; - RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t JetsonVideoEncoder::Release() { - JetsonRelease(); - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t JetsonVideoEncoder::JetsonConfigure() { - int ret = 0; - bool use_converter = - use_native_ && (width_ != raw_width_ || height_ != raw_height_ || - decode_pixfmt_ != V4L2_PIX_FMT_YUV420M); - - encoder_ = NvVideoEncoder::createVideoEncoder("enc0"); - INIT_ERROR(!encoder_, "Failed to createVideoEncoder"); - - if (codec_.codecType == webrtc::kVideoCodecH264) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, width_, height_, - 2 * 1024 * 1024); - } else if (codec_.codecType == webrtc::kVideoCodecVP8) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_VP8, width_, height_, - 2 * 1024 * 1024); - } else if (codec_.codecType == webrtc::kVideoCodecVP9) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_VP9, width_, height_, - 2 * 1024 * 1024); - } else if (codec_.codecType == webrtc::kVideoCodecAV1) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, width_, height_, - 2 * 1024 * 1024); - } - INIT_ERROR(ret < 0, "Failed to encoder setCapturePlaneFormat"); - - ret = encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, width_, height_); - INIT_ERROR(ret < 0, "Failed to encoder setOutputPlaneFormat"); - - if (codec_.codecType == webrtc::kVideoCodecH264) { - ret = encoder_->setProfile(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH); - INIT_ERROR(ret < 0, "Failed to setProfile"); - - ret = encoder_->setLevel(V4L2_MPEG_VIDEO_H264_LEVEL_5_1); - INIT_ERROR(ret < 0, "Failed to setLevel"); - - // 必須 なければ H264 でフレームレートが出ない - ret = encoder_->setNumBFrames(0); - INIT_ERROR(ret < 0, "Failed to setNumBFrames"); - - ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); - INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); - - ret = encoder_->setInsertVuiEnabled(true); - INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); - - // V4L2_ENC_HW_PRESET_ULTRAFAST が推奨値だけど MEDIUM でも Nano, AGX では OK - // NX は V4L2_ENC_HW_PRESET_FAST でないとフレームレートがでない - ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_FAST); - INIT_ERROR(ret < 0, "Failed to setHWPresetType"); - } else if (codec_.codecType == webrtc::kVideoCodecVP8) { - uint32_t qp_min = - codec_.mode == webrtc::VideoCodecMode::kScreensharing ? 12 : 2; - uint32_t qp_max = 56; - if (codec_.qpMax >= qp_min) { - qp_max = codec_.qpMax; - } - ret = encoder_->setQpRange(qp_min, qp_max, qp_min, qp_max, qp_min, qp_max); - - // V4L2_ENC_HW_PRESET_ULTRAFAST を指定しないと 30fps 出ない - ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_ULTRAFAST); - INIT_ERROR(ret < 0, "Failed to setHWPresetType"); - } else if (codec_.codecType == webrtc::kVideoCodecVP9) { - // QP:150 が 30fps が出る下限。これ以上下げると 30fps を割る - ret = encoder_->setQpRange(QP_RETAIN_VAL, 150, QP_RETAIN_VAL, 150, - QP_RETAIN_VAL, 150); - INIT_ERROR(ret < 0, "Failed to setQpRange"); - - // V4L2_ENC_HW_PRESET_ULTRAFAST が推奨値だけど SLOW でもフレームレートの落ち方が変わらない - ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_SLOW); - INIT_ERROR(ret < 0, "Failed to setHWPresetType"); - } - - ret = encoder_->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR); - INIT_ERROR(ret < 0, "Failed to setRateControlMode"); - - /* ここに来たということはエンコーダは初期化されている - 初期化されているということは設定するべきは調整されたレートではなく - 最初の目標値であるべき BitrateAdjuster も初期化する*/ - bitrate_adjuster_.reset(new webrtc::BitrateAdjuster(.5, .95)); - bitrate_adjuster_->SetTargetBitrateBps(target_bitrate_bps_); - SetBitrateBps(target_bitrate_bps_); - - ret = encoder_->setIDRInterval(key_frame_interval_); - INIT_ERROR(ret < 0, "Failed to setIDRInterval"); - - ret = encoder_->setIFrameInterval(0); - INIT_ERROR(ret < 0, "Failed to setIFrameInterval"); - - ret = encoder_->setFrameRate(framerate_, 1); - INIT_ERROR(ret < 0, "Failed to setFrameRate"); - - if (use_native_) { - if (use_dmabuff_ || use_converter) { - ret = encoder_->output_plane.reqbufs(V4L2_MEMORY_DMABUF, 10); - INIT_ERROR(ret < 0, "Failed to reqbufs at encoder output_plane"); - - int fd; - NvBufSurf::NvCommonAllocateParams cParams; - cParams.width = width_; - cParams.height = height_; - cParams.layout = NVBUF_LAYOUT_PITCH; - cParams.colorFormat = NVBUF_COLOR_FORMAT_YUV420; - cParams.memtag = NvBufSurfaceTag_VIDEO_ENC; - cParams.memType = NVBUF_MEM_SURFACE_ARRAY; - for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) { - ret = NvBufSurf::NvAllocate(&cParams, 1, &fd); - INIT_ERROR(ret, "Failed to create NvBuffer"); - RTC_LOG(LS_ERROR) << "NvBufferCreateEx i:" << i << " fd:" << fd; - output_plane_fd_[i] = fd; - } - } else { - ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_USERPTR, 1, false, - false); - INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); - } - } else { - ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 1, true, false); - INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); - } - - ret = encoder_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 1, true, false); - INIT_ERROR(ret < 0, "Failed to setupPlane at capture_plane"); - - ret = encoder_->subscribeEvent(V4L2_EVENT_EOS, 0, 0); - INIT_ERROR(ret < 0, "Failed to subscribeEvent V4L2_EVENT_EOS"); - - ret = encoder_->output_plane.setStreamStatus(true); - INIT_ERROR(ret < 0, "Failed to setStreamStatus at encoder output_plane"); - - ret = encoder_->capture_plane.setStreamStatus(true); - INIT_ERROR(ret < 0, "Failed to setStreamStatus at encoder capture_plane"); - - encoder_->capture_plane.setDQThreadCallback(EncodeFinishedCallbackFunction); - encoder_->capture_plane.startDQThread(this); - - for (uint32_t i = 0; i < encoder_->capture_plane.getNumBuffers(); i++) { - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); - v4l2_buf.index = i; - v4l2_buf.m.planes = planes; - ret = encoder_->capture_plane.qBuffer(v4l2_buf, NULL); - INIT_ERROR(ret < 0, "Failed to qBuffer at encoder capture_plane"); - } - - configured_framerate_ = framerate_; - - return WEBRTC_VIDEO_CODEC_OK; -} - -void JetsonVideoEncoder::JetsonRelease() { - if (!encoder_) - return; - SendEOS(); - encoder_->capture_plane.waitForDQThread(2000); - encoder_->capture_plane.deinitPlane(); - if (use_dmabuff_) { - for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) { - if (encoder_->output_plane.unmapOutputBuffers(i, output_plane_fd_[i]) < - 0) { - RTC_LOG(LS_ERROR) - << "Failed to unmapOutputBuffers at encoder output_plane"; - } - if (NvBufSurf::NvDestroy(output_plane_fd_[i]) < 0) { - RTC_LOG(LS_ERROR) - << "Failed to NvBufferDestroy at encoder output_plane"; - } - } - } else { - encoder_->output_plane.deinitPlane(); - } - delete encoder_; - encoder_ = nullptr; -} - -void JetsonVideoEncoder::SendEOS() { - if (encoder_->output_plane.getStreamStatus()) { - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - NvBuffer* buffer; - - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); - v4l2_buf.m.planes = planes; - - if (encoder_->output_plane.getNumQueuedBuffers() == - encoder_->output_plane.getNumBuffers()) { - if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; - } - } - planes[0].bytesused = 0; - for (int i = 0; i < buffer->n_planes; i++) { - buffer->planes[i].bytesused = 0; - } - if (encoder_->output_plane.qBuffer(v4l2_buf, NULL) < 0) { - RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; - } - } -} - -bool JetsonVideoEncoder::EncodeFinishedCallbackFunction( - struct v4l2_buffer* v4l2_buf, - NvBuffer* buffer, - NvBuffer* shared_buffer, - void* data) { - return ((JetsonVideoEncoder*)data) - ->EncodeFinishedCallback(v4l2_buf, buffer, shared_buffer); -} - -bool JetsonVideoEncoder::EncodeFinishedCallback(struct v4l2_buffer* v4l2_buf, - NvBuffer* buffer, - NvBuffer* shared_buffer) { - if (!v4l2_buf) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; - return false; - } - - if (buffer->planes[0].bytesused == 0) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " buffer size is zero"; - return false; - } - - uint64_t timestamp = v4l2_buf->timestamp.tv_sec * rtc::kNumMicrosecsPerSec + - v4l2_buf->timestamp.tv_usec; - - std::unique_ptr params; - { - webrtc::MutexLock lock(&frame_params_lock_); - do { - if (frame_params_.empty()) { - RTC_LOG(LS_WARNING) - << __FUNCTION__ - << "Frame parameter is not found. SkipFrame timestamp:" - << timestamp; - return true; - } - params = std::move(frame_params_.front()); - frame_params_.pop(); - } while (params->timestamp_us < timestamp); - if (params->timestamp_us != timestamp) { - RTC_LOG(LS_WARNING) - << __FUNCTION__ - << "Frame parameter is not found. SkipFrame timestamp:" << timestamp; - return true; - } - } - - v4l2_ctrl_videoenc_outputbuf_metadata enc_metadata; - if (encoder_->getMetadata(v4l2_buf->index, enc_metadata) != 0) { - RTC_LOG(LS_WARNING) << __FUNCTION__ - << "getMetadata failed. SkipFrame timestamp:" - << timestamp; - return true; - } - - SendFrame(buffer->planes[0].data, buffer->planes[0].bytesused, - std::move(params), &enc_metadata); - - if (encoder_->capture_plane.qBuffer(*v4l2_buf, NULL) < 0) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to qBuffer at capture_plane"; - return false; - } - - return true; -} - -int32_t JetsonVideoEncoder::RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback* callback) { - callback_ = callback; - return WEBRTC_VIDEO_CODEC_OK; -} - -void JetsonVideoEncoder::SetRates(const RateControlParameters& parameters) { - if (encoder_ == nullptr) - return; - if (parameters.bitrate.get_sum_bps() <= 0 || parameters.framerate_fps <= 0) - return; - - RTC_LOG(LS_INFO) << __FUNCTION__ << " framerate:" << parameters.framerate_fps - << " bitrate:" << parameters.bitrate.ToString(); - if (svc_controller_) { - svc_controller_->OnRatesUpdated(parameters.bitrate); - } - - framerate_ = parameters.framerate_fps; - target_bitrate_bps_ = parameters.bitrate.get_sum_bps(); - - bitrate_adjuster_->SetTargetBitrateBps(target_bitrate_bps_); - return; -} - -void JetsonVideoEncoder::SetFramerate(uint32_t framerate) { - if (configured_framerate_ == framerate) { - return; - } - RTC_LOG(LS_INFO) << __FUNCTION__ << " " << framerate << "fps"; - if (encoder_->setFrameRate(framerate, 1) < 0) { - RTC_LOG(LS_ERROR) << "Failed to set bitrate"; - return; - } - configured_framerate_ = framerate; -} - -void JetsonVideoEncoder::SetBitrateBps(uint32_t bitrate_bps) { - if (bitrate_bps < 300000 || (configured_bitrate_bps_ == bitrate_bps && - configured_framerate_ == framerate_)) { - return; - } - configured_bitrate_bps_ = bitrate_bps; - - // VP9 の setBitrate は、設定されたフレームレートを見ずに - // 60fps での bps を見てるっぽいので、ここで渡す bps を調整する - if (codec_.codecType == webrtc::kVideoCodecVP9) { - auto adjusted_bps = bitrate_bps * 60 / configured_framerate_; - RTC_LOG(LS_INFO) << __FUNCTION__ << " bps=" << bitrate_bps - << " adjusted_bps=" << adjusted_bps; - bitrate_bps = adjusted_bps; - } else { - RTC_LOG(LS_INFO) << __FUNCTION__ << " bps=" << bitrate_bps; - } - - if (encoder_->setBitrate(bitrate_bps) < 0) { - RTC_LOG(LS_ERROR) << "Failed to setBitrate"; - return; - } -} - -webrtc::VideoEncoder::EncoderInfo JetsonVideoEncoder::GetEncoderInfo() const { - EncoderInfo info; - info.supports_native_handle = true; - info.implementation_name = "Jetson Video Encoder"; - if (codec_.codecType == webrtc::kVideoCodecH264) { - static const int kLowH264QpThreshold = 34; - static const int kHighH264QpThreshold = 40; - info.scaling_settings = VideoEncoder::ScalingSettings(kLowH264QpThreshold, - kHighH264QpThreshold); - } else if (codec_.codecType == webrtc::kVideoCodecVP8) { - static const int kLowVp8QpThreshold = 29; - static const int kHighVp8QpThreshold = 95; - info.scaling_settings = - VideoEncoder::ScalingSettings(kLowVp8QpThreshold, kHighVp8QpThreshold); - } else if (codec_.codecType == webrtc::kVideoCodecVP9) { - static const int kLowVp9QpThreshold = 150; - static const int kHighVp9QpThreshold = 151; - info.scaling_settings = - VideoEncoder::ScalingSettings(kLowVp9QpThreshold, kHighVp9QpThreshold); - } else if (codec_.codecType == webrtc::kVideoCodecAV1) { - static const int kLowAv1QpThreshold = 145; - static const int kHighAv1QpThreshold = 205; - info.scaling_settings = - VideoEncoder::ScalingSettings(kLowAv1QpThreshold, kHighAv1QpThreshold); - } - return info; -} - -int32_t JetsonVideoEncoder::Encode( - const webrtc::VideoFrame& input_frame, - const std::vector* frame_types) { - if (!callback_) { - RTC_LOG(LS_WARNING) - << "InitEncode() has been called, but a callback function " - << "has not been set with RegisterEncodeCompleteCallback()"; - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - - int fd = 0; - webrtc::VideoType video_type; - uint8_t* native_data; - rtc::scoped_refptr frame_buffer = - input_frame.video_frame_buffer(); - std::shared_ptr decoder; - if (frame_buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) { - use_native_ = true; - JetsonBuffer* jetson_buffer = - static_cast(frame_buffer.get()); - video_type = jetson_buffer->VideoType(); - raw_width_ = jetson_buffer->RawWidth(); - raw_height_ = jetson_buffer->RawHeight(); - use_dmabuff_ = true; - fd = jetson_buffer->DecodedFd(); - decode_pixfmt_ = jetson_buffer->V4L2PixelFormat(); - decoder = jetson_buffer->JpegDecoder(); - } else { - use_native_ = false; - } - - if (encoder_ == nullptr) { - if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) { - RTC_LOG(LS_ERROR) << "Failed to JetsonConfigure"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - } - - bool force_key_frame = false; - if (frame_types != nullptr) { - RTC_DCHECK_EQ(frame_types->size(), static_cast(1)); - if ((*frame_types)[0] == webrtc::VideoFrameType::kEmptyFrame) { - return WEBRTC_VIDEO_CODEC_OK; - } - if ((*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey) { - if (encoder_->forceIDR() < 0) { - RTC_LOG(LS_ERROR) << "Failed to forceIDR"; - } - } - } - - SetFramerate(framerate_); - SetBitrateBps(bitrate_adjuster_->GetAdjustedBitrateBps()); - { - webrtc::MutexLock lock(&frame_params_lock_); - frame_params_.push(absl::make_unique( - frame_buffer->width(), frame_buffer->height(), - input_frame.render_time_ms(), input_frame.ntp_time_ms(), - input_frame.timestamp_us(), input_frame.timestamp(), - input_frame.rotation(), input_frame.color_space(), decoder)); - } - - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, sizeof(planes)); - v4l2_buf.m.planes = planes; - - if (use_native_) { - NvBuffer* buffer; - if (encoder_->output_plane.getNumQueuedBuffers() == - encoder_->output_plane.getNumBuffers()) { - if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - } else { - buffer = encoder_->output_plane.getNthBuffer( - encoder_->output_plane.getNumQueuedBuffers()); - v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); - } - - int src_dma_fd = -1; - if (use_dmabuff_) { - src_dma_fd = fd; - } else if (video_type == webrtc::VideoType::kYUY2 || - video_type == webrtc::VideoType::kUYVY) { - buffer->planes[0].bytesused = buffer->planes[0].fmt.width * - buffer->planes[0].fmt.bytesperpixel * - buffer->planes[0].fmt.height; - buffer->planes[0].data = native_data; - } else if (video_type == webrtc::VideoType::kI420) { - size_t offset = 0; - for (int i = 0; i < buffer->n_planes; i++) { - buffer->planes[i].bytesused = buffer->planes[i].fmt.width * - buffer->planes[i].fmt.bytesperpixel * - buffer->planes[i].fmt.height; - buffer->planes[i].data = native_data + offset; - offset += buffer->planes[i].bytesused; - } - } else if (video_type == webrtc::VideoType::kYV12) { - size_t offset = 0; - buffer->planes[0].bytesused = buffer->planes[0].fmt.width * - buffer->planes[0].fmt.bytesperpixel * - buffer->planes[0].fmt.height; - buffer->planes[0].data = native_data; - offset += buffer->planes[0].bytesused; - buffer->planes[2].bytesused = buffer->planes[1].fmt.width * - buffer->planes[1].fmt.bytesperpixel * - buffer->planes[1].fmt.height; - buffer->planes[2].data = native_data + offset; - offset += buffer->planes[2].bytesused; - buffer->planes[1].bytesused = buffer->planes[2].fmt.width * - buffer->planes[2].fmt.bytesperpixel * - buffer->planes[2].fmt.height; - buffer->planes[1].data = native_data + offset; - } else { - RTC_LOG(LS_ERROR) << "Unsupported webrtc::VideoType"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - NvBufSurf::NvCommonTransformParams transform_params; - /* Indicates which of the transform parameters are valid */ - memset(&transform_params, 0, sizeof(transform_params)); - transform_params.src_top = 0; - transform_params.src_left = 0; - transform_params.src_width = raw_width_; - transform_params.src_height = raw_height_; - transform_params.dst_top = 0; - transform_params.dst_left = 0; - transform_params.dst_width = width_; - transform_params.dst_height = height_; - transform_params.flag = - (NvBufSurfTransform_Transform_Flag)(NVBUFSURF_TRANSFORM_FILTER | - NVBUFSURF_TRANSFORM_CROP_SRC); - transform_params.flip = NvBufSurfTransform_None; - transform_params.filter = NvBufSurfTransformInter_Bilinear; - if (NvBufSurf::NvTransform(&transform_params, src_dma_fd, - output_plane_fd_[v4l2_buf.index])) { - RTC_LOG(LS_ERROR) << "Failed to NvBufferTransform"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - planes[0].m.fd = output_plane_fd_[v4l2_buf.index]; - planes[0].bytesused = 1234; - - v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - v4l2_buf.memory = V4L2_MEMORY_DMABUF; - v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; - v4l2_buf.timestamp.tv_sec = - input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; - v4l2_buf.timestamp.tv_usec = - input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; - - if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { - RTC_LOG(LS_ERROR) << "Failed to qBuffer at converter output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - } else { - NvBuffer* buffer; - - RTC_LOG(LS_VERBOSE) << __FUNCTION__ << " output_plane.getNumBuffers: " - << encoder_->output_plane.getNumBuffers() - << " output_plane.getNumQueuedBuffers: " - << encoder_->output_plane.getNumQueuedBuffers(); - - if (encoder_->output_plane.getNumQueuedBuffers() == - encoder_->output_plane.getNumBuffers()) { - if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - } else { - buffer = encoder_->output_plane.getNthBuffer( - encoder_->output_plane.getNumQueuedBuffers()); - v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); - } - - rtc::scoped_refptr i420_buffer = - frame_buffer->ToI420(); - for (uint32_t i = 0; i < buffer->n_planes; i++) { - const uint8_t* source_data; - int source_stride; - if (i == 0) { - source_data = i420_buffer->DataY(); - source_stride = i420_buffer->StrideY(); - } else if (i == 1) { - source_data = i420_buffer->DataU(); - source_stride = i420_buffer->StrideU(); - } else if (i == 2) { - source_data = i420_buffer->DataV(); - source_stride = i420_buffer->StrideV(); - } else { - break; - } - NvBuffer::NvBufferPlane& plane = buffer->planes[i]; - std::streamsize bytes_to_read = plane.fmt.bytesperpixel * plane.fmt.width; - uint8_t* input_data = plane.data; - plane.bytesused = 0; - for (uint32_t j = 0; j < plane.fmt.height; j++) { - memcpy(input_data, source_data + (source_stride * j), bytes_to_read); - input_data += plane.fmt.stride; - } - plane.bytesused = plane.fmt.stride * plane.fmt.height; - } - - v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; - v4l2_buf.timestamp.tv_sec = - input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; - v4l2_buf.timestamp.tv_usec = - input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; - - for (int i = 0; i < MAX_PLANES; i++) { - NvBufSurface* surf = 0; - if (NvBufSurfaceFromFd(buffer->planes[i].fd, (void**)(&surf)) == -1) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to NvBufSurfaceFromFd"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - if (NvBufSurfaceSyncForDevice(surf, 0, i) == -1) { - RTC_LOG(LS_ERROR) << "Failed to NvBufSurfaceSyncForDevice"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - // ここで NvBufSurfaceDestroy が必要かなと思ったが、以下のサンプル・コードを確認したところ不要そうだった - // 参照: jetson_multimedia_api/samples/01_video_encode/video_encode_main.cpp - // NvBufSurfaceDestroy(surf); - } - - if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { - RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; - } - } - - return WEBRTC_VIDEO_CODEC_OK; -} - -int32_t JetsonVideoEncoder::SendFrame( - unsigned char* buffer, - size_t size, - std::unique_ptr params, - v4l2_ctrl_videoenc_outputbuf_metadata* enc_metadata) { - if (!callback_) { - RTC_LOG(LS_WARNING) - << "InitEncode() has been called, but a callback function " - << "has not been set with RegisterEncodeCompleteCallback()"; - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - - encoded_image_.SetRtpTimestamp(params->timestamp_rtp); - encoded_image_.SetColorSpace(params->color_space); - encoded_image_._encodedWidth = params->width; - encoded_image_._encodedHeight = params->height; - encoded_image_.capture_time_ms_ = params->render_time_ms; - encoded_image_.ntp_time_ms_ = params->ntp_time_ms; - encoded_image_.rotation_ = params->rotation; - encoded_image_.qp_ = enc_metadata->AvgQP; - if (enc_metadata->KeyFrame) { - encoded_image_.SetFrameType(webrtc::VideoFrameType::kVideoFrameKey); - } else { - encoded_image_.SetFrameType(webrtc::VideoFrameType::kVideoFrameDelta); - } - - webrtc::CodecSpecificInfo codec_specific; - codec_specific.codecType = codec_.codecType; - if (codec_.codecType == webrtc::kVideoCodecH264) { - auto encoded_image_buffer = - webrtc::EncodedImageBuffer::Create(buffer, size); - encoded_image_.SetEncodedData(encoded_image_buffer); - - codec_specific.codecSpecific.H264.packetization_mode = - webrtc::H264PacketizationMode::NonInterleaved; - } else if (codec_.codecType == webrtc::kVideoCodecAV1 || - codec_.codecType == webrtc::kVideoCodecVP9 || - codec_.codecType == webrtc::kVideoCodecVP8) { - // VP8, VP9, AV1 はIVFヘッダーがエンコードフレームについているので取り除く - if ((buffer[0] == 'D') && (buffer[1] == 'K') && (buffer[2] == 'I') && - (buffer[3] == 'F')) { - buffer += 32; - size -= 32; - } - buffer += 12; - size -= 12; - - rtc::scoped_refptr encoded_image_buffer; - - if (codec_.codecType == webrtc::kVideoCodecAV1) { - // JetPack 5.1.1 以降、AV1 のエンコードフレームにシーケンスヘッダー(OBU_SEQUENCE_HEADER)が含まれなくなってしまった。 - // キーフレームには必ずシーケンスヘッダーを設定しておかないと、途中から受信したデコーダーでデコードができなくなってしまう。 - // そのため、初回のシーケンスヘッダーを保存しておき、キーフレームの前に挿入することで対応する。 - - // buffer の最初の 2 バイトは 0x12 0x00 で、これは OBU_TEMPORAL_DELIMITER なのでこれは無視して、その次の OBU を見る - if (buffer[2] == 0x0a) { - // 0x0a は OBU_SEQUENCE_HEADER なので、この OBU を保存しておく - // obu_size は本当は LEB128 だけど、128 バイト以上になることはなさそうなので、そのまま 1 バイトで取得する - int obu_size = buffer[3]; - obu_seq_header_.resize(obu_size + 2); - memcpy(obu_seq_header_.data(), buffer + 2, obu_size + 2); - } - - // キーフレームとしてマークされているのにシーケンスヘッダーが入っていない場合、 - // フレームの前にシーケンスヘッダーを入れる - if (enc_metadata->KeyFrame && buffer[2] != 0x0a) { - std::vector new_buffer; - new_buffer.resize(obu_seq_header_.size() + size); - uint8_t* p = new_buffer.data(); - // OBU_TEMPORAL_DELIMITER - memcpy(p, buffer, 2); - p += 2; - // OBU_SEQUENCE_HEADER - memcpy(p, obu_seq_header_.data(), obu_seq_header_.size()); - p += obu_seq_header_.size(); - // OBU_FRAME - memcpy(p, buffer + 2, size - 2); - - // RTC_LOG(LS_ERROR) << "\n" << hex_dump(new_buffer.data(), 64); - // save_to_file("keyframe2.obu", new_buffer.data(), new_buffer.size()); - encoded_image_buffer = webrtc::EncodedImageBuffer::Create( - new_buffer.data(), new_buffer.size()); - } else { - encoded_image_buffer = webrtc::EncodedImageBuffer::Create(buffer, size); - } - } else { - encoded_image_buffer = webrtc::EncodedImageBuffer::Create(buffer, size); - } - encoded_image_.SetEncodedData(encoded_image_buffer); - - if (codec_.codecType == webrtc::kVideoCodecVP8) { - codec_specific.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; - // nonReference かを知ることはできなかった - codec_specific.codecSpecific.VP8.nonReference = false; - } else if (codec_.codecType == webrtc::kVideoCodecVP9) { - if (enc_metadata->KeyFrame) { - gof_idx_ = 0; - } - codec_specific.codecSpecific.VP9.inter_pic_predicted = - enc_metadata->KeyFrame ? false : true; - codec_specific.codecSpecific.VP9.flexible_mode = false; - codec_specific.codecSpecific.VP9.ss_data_available = - enc_metadata->KeyFrame ? true : false; - codec_specific.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; - codec_specific.codecSpecific.VP9.temporal_up_switch = true; - codec_specific.codecSpecific.VP9.inter_layer_predicted = false; - codec_specific.codecSpecific.VP9.gof_idx = - static_cast(gof_idx_++ % gof_.num_frames_in_gof); - codec_specific.codecSpecific.VP9.num_spatial_layers = 1; - codec_specific.codecSpecific.VP9.first_frame_in_picture = true; - codec_specific.codecSpecific.VP9.spatial_layer_resolution_present = false; - if (codec_specific.codecSpecific.VP9.ss_data_available) { - codec_specific.codecSpecific.VP9.spatial_layer_resolution_present = - true; - codec_specific.codecSpecific.VP9.width[0] = - encoded_image_._encodedWidth; - codec_specific.codecSpecific.VP9.height[0] = - encoded_image_._encodedHeight; - codec_specific.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); - } - } else if (codec_.codecType == webrtc::kVideoCodecAV1) { - bool is_key = - encoded_image_._frameType == webrtc::VideoFrameType::kVideoFrameKey; - std::vector - layer_frames = svc_controller_->NextFrameConfig(is_key); - codec_specific.end_of_picture = true; - codec_specific.scalability_mode = scalability_mode_; - codec_specific.generic_frame_info = - svc_controller_->OnEncodeDone(layer_frames[0]); - if (is_key && codec_specific.generic_frame_info) { - codec_specific.template_structure = - svc_controller_->DependencyStructure(); - auto& resolutions = codec_specific.template_structure->resolutions; - resolutions = {webrtc::RenderResolution(encoded_image_._encodedWidth, - encoded_image_._encodedHeight)}; - } - } - } - - RTC_LOG(LS_VERBOSE) << "key_frame=" << enc_metadata->KeyFrame - << " size=" << size << " qp=" << encoded_image_.qp_; - - webrtc::EncodedImageCallback::Result result = - callback_->OnEncodedImage(encoded_image_, &codec_specific); - if (result.error != webrtc::EncodedImageCallback::Result::OK) { - RTC_LOG(LS_ERROR) << __FUNCTION__ - << " OnEncodedImage failed error:" << result.error; - return WEBRTC_VIDEO_CODEC_ERROR; - } - bitrate_adjuster_->Update(size); - return WEBRTC_VIDEO_CODEC_OK; -} - -} // namespace sora diff --git a/src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp b/src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp index 2f0b3015..da2e5849 100644 --- a/src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp +++ b/src/hwenc_nvcodec/nvcodec_decoder_cuda.cpp @@ -8,10 +8,12 @@ namespace sora { static cudaVideoCodec ToCudaVideoCodec(CudaVideoCodec codec) { - return codec == CudaVideoCodec::H264 ? cudaVideoCodec_H264 - : codec == CudaVideoCodec::VP8 ? cudaVideoCodec_VP8 - : codec == CudaVideoCodec::VP9 ? cudaVideoCodec_VP9 - : cudaVideoCodec_JPEG; + return codec == CudaVideoCodec::H264 ? cudaVideoCodec_H264 + : codec == CudaVideoCodec::H265 ? cudaVideoCodec_HEVC + : codec == CudaVideoCodec::VP8 ? cudaVideoCodec_VP8 + : codec == CudaVideoCodec::VP9 ? cudaVideoCodec_VP9 + : codec == CudaVideoCodec::AV1 ? cudaVideoCodec_AV1 + : cudaVideoCodec_JPEG; } #define CUDA_DRVAPI_CALL(call) \ diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp b/src/hwenc_nvcodec/nvcodec_video_encoder.cpp similarity index 78% rename from src/hwenc_nvcodec/nvcodec_h264_encoder.cpp rename to src/hwenc_nvcodec/nvcodec_video_encoder.cpp index ec24d935..ed05d326 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder.cpp +++ b/src/hwenc_nvcodec/nvcodec_video_encoder.cpp @@ -1,6 +1,6 @@ #include "sora/fix_cuda_noinline_macro_error.h" -#include "sora/hwenc_nvcodec/nvcodec_h264_encoder.h" +#include "sora/hwenc_nvcodec/nvcodec_video_encoder.h" #ifdef _WIN32 #include @@ -15,17 +15,22 @@ // WebRTC #include #include +#include #include -#include #include +#include +#include #include +// libyuv +#include + // NvCodec #ifdef _WIN32 #include #endif #ifdef __linux__ -#include "nvcodec_h264_encoder_cuda.h" +#include "nvcodec_video_encoder_cuda.h" #endif #ifdef __linux__ @@ -47,13 +52,14 @@ struct nal_entry { using Microsoft::WRL::ComPtr; #endif -class NvCodecH264EncoderImpl : public NvCodecH264Encoder { +class NvCodecVideoEncoderImpl : public NvCodecVideoEncoder { public: - NvCodecH264EncoderImpl(const cricket::VideoCodec& codec, - std::shared_ptr cuda_context); - ~NvCodecH264EncoderImpl() override; + NvCodecVideoEncoderImpl(std::shared_ptr cuda_context, + CudaVideoCodec codec); + ~NvCodecVideoEncoderImpl() override; - static bool IsSupported(std::shared_ptr cuda_context); + static bool IsSupported(std::shared_ptr cuda_context, + CudaVideoCodec codec); int32_t InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, @@ -69,6 +75,7 @@ class NvCodecH264EncoderImpl : public NvCodecH264Encoder { webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; static std::unique_ptr CreateEncoder( + CudaVideoCodec codec, int width, int height, int framerate, @@ -81,7 +88,7 @@ class NvCodecH264EncoderImpl : public NvCodecH264Encoder { #endif #ifdef __linux__ , - NvCodecH264EncoderCuda* cuda, + NvCodecVideoEncoderCuda* cuda, bool is_nv12 #endif ); @@ -96,8 +103,10 @@ class NvCodecH264EncoderImpl : public NvCodecH264Encoder { int32_t InitNvEnc(); int32_t ReleaseNvEnc(); webrtc::H264BitstreamParser h264_bitstream_parser_; + webrtc::H265BitstreamParser h265_bitstream_parser_; std::shared_ptr cuda_context_; + CudaVideoCodec codec_; std::unique_ptr nv_encoder_; #ifdef _WIN32 Microsoft::WRL::ComPtr id3d11_device_; @@ -105,7 +114,7 @@ class NvCodecH264EncoderImpl : public NvCodecH264Encoder { Microsoft::WRL::ComPtr id3d11_texture_; #endif #ifdef __linux__ - std::unique_ptr cuda_; + std::unique_ptr cuda_; bool is_nv12_ = false; #endif bool reconfigure_needed_ = false; @@ -118,10 +127,10 @@ class NvCodecH264EncoderImpl : public NvCodecH264Encoder { webrtc::EncodedImage encoded_image_; }; -NvCodecH264EncoderImpl::NvCodecH264EncoderImpl( - const cricket::VideoCodec& codec, - std::shared_ptr cuda_context) - : cuda_context_(cuda_context), bitrate_adjuster_(0.5, 0.95) { +NvCodecVideoEncoderImpl::NvCodecVideoEncoderImpl( + std::shared_ptr cuda_context, + CudaVideoCodec codec) + : cuda_context_(cuda_context), codec_(codec), bitrate_adjuster_(0.5, 0.95) { #ifdef _WIN32 ComPtr idxgi_factory; RTC_CHECK(!FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory1), @@ -143,18 +152,17 @@ NvCodecH264EncoderImpl::NvCodecH264EncoderImpl( RTC_LOG(LS_INFO) << __FUNCTION__ << "GPU in use: " << szDesc; #endif #ifdef __linux__ - cuda_.reset(new NvCodecH264EncoderCuda(cuda_context_)); + cuda_.reset(new NvCodecVideoEncoderCuda(cuda_context_)); #endif } -NvCodecH264EncoderImpl::~NvCodecH264EncoderImpl() {} +NvCodecVideoEncoderImpl::~NvCodecVideoEncoderImpl() {} -int32_t NvCodecH264EncoderImpl::InitEncode( +int32_t NvCodecVideoEncoderImpl::InitEncode( const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { RTC_DCHECK(codec_settings); - RTC_DCHECK_EQ(codec_settings->codecType, webrtc::kVideoCodecH264); int32_t release_ret = Release(); if (release_ret != WEBRTC_VIDEO_CODEC_OK) { @@ -174,18 +182,18 @@ int32_t NvCodecH264EncoderImpl::InitEncode( return InitNvEnc(); } -int32_t NvCodecH264EncoderImpl::RegisterEncodeCompleteCallback( +int32_t NvCodecVideoEncoderImpl::RegisterEncodeCompleteCallback( webrtc::EncodedImageCallback* callback) { std::lock_guard lock(mutex_); callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } -int32_t NvCodecH264EncoderImpl::Release() { +int32_t NvCodecVideoEncoderImpl::Release() { return ReleaseNvEnc(); } -int32_t NvCodecH264EncoderImpl::Encode( +int32_t NvCodecVideoEncoderImpl::Encode( const webrtc::VideoFrame& frame, const std::vector* frame_types) { //RTC_LOG(LS_ERROR) << __FUNCTION__ << " Start"; @@ -370,12 +378,21 @@ int32_t NvCodecH264EncoderImpl::Encode( } webrtc::CodecSpecificInfo codec_specific; - codec_specific.codecType = webrtc::kVideoCodecH264; - codec_specific.codecSpecific.H264.packetization_mode = - webrtc::H264PacketizationMode::NonInterleaved; - - h264_bitstream_parser_.ParseBitstream(encoded_image_); - encoded_image_.qp_ = h264_bitstream_parser_.GetLastSliceQp().value_or(-1); + if (codec_ == CudaVideoCodec::H264) { + codec_specific.codecType = webrtc::kVideoCodecH264; + codec_specific.codecSpecific.H264.packetization_mode = + webrtc::H264PacketizationMode::NonInterleaved; + + h264_bitstream_parser_.ParseBitstream(encoded_image_); + encoded_image_.qp_ = h264_bitstream_parser_.GetLastSliceQp().value_or(-1); + } else if (codec_ == CudaVideoCodec::H265) { + codec_specific.codecType = webrtc::kVideoCodecH265; + codec_specific.codecSpecific.H265.packetization_mode = + webrtc::H265PacketizationMode::NonInterleaved; + + h265_bitstream_parser_.ParseBitstream(encoded_image_); + encoded_image_.qp_ = h265_bitstream_parser_.GetLastSliceQp().value_or(-1); + } webrtc::EncodedImageCallback::Result result = callback_->OnEncodedImage(encoded_image_, &codec_specific); @@ -390,7 +407,7 @@ int32_t NvCodecH264EncoderImpl::Encode( return WEBRTC_VIDEO_CODEC_OK; } -void NvCodecH264EncoderImpl::SetRates( +void NvCodecVideoEncoderImpl::SetRates( const webrtc::VideoEncoder::RateControlParameters& parameters) { if (!nv_encoder_) { RTC_LOG(LS_WARNING) << "SetRates() while uninitialized."; @@ -415,25 +432,26 @@ void NvCodecH264EncoderImpl::SetRates( reconfigure_needed_ = true; } -webrtc::VideoEncoder::EncoderInfo NvCodecH264EncoderImpl::GetEncoderInfo() +webrtc::VideoEncoder::EncoderInfo NvCodecVideoEncoderImpl::GetEncoderInfo() const { webrtc::VideoEncoder::EncoderInfo info; info.supports_native_handle = true; - info.implementation_name = "NvCodec H264"; + info.implementation_name = "NvCodec"; info.scaling_settings = webrtc::VideoEncoder::ScalingSettings( kLowH264QpThreshold, kHighH264QpThreshold); return info; } -int32_t NvCodecH264EncoderImpl::InitNvEnc() { +int32_t NvCodecVideoEncoderImpl::InitNvEnc() { #ifdef _WIN32 - nv_encoder_ = CreateEncoder(width_, height_, framerate_, target_bitrate_bps_, - max_bitrate_bps_, id3d11_device_.Get(), - id3d11_texture_.GetAddressOf()); + nv_encoder_ = CreateEncoder( + codec_, width_, height_, framerate_, target_bitrate_bps_, + max_bitrate_bps_, id3d11_device_.Get(), id3d11_texture_.GetAddressOf()); #endif #ifdef __linux__ - nv_encoder_ = CreateEncoder(width_, height_, framerate_, target_bitrate_bps_, - max_bitrate_bps_, cuda_.get(), is_nv12_); + nv_encoder_ = + CreateEncoder(codec_, width_, height_, framerate_, target_bitrate_bps_, + max_bitrate_bps_, cuda_.get(), is_nv12_); #endif if (nv_encoder_ == nullptr) { @@ -445,7 +463,7 @@ int32_t NvCodecH264EncoderImpl::InitNvEnc() { return WEBRTC_VIDEO_CODEC_OK; } -int32_t NvCodecH264EncoderImpl::ReleaseNvEnc() { +int32_t NvCodecVideoEncoderImpl::ReleaseNvEnc() { if (nv_encoder_) { try { nv_encoder_->EndEncode(v_packet_); @@ -461,7 +479,8 @@ int32_t NvCodecH264EncoderImpl::ReleaseNvEnc() { return WEBRTC_VIDEO_CODEC_OK; } -std::unique_ptr NvCodecH264EncoderImpl::CreateEncoder( +std::unique_ptr NvCodecVideoEncoderImpl::CreateEncoder( + CudaVideoCodec codec, int width, int height, int framerate, @@ -474,7 +493,7 @@ std::unique_ptr NvCodecH264EncoderImpl::CreateEncoder( #endif #ifdef __linux__ , - NvCodecH264EncoderCuda* cuda, + NvCodecVideoEncoderCuda* cuda, bool is_nv12 #endif ) { @@ -519,9 +538,15 @@ std::unique_ptr NvCodecH264EncoderImpl::CreateEncoder( NV_ENC_CONFIG encode_config = {NV_ENC_CONFIG_VER}; initialize_params.encodeConfig = &encode_config; try { - encoder->CreateDefaultEncoderParams( - &initialize_params, NV_ENC_CODEC_H264_GUID, NV_ENC_PRESET_P3_GUID, - NV_ENC_TUNING_INFO_LOW_LATENCY); + if (codec == CudaVideoCodec::H264) { + encoder->CreateDefaultEncoderParams( + &initialize_params, NV_ENC_CODEC_H264_GUID, NV_ENC_PRESET_P3_GUID, + NV_ENC_TUNING_INFO_LOW_LATENCY); + } else if (codec == CudaVideoCodec::H265) { + encoder->CreateDefaultEncoderParams( + &initialize_params, NV_ENC_CODEC_HEVC_GUID, NV_ENC_PRESET_P2_GUID, + NV_ENC_TUNING_INFO_LOW_LATENCY); + } //initialize_params.enablePTD = 1; initialize_params.frameRateDen = 1; @@ -544,14 +569,22 @@ std::unique_ptr NvCodecH264EncoderImpl::CreateEncoder( encode_config.frameIntervalP = 1; encode_config.rcParams.enableAQ = 1; - //encode_config.encodeCodecConfig.h264Config.outputAUD = 1; - //encode_config.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_H264_31; - //encode_config.encodeCodecConfig.h264Config.entropyCodingMode = NV_ENC_H264_ENTROPY_CODING_MODE_CAVLC; - encode_config.encodeCodecConfig.h264Config.idrPeriod = - NVENC_INFINITE_GOPLENGTH; - encode_config.encodeCodecConfig.h264Config.repeatSPSPPS = 1; - encode_config.encodeCodecConfig.h264Config.sliceMode = 0; - encode_config.encodeCodecConfig.h264Config.sliceModeData = 0; + if (codec == CudaVideoCodec::H264) { + //encode_config.encodeCodecConfig.h264Config.outputAUD = 1; + //encode_config.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_H264_31; + //encode_config.encodeCodecConfig.h264Config.entropyCodingMode = NV_ENC_H264_ENTROPY_CODING_MODE_CAVLC; + encode_config.encodeCodecConfig.h264Config.idrPeriod = + NVENC_INFINITE_GOPLENGTH; + encode_config.encodeCodecConfig.h264Config.repeatSPSPPS = 1; + encode_config.encodeCodecConfig.h264Config.sliceMode = 0; + encode_config.encodeCodecConfig.h264Config.sliceModeData = 0; + } else if (codec == CudaVideoCodec::H265) { + encode_config.encodeCodecConfig.hevcConfig.idrPeriod = + NVENC_INFINITE_GOPLENGTH; + encode_config.encodeCodecConfig.hevcConfig.repeatSPSPPS = 1; + encode_config.encodeCodecConfig.hevcConfig.sliceMode = 0; + encode_config.encodeCodecConfig.hevcConfig.sliceModeData = 0; + } encoder->CreateEncoder(&initialize_params); @@ -566,8 +599,8 @@ std::unique_ptr NvCodecH264EncoderImpl::CreateEncoder( return encoder; } -bool NvCodecH264Encoder::IsSupported( - std::shared_ptr cuda_context) { +bool NvCodecVideoEncoder::IsSupported(std::shared_ptr cuda_context, + CudaVideoCodec codec) { try { NvEncoder::TryLoadNvEncApi(); @@ -610,15 +643,15 @@ bool NvCodecH264Encoder::IsSupported( D3D11_SDK_VERSION, id3d11_device.GetAddressOf(), NULL, id3d11_context.GetAddressOf()))); - auto encoder = NvCodecH264EncoderImpl::CreateEncoder( - 640, 480, 30, 100 * 1000, 500 * 1000, id3d11_device.Get(), + auto encoder = NvCodecVideoEncoderImpl::CreateEncoder( + codec, 640, 480, 30, 100 * 1000, 500 * 1000, id3d11_device.Get(), id3d11_texture.GetAddressOf()); #endif #ifdef __linux__ - auto cuda = std::unique_ptr( - new NvCodecH264EncoderCuda(cuda_context)); - auto encoder = NvCodecH264EncoderImpl::CreateEncoder( - 640, 480, 30, 100 * 1000, 500 * 1000, cuda.get(), true); + auto cuda = std::unique_ptr( + new NvCodecVideoEncoderCuda(cuda_context)); + auto encoder = NvCodecVideoEncoderImpl::CreateEncoder( + codec, 640, 480, 30, 100 * 1000, 500 * 1000, cuda.get(), true); #endif if (encoder == nullptr) { return false; @@ -631,11 +664,11 @@ bool NvCodecH264Encoder::IsSupported( } } -std::unique_ptr NvCodecH264Encoder::Create( - const cricket::VideoCodec& codec, - std::shared_ptr cuda_context) { - return std::unique_ptr( - new NvCodecH264EncoderImpl(codec, cuda_context)); +std::unique_ptr NvCodecVideoEncoder::Create( + std::shared_ptr cuda_context, + CudaVideoCodec codec) { + return std::unique_ptr( + new NvCodecVideoEncoderImpl(cuda_context, codec)); } } // namespace sora diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp b/src/hwenc_nvcodec/nvcodec_video_encoder_cuda.cpp similarity index 71% rename from src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp rename to src/hwenc_nvcodec/nvcodec_video_encoder_cuda.cpp index e5ba12a0..a9914d77 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.cpp +++ b/src/hwenc_nvcodec/nvcodec_video_encoder_cuda.cpp @@ -1,6 +1,6 @@ #include "sora/fix_cuda_noinline_macro_error.h" -#include "nvcodec_h264_encoder_cuda.h" +#include "nvcodec_video_encoder_cuda.h" #include @@ -13,10 +13,10 @@ namespace sora { -class NvCodecH264EncoderCudaImpl { +class NvCodecVideoEncoderCudaImpl { public: - NvCodecH264EncoderCudaImpl(std::shared_ptr ctx); - ~NvCodecH264EncoderCudaImpl(); + NvCodecVideoEncoderCudaImpl(std::shared_ptr ctx); + ~NvCodecVideoEncoderCudaImpl(); void Copy(NvEncoder* nv_encoder, const void* ptr, int width, int height); NvEncoder* CreateNvEncoder(int width, int height, bool is_nv12); @@ -25,21 +25,22 @@ class NvCodecH264EncoderCudaImpl { std::shared_ptr cuda_context_; }; -NvCodecH264EncoderCuda::NvCodecH264EncoderCuda(std::shared_ptr ctx) - : impl_(new NvCodecH264EncoderCudaImpl(ctx)) {} -NvCodecH264EncoderCuda::~NvCodecH264EncoderCuda() { +NvCodecVideoEncoderCuda::NvCodecVideoEncoderCuda( + std::shared_ptr ctx) + : impl_(new NvCodecVideoEncoderCudaImpl(ctx)) {} +NvCodecVideoEncoderCuda::~NvCodecVideoEncoderCuda() { delete impl_; } -void NvCodecH264EncoderCuda::Copy(NvEncoder* nv_encoder, - const void* ptr, - int width, - int height) { +void NvCodecVideoEncoderCuda::Copy(NvEncoder* nv_encoder, + const void* ptr, + int width, + int height) { impl_->Copy(nv_encoder, ptr, width, height); } -NvEncoder* NvCodecH264EncoderCuda::CreateNvEncoder(int width, - int height, - bool is_nv12) { +NvEncoder* NvCodecVideoEncoderCuda::CreateNvEncoder(int width, + int height, + bool is_nv12) { return impl_->CreateNvEncoder(width, height, is_nv12); } @@ -63,79 +64,68 @@ void ShowEncoderCapability() { std::cout << "GPU " << iGpu << " - " << szDeviceName << std::endl << std::endl; std::cout - << "\tH264:\t\t" - << " " + << "\tH264:\t\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES) ? "yes" : "no") << std::endl - << "\tH264_444:\t" - << " " + << "\tH264_444:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_SUPPORT_YUV444_ENCODE) ? "yes" : "no") << std::endl - << "\tH264_ME:\t" - << " " + << "\tH264_ME:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_SUPPORT_MEONLY_MODE) ? "yes" : "no") << std::endl - << "\tH264_WxH:\t" - << " " + << "\tH264_WxH:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_WIDTH_MAX)) << "*" << (enc.GetCapabilityValue(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_HEIGHT_MAX)) << std::endl - << "\tHEVC:\t\t" - << " " + << "\tHEVC:\t\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES) ? "yes" : "no") << std::endl - << "\tHEVC_Main10:\t" - << " " + << "\tHEVC_Main10:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_SUPPORT_10BIT_ENCODE) ? "yes" : "no") << std::endl - << "\tHEVC_Lossless:\t" - << " " + << "\tHEVC_Lossless:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_SUPPORT_LOSSLESS_ENCODE) ? "yes" : "no") << std::endl - << "\tHEVC_SAO:\t" - << " " + << "\tHEVC_SAO:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_SUPPORT_SAO) ? "yes" : "no") << std::endl - << "\tHEVC_444:\t" - << " " + << "\tHEVC_444:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_SUPPORT_YUV444_ENCODE) ? "yes" : "no") << std::endl - << "\tHEVC_ME:\t" - << " " + << "\tHEVC_ME:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_SUPPORT_MEONLY_MODE) ? "yes" : "no") << std::endl - << "\tHEVC_WxH:\t" - << " " + << "\tHEVC_WxH:\t" << " " << (enc.GetCapabilityValue(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_WIDTH_MAX)) << "*" @@ -150,16 +140,16 @@ void ShowEncoderCapability() { } } -NvCodecH264EncoderCudaImpl::NvCodecH264EncoderCudaImpl( +NvCodecVideoEncoderCudaImpl::NvCodecVideoEncoderCudaImpl( std::shared_ptr ctx) { ShowEncoderCapability(); cuda_context_ = ctx; } -NvCodecH264EncoderCudaImpl::~NvCodecH264EncoderCudaImpl() {} -void NvCodecH264EncoderCudaImpl::Copy(NvEncoder* nv_encoder, - const void* ptr, - int width, - int height) { +NvCodecVideoEncoderCudaImpl::~NvCodecVideoEncoderCudaImpl() {} +void NvCodecVideoEncoderCudaImpl::Copy(NvEncoder* nv_encoder, + const void* ptr, + int width, + int height) { const NvEncInputFrame* input_frame = nv_encoder->GetNextInputFrame(); CUcontext context = GetCudaContext(cuda_context_); NvEncoderCuda::CopyToDeviceFrame( @@ -168,9 +158,9 @@ void NvCodecH264EncoderCudaImpl::Copy(NvEncoder* nv_encoder, input_frame->bufferFormat, input_frame->chromaOffsets, input_frame->numChromaPlanes); } -NvEncoder* NvCodecH264EncoderCudaImpl::CreateNvEncoder(int width, - int height, - bool is_nv12) { +NvEncoder* NvCodecVideoEncoderCudaImpl::CreateNvEncoder(int width, + int height, + bool is_nv12) { NV_ENC_BUFFER_FORMAT nvenc_format = is_nv12 ? NV_ENC_BUFFER_FORMAT_NV12 : NV_ENC_BUFFER_FORMAT_IYUV; CUcontext context = GetCudaContext(cuda_context_); diff --git a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h b/src/hwenc_nvcodec/nvcodec_video_encoder_cuda.h similarity index 64% rename from src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h rename to src/hwenc_nvcodec/nvcodec_video_encoder_cuda.h index 424aa42d..1757dd3a 100644 --- a/src/hwenc_nvcodec/nvcodec_h264_encoder_cuda.h +++ b/src/hwenc_nvcodec/nvcodec_video_encoder_cuda.h @@ -1,5 +1,5 @@ -#ifndef SORA_HWENC_NVCODEC_NVCODEC_H264_ENCODER_CUDA_H_ -#define SORA_HWENC_NVCODEC_NVCODEC_H264_ENCODER_CUDA_H_ +#ifndef SORA_HWENC_NVCODEC_NVCODEC_VIDEO_ENCODER_CUDA_H_ +#define SORA_HWENC_NVCODEC_NVCODEC_VIDEO_ENCODER_CUDA_H_ // CUDA と WebRTC のヘッダを混ぜてコンパイルすると大量のエラーが発生したので、 // CUDA の処理だけさせる単純な CUDA ファイルを用意する @@ -12,19 +12,19 @@ class NvEncoder; namespace sora { -class NvCodecH264EncoderCudaImpl; +class NvCodecVideoEncoderCudaImpl; -class NvCodecH264EncoderCuda { +class NvCodecVideoEncoderCuda { public: - NvCodecH264EncoderCuda(std::shared_ptr ctx); - ~NvCodecH264EncoderCuda(); + NvCodecVideoEncoderCuda(std::shared_ptr ctx); + ~NvCodecVideoEncoderCuda(); void Copy(NvEncoder* nv_encoder, const void* ptr, int width, int height); // 念のため も include せずポインタを利用する NvEncoder* CreateNvEncoder(int width, int height, bool is_nv12); private: - NvCodecH264EncoderCudaImpl* impl_; + NvCodecVideoEncoderCudaImpl* impl_; }; } // namespace sora diff --git a/src/hwenc_vpl/vpl_video_decoder.cpp b/src/hwenc_vpl/vpl_video_decoder.cpp index 4bd459f2..5b837ba4 100644 --- a/src/hwenc_vpl/vpl_video_decoder.cpp +++ b/src/hwenc_vpl/vpl_video_decoder.cpp @@ -46,13 +46,15 @@ class VplVideoDecoderImpl : public VplVideoDecoder { std::shared_ptr session, mfxU32 codec, std::vector> sizes, - bool init); + bool init, + mfxFrameAllocRequest* alloc_request); static std::unique_ptr CreateDecoderInternal( std::shared_ptr session, mfxU32 codec, int width, int height, - bool init); + bool init, + mfxFrameAllocRequest* alloc_request); private: bool InitVpl(); @@ -89,10 +91,12 @@ std::unique_ptr VplVideoDecoderImpl::CreateDecoder( std::shared_ptr session, mfxU32 codec, std::vector> sizes, - bool init) { + bool init, + mfxFrameAllocRequest* alloc_request) { for (auto size : sizes) { - auto decoder = - CreateDecoderInternal(session, codec, size.first, size.second, init); + memset(alloc_request, 0, sizeof(*alloc_request)); + auto decoder = CreateDecoderInternal(session, codec, size.first, + size.second, init, alloc_request); if (decoder != nullptr) { return decoder; } @@ -105,7 +109,8 @@ std::unique_ptr VplVideoDecoderImpl::CreateDecoderInternal( mfxU32 codec, int width, int height, - bool init) { + bool init, + mfxFrameAllocRequest* alloc_request) { std::unique_ptr decoder( new MFXVideoDECODE(GetVplSession(session))); @@ -119,6 +124,11 @@ std::unique_ptr VplVideoDecoderImpl::CreateDecoderInternal( if (codec == MFX_CODEC_HEVC) { // この設定がないと H.265 デコーダーの Init が sts=-15 で失敗する param.mfx.CodecProfile = MFX_PROFILE_HEVC_MAIN; + } else if (codec == MFX_CODEC_AV1) { + // param.mfx.CodecProfile = MFX_PROFILE_AV1_MAIN; + // この設定がないと Query しても sts=-3 で失敗する + // MFX_LEVEL_AV1_2 で妥当かどうかはよく分からない + param.mfx.CodecLevel = MFX_LEVEL_AV1_2; } param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; @@ -148,23 +158,36 @@ std::unique_ptr VplVideoDecoderImpl::CreateDecoderInternal( sts = decoder->Query(¶m, ¶m); if (sts < 0) { - RTC_LOG(LS_VERBOSE) << "Unsupported decoder codec: codec=" - << CodecToString(codec) << " sts=" << sts; + RTC_LOG(LS_VERBOSE) << "Unsupported decoder codec: resolution=" << width + << "x" << height << " codec=" << CodecToString(codec) + << " sts=" << sts; return nullptr; } if (sts != MFX_ERR_NONE) { - RTC_LOG(LS_VERBOSE) << "Supported specified codec but has warning: sts=" - << sts; + RTC_LOG(LS_VERBOSE) + << "Supported specified codec but has warning: resolution=" << width + << "x" << height << " sts=" << sts; } + // MFX_CODEC_AV1 の時に Init より後に QueryIOSurf すると + // MFX_ERR_UNSUPPORTED になったのでここで QueryIOSurf しておく + // (MFX_CODEC_AVC や MFX_CODEC_HEVC の時には Init 後に QueryIOSurf しても動いた) + memset(alloc_request, 0, sizeof(*alloc_request)); + sts = decoder->QueryIOSurf(¶m, alloc_request); + VPL_CHECK_RESULT(sts, MFX_ERR_NONE, sts); + + RTC_LOG(LS_INFO) << "Decoder NumFrameSuggested=" + << alloc_request->NumFrameSuggested; + // Query した上で Init しても MFX_ERR_UNSUPPORTED になることがあるので // 本来 Init が不要な時も常に呼ぶようにして確認する /*if (init)*/ { // Initialize the Intel VPL encoder sts = decoder->Init(¶m); if (sts != MFX_ERR_NONE) { - RTC_LOG(LS_VERBOSE) << "Init failed: codec=" << CodecToString(codec) + RTC_LOG(LS_VERBOSE) << "Init failed: resolution=" << width << "x" + << height << " codec=" << CodecToString(codec) << " sts=" << sts; return nullptr; } @@ -311,8 +334,8 @@ const char* VplVideoDecoderImpl::ImplementationName() const { } bool VplVideoDecoderImpl::InitVpl() { - decoder_ = - CreateDecoder(session_, codec_, {{4096, 4096}, {2048, 2048}}, true); + decoder_ = CreateDecoder(session_, codec_, {{4096, 4096}, {2048, 2048}}, true, + &alloc_request_); mfxStatus sts = MFX_ERR_NONE; @@ -323,14 +346,6 @@ bool VplVideoDecoderImpl::InitVpl() { return false; } - // Query number of required surfaces for encoder - memset(&alloc_request_, 0, sizeof(alloc_request_)); - sts = decoder_->QueryIOSurf(¶m, &alloc_request_); - VPL_CHECK_RESULT(sts, MFX_ERR_NONE, sts); - - RTC_LOG(LS_INFO) << "Decoder NumFrameSuggested=" - << alloc_request_.NumFrameSuggested; - // 入力ビットストリーム bitstream_buffer_.resize(1024 * 1024); memset(&bitstream_, 0, sizeof(bitstream_)); @@ -380,8 +395,10 @@ bool VplVideoDecoder::IsSupported(std::shared_ptr session, return false; } + mfxFrameAllocRequest alloc_request; auto decoder = VplVideoDecoderImpl::CreateDecoder( - session, ToMfxCodec(codec), {{4096, 4096}, {2048, 2048}}, false); + session, ToMfxCodec(codec), {{4096, 4096}, {2048, 2048}}, false, + &alloc_request); return decoder != nullptr; } diff --git a/src/hwenc_vpl/vpl_video_encoder.cpp b/src/hwenc_vpl/vpl_video_encoder.cpp index 24aac7ab..8b6b79b7 100644 --- a/src/hwenc_vpl/vpl_video_encoder.cpp +++ b/src/hwenc_vpl/vpl_video_encoder.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -51,6 +53,24 @@ class VplVideoEncoderImpl : public VplVideoEncoder { int max_kbps, bool init); + private: + struct ExtBuffer { + mfxExtBuffer* ext_buffers[10]; + mfxExtCodingOption ext_coding_option; + mfxExtCodingOption2 ext_coding_option2; + }; + // いろいろなパターンでクエリを投げて、 + // 成功した時の param を返す + static mfxStatus Queries(MFXVideoENCODE* encoder, + mfxU32 codec, + int width, + int height, + int framerate, + int target_kbps, + int max_kbps, + mfxVideoParam& param, + ExtBuffer& ext); + private: std::mutex mutex_; webrtc::EncodedImageCallback* callback_ = nullptr; @@ -103,9 +123,51 @@ std::unique_ptr VplVideoEncoderImpl::CreateEncoder( int target_kbps, int max_kbps, bool init) { - mfxStatus sts = MFX_ERR_NONE; + std::unique_ptr encoder( + new MFXVideoENCODE(GetVplSession(session))); + + mfxPlatform platform; + memset(&platform, 0, sizeof(platform)); + MFXVideoCORE_QueryPlatform(GetVplSession(session), &platform); + RTC_LOG(LS_VERBOSE) << "--------------- codec=" << CodecToString(codec) + << " CodeName=" << platform.CodeName + << " DeviceId=" << platform.DeviceId + << " MediaAdapterType=" << platform.MediaAdapterType; mfxVideoParam param; + ExtBuffer ext; + mfxStatus sts = Queries(encoder.get(), codec, width, height, framerate, + target_kbps, max_kbps, param, ext); + if (sts < MFX_ERR_NONE) { + return nullptr; + } + if (sts > MFX_ERR_NONE) { + RTC_LOG(LS_VERBOSE) << "Supported specified codec but has warning: codec=" + << CodecToString(codec) << " sts=" << sts; + } + + if (init) { + sts = encoder->Init(¶m); + if (sts != MFX_ERR_NONE) { + RTC_LOG(LS_ERROR) << "Failed to Init: sts=" << sts; + return nullptr; + } + } + + return encoder; +} + +mfxStatus VplVideoEncoderImpl::Queries(MFXVideoENCODE* encoder, + mfxU32 codec, + int width, + int height, + int framerate, + int target_kbps, + int max_kbps, + mfxVideoParam& param, + ExtBuffer& ext) { + mfxStatus sts = MFX_ERR_NONE; + memset(¶m, 0, sizeof(param)); param.mfx.CodecId = codec; @@ -150,9 +212,9 @@ std::unique_ptr VplVideoEncoderImpl::CreateEncoder( param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; - mfxExtBuffer* ext_buffers[10]; - mfxExtCodingOption ext_coding_option; - mfxExtCodingOption2 ext_coding_option2; + mfxExtBuffer** ext_buffers = ext.ext_buffers; + mfxExtCodingOption& ext_coding_option = ext.ext_coding_option; + mfxExtCodingOption2& ext_coding_option2 = ext.ext_coding_option2; int ext_buffers_size = 0; if (codec == MFX_CODEC_AVC) { memset(&ext_coding_option, 0, sizeof(ext_coding_option)); @@ -194,106 +256,105 @@ std::unique_ptr VplVideoEncoderImpl::CreateEncoder( param.NumExtParam = ext_buffers_size; } - std::unique_ptr encoder( - new MFXVideoENCODE(GetVplSession(session))); - - mfxPlatform platform; - memset(&platform, 0, sizeof(platform)); - MFXVideoCORE_QueryPlatform(GetVplSession(session), &platform); - RTC_LOG(LS_VERBOSE) << "--------------- codec=" << CodecToString(codec) - << " CodeName=" << platform.CodeName - << " DeviceId=" << platform.DeviceId - << " MediaAdapterType=" << platform.MediaAdapterType; - - // MFX_ERR_NONE The function completed successfully. - // MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features. - // MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code. - // MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved. - mfxVideoParam bk_param; - memcpy(&bk_param, ¶m, sizeof(bk_param)); - sts = encoder->Query(¶m, ¶m); - if (sts < 0) { - RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" - << CodecToString(codec) << " sts=" << sts - << " ... Retry with low power mode"; - memcpy(¶m, &bk_param, sizeof(bk_param)); - - // 失敗したら LowPower ON にした状態でもう一度確認する - param.mfx.LowPower = MFX_CODINGOPTION_ON; - if (codec == MFX_CODEC_AVC || codec == MFX_CODEC_HEVC) { - param.mfx.RateControlMethod = MFX_RATECONTROL_CQP; - param.mfx.QPI = 25; - param.mfx.QPP = 33; - param.mfx.QPB = 40; - param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY; - } - memcpy(&bk_param, ¶m, sizeof(bk_param)); - sts = encoder->Query(¶m, ¶m); - if (sts < 0) { - RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" - << CodecToString(codec) << " sts=" << sts; - return nullptr; + // Query 関数を呼び出す。 + // 失敗した場合 param は一切書き換わらない + // 成功した場合 param は書き換わる可能性がある + auto query = [](MFXVideoENCODE* encoder, mfxVideoParam& param) { + mfxVideoParam query_param; + memcpy(&query_param, ¶m, sizeof(param)); + // ドキュメントによると、Query は以下のエラーを返す可能性がある。 + // MFX_ERR_NONE The function completed successfully. + // MFX_ERR_UNSUPPORTED The function failed to identify a specific implementation for the required features. + // MFX_WRN_PARTIAL_ACCELERATION The underlying hardware does not fully support the specified video parameters; The encoding may be partially accelerated. Only SDK HW implementations may return this status code. + // MFX_WRN_INCOMPATIBLE_VIDEO_PARAM The function detected some video parameters were incompatible with others; incompatibility resolved. + mfxStatus sts = encoder->Query(&query_param, &query_param); + if (sts >= 0) { + // デバッグ用。 + // Query によってどのパラメータが変更されたかを表示する + // #define F(NAME) \ + // if (param.NAME != query_param.NAME) \ + // std::cout << "param " << #NAME << " old=" << param.NAME \ + // << " new=" << query_param.NAME << std::endl + // F(mfx.LowPower); + // F(mfx.BRCParamMultiplier); + // F(mfx.FrameInfo.FrameRateExtN); + // F(mfx.FrameInfo.FrameRateExtD); + // F(mfx.FrameInfo.FourCC); + // F(mfx.FrameInfo.ChromaFormat); + // F(mfx.FrameInfo.PicStruct); + // F(mfx.FrameInfo.CropX); + // F(mfx.FrameInfo.CropY); + // F(mfx.FrameInfo.CropW); + // F(mfx.FrameInfo.CropH); + // F(mfx.FrameInfo.Width); + // F(mfx.FrameInfo.Height); + // F(mfx.CodecId); + // F(mfx.CodecProfile); + // F(mfx.CodecLevel); + // F(mfx.GopPicSize); + // F(mfx.GopRefDist); + // F(mfx.GopOptFlag); + // F(mfx.IdrInterval); + // F(mfx.TargetUsage); + // F(mfx.RateControlMethod); + // F(mfx.InitialDelayInKB); + // F(mfx.TargetKbps); + // F(mfx.MaxKbps); + // F(mfx.BufferSizeInKB); + // F(mfx.NumSlice); + // F(mfx.NumRefFrame); + // F(mfx.EncodedOrder); + // F(mfx.DecodedOrder); + // F(mfx.ExtendedPicStruct); + // F(mfx.TimeStampCalc); + // F(mfx.SliceGroupsPresent); + // F(mfx.MaxDecFrameBuffering); + // F(mfx.EnableReallocRequest); + // F(AsyncDepth); + // F(IOPattern); + // #undef F + + memcpy(¶m, &query_param, sizeof(param)); } + return sts; + }; + + // ここからは、ひたすらパラメータを変えて query を呼び出していく + sts = query(encoder, param); + if (sts >= 0) { + return sts; } - //#define F(NAME) \ - // if (bk_param.NAME != param.NAME) \ - // std::cout << "param " << #NAME << " old=" << bk_param.NAME \ - // << " new=" << param.NAME << std::endl - // - // F(mfx.LowPower); - // F(mfx.BRCParamMultiplier); - // F(mfx.FrameInfo.FrameRateExtN); - // F(mfx.FrameInfo.FrameRateExtD); - // F(mfx.FrameInfo.FourCC); - // F(mfx.FrameInfo.ChromaFormat); - // F(mfx.FrameInfo.PicStruct); - // F(mfx.FrameInfo.CropX); - // F(mfx.FrameInfo.CropY); - // F(mfx.FrameInfo.CropW); - // F(mfx.FrameInfo.CropH); - // F(mfx.FrameInfo.Width); - // F(mfx.FrameInfo.Height); - // F(mfx.CodecId); - // F(mfx.CodecProfile); - // F(mfx.CodecLevel); - // F(mfx.GopPicSize); - // F(mfx.GopRefDist); - // F(mfx.GopOptFlag); - // F(mfx.IdrInterval); - // F(mfx.TargetUsage); - // F(mfx.RateControlMethod); - // F(mfx.InitialDelayInKB); - // F(mfx.TargetKbps); - // F(mfx.MaxKbps); - // F(mfx.BufferSizeInKB); - // F(mfx.NumSlice); - // F(mfx.NumRefFrame); - // F(mfx.EncodedOrder); - // F(mfx.DecodedOrder); - // F(mfx.ExtendedPicStruct); - // F(mfx.TimeStampCalc); - // F(mfx.SliceGroupsPresent); - // F(mfx.MaxDecFrameBuffering); - // F(mfx.EnableReallocRequest); - // F(AsyncDepth); - // F(IOPattern); - //#undef F - - if (sts != MFX_ERR_NONE) { - RTC_LOG(LS_VERBOSE) << "Supported specified codec but has warning: codec=" - << CodecToString(codec) << " sts=" << sts; + // IOPattern を MFX_IOPATTERN_IN_SYSTEM_MEMORY のみにしてみる + // Coffee Lake の H265 はこのパターンでないと通らない + RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" + << CodecToString(codec) << " sts=" << sts + << " ... Retry with IOPattern IN_SYSTEM_MEMORY only"; + param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY; + sts = query(encoder, param); + if (sts >= 0) { + return sts; } - if (init) { - sts = encoder->Init(¶m); - if (sts != MFX_ERR_NONE) { - RTC_LOG(LS_VERBOSE) << "Failed to Init: sts=" << sts; - return nullptr; - } + // LowPower ON にして、更に H264/H265 は固定 QP モードにしてみる + RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" + << CodecToString(codec) << " sts=" << sts + << " ... Retry with low power mode"; + param.mfx.LowPower = MFX_CODINGOPTION_ON; + if (codec == MFX_CODEC_AVC || codec == MFX_CODEC_HEVC) { + param.mfx.RateControlMethod = MFX_RATECONTROL_CQP; + param.mfx.QPI = 25; + param.mfx.QPP = 33; + param.mfx.QPB = 40; + } + sts = query(encoder, param); + if (sts >= 0) { + return sts; } + RTC_LOG(LS_VERBOSE) << "Unsupported encoder codec: codec=" + << CodecToString(codec) << " sts=" << sts; - return encoder; + return sts; } int32_t VplVideoEncoderImpl::InitEncode( @@ -608,9 +669,27 @@ bool VplVideoEncoder::IsSupported(std::shared_ptr session, return false; } + // FIXME(melpon): IsSupported(VP9) == true になるにも関わらず、実際に使ってみると + // 実行時エラーでクラッシュするため、とりあえず VP9 だったら未サポートとして返す。 + // (VPL の問題なのか使い方の問題なのかは不明) + if (codec == webrtc::kVideoCodecVP9) { + return false; + } + + // FIXME(miosakuma): Intel Core Ultra 7 では IsSupported(AV1) == true となるが、 + // 実際に使ってみると映像が送信されないため、一時的に AV1 だったら未サポートとして返す。 + // (VPL の問題なのか使い方の問題なのかは不明) + if (codec == webrtc::kVideoCodecAV1) { + return false; + } + auto encoder = VplVideoEncoderImpl::CreateEncoder( session, ToMfxCodec(codec), 1920, 1080, 30, 10, 20, false); - return encoder != nullptr; + bool result = encoder != nullptr; + RTC_LOG(LS_VERBOSE) << "IsSupported: codec=" + << CodecToString(ToMfxCodec(codec)) + << " result=" << result; + return result; } std::unique_ptr VplVideoEncoder::Create( diff --git a/src/open_h264_video_encoder.cpp b/src/open_h264_video_encoder.cpp index a13a0730..e24b73cb 100644 --- a/src/open_h264_video_encoder.cpp +++ b/src/open_h264_video_encoder.cpp @@ -28,12 +28,22 @@ #endif // WebRTC +#include #include +#include +#include +#include #include +#include +#include #include #include #include #include +#include +#include +#include +#include #include #include #include @@ -50,16 +60,11 @@ #include #include -namespace webrtc { +class ISVCEncoder; -class OpenH264VideoEncoder : public H264Encoder { - public: - static std::unique_ptr Create(const cricket::VideoCodec& codec, - std::string openh264) { - return std::unique_ptr( - new OpenH264VideoEncoder(codec, std::move(openh264))); - } +namespace webrtc { +class OpenH264VideoEncoder : public VideoEncoder { public: struct LayerConfig { int simulcast_idx = 0; @@ -77,13 +82,14 @@ class OpenH264VideoEncoder : public H264Encoder { void SetStreamState(bool send_stream); }; - public: - explicit OpenH264VideoEncoder(const cricket::VideoCodec& codec, - std::string openh264); + OpenH264VideoEncoder(const Environment& env, + H264EncoderSettings settings, + std::string openh264); + ~OpenH264VideoEncoder() override; - // |settings.max_payload_size| is ignored. - // The following members of |codec_settings| are used. The rest are ignored. + // `settings.max_payload_size` is ignored. + // The following members of `codec_settings` are used. The rest are ignored. // - codecType (must be kVideoCodecH264) // - targetBitrate // - maxFramerate @@ -122,11 +128,16 @@ class OpenH264VideoEncoder : public H264Encoder { std::vector> downscaled_buffers_; std::vector configurations_; std::vector encoded_images_; + std::vector> svc_controllers_; + absl::InlinedVector, kMaxSimulcastStreams> + scalability_modes_; + const Environment env_; VideoCodec codec_; H264PacketizationMode packetization_mode_; size_t max_payload_size_; int32_t number_of_cores_; + absl::optional encoder_thread_limit_; EncodedImageCallback* encoded_image_callback_; bool has_reported_init_; @@ -169,19 +180,30 @@ enum H264EncoderImplEvent { kH264EncoderEventMax = 16, }; -int NumberOfThreads(int width, int height, int number_of_cores) { +int NumberOfThreads(absl::optional encoder_thread_limit, + int width, + int height, + int number_of_cores) { // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac, // see crbug.com/583348. Until further investigated, only use one thread. - // if (width * height >= 1920 * 1080 && number_of_cores > 8) { - // return 8; // 8 threads for 1080p on high perf machines. - // } else if (width * height > 1280 * 960 && number_of_cores >= 6) { - // return 3; // 3 threads for 1080p. - // } else if (width * height > 640 * 480 && number_of_cores >= 3) { - // return 2; // 2 threads for qHD/HD. - // } else { - // return 1; // 1 thread for VGA or less. - // } - // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(), + // While this limitation is gone, this changes the bitstream format (see + // bugs.webrtc.org/14368) so still guarded by field trial to allow for + // experimentation using th experimental + // WebRTC-VideoEncoderSettings/encoder_thread_limit trial. + if (encoder_thread_limit.has_value()) { + int limit = encoder_thread_limit.value(); + RTC_DCHECK_GE(limit, 1); + if (width * height >= 1920 * 1080 && number_of_cores > 8) { + return std::min(limit, 8); // 8 threads for 1080p on high perf machines. + } else if (width * height > 1280 * 960 && number_of_cores >= 6) { + return std::min(limit, 3); // 3 threads for 1080p. + } else if (width * height > 640 * 480 && number_of_cores >= 3) { + return std::min(limit, 2); // 2 threads for qHD/HD. + } else { + return 1; // 1 thread for VGA or less. + } + } + // TODO(sprang): Also check sSliceArgument.uiSliceNum on GetEncoderParams(), // before enabling multithreading here. return 1; } @@ -202,17 +224,34 @@ VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) { return VideoFrameType::kEmptyFrame; } +absl::optional ScalabilityModeFromTemporalLayers( + int num_temporal_layers) { + switch (num_temporal_layers) { + case 0: + break; + case 1: + return ScalabilityMode::kL1T1; + case 2: + return ScalabilityMode::kL1T2; + case 3: + return ScalabilityMode::kL1T3; + default: + RTC_DCHECK_NOTREACHED(); + } + return absl::nullopt; +} + } // namespace // Helper method used by OpenH264VideoEncoder::Encode. -// Copies the encoded bytes from |info| to |encoded_image|. The -// |encoded_image->_buffer| may be deleted and reallocated if a bigger buffer is +// Copies the encoded bytes from `info` to `encoded_image`. The +// `encoded_image->_buffer` may be deleted and reallocated if a bigger buffer is // required. // -// After OpenH264 encoding, the encoded bytes are stored in |info| spread out +// After OpenH264 encoding, the encoded bytes are stored in `info` spread out // over a number of layers and "NAL units". Each NAL unit is a fragment starting // with the four-byte start code {0,0,0,1}. All of this data (including the -// start codes) is copied to the |encoded_image->_buffer|. +// start codes) is copied to the `encoded_image->_buffer`. static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { // Calculate minimum buffer size required to hold encoded data. size_t required_capacity = 0; @@ -221,18 +260,17 @@ static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) { RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0); - // Ensure |required_capacity| will not overflow. + // Ensure `required_capacity` will not overflow. RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal], std::numeric_limits::max() - required_capacity); required_capacity += layerInfo.pNalLengthInByte[nal]; } } - // TODO(nisse): Use a cache or buffer pool to avoid allocation? auto buffer = EncodedImageBuffer::Create(required_capacity); encoded_image->SetEncodedData(buffer); // Iterate layers and NAL units, note each NAL unit as a fragment and copy - // the data to |encoded_image->_buffer|. + // the data to `encoded_image->_buffer`. const uint8_t start_code[4] = {0, 0, 0, 1}; size_t frag = 0; encoded_image->set_size(0); @@ -241,8 +279,8 @@ static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { // Iterate NAL units making up this layer, noting fragments. size_t layer_len = 0; for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) { - // Because the sum of all layer lengths, |required_capacity|, fits in a - // |size_t|, we know that any indices in-between will not overflow. + // Because the sum of all layer lengths, `required_capacity`, fits in a + // `size_t`, we know that any indices in-between will not overflow. RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]); @@ -256,27 +294,23 @@ static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { } } -OpenH264VideoEncoder::OpenH264VideoEncoder(const cricket::VideoCodec& codec, +OpenH264VideoEncoder::OpenH264VideoEncoder(const Environment& env, + H264EncoderSettings settings, std::string openh264) - : packetization_mode_(H264PacketizationMode::SingleNalUnit), + : env_(env), + packetization_mode_(settings.packetization_mode), max_payload_size_(0), number_of_cores_(0), encoded_image_callback_(nullptr), has_reported_init_(false), has_reported_error_(false), openh264_(std::move(openh264)) { - RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName)); - std::string packetization_mode_string; - if (codec.GetParam(cricket::kH264FmtpPacketizationMode, - &packetization_mode_string) && - packetization_mode_string == "1") { - packetization_mode_ = H264PacketizationMode::NonInterleaved; - } downscaled_buffers_.reserve(kMaxSimulcastStreams - 1); encoded_images_.reserve(kMaxSimulcastStreams); encoders_.reserve(kMaxSimulcastStreams); configurations_.reserve(kMaxSimulcastStreams); tl0sync_limit_.reserve(kMaxSimulcastStreams); + svc_controllers_.reserve(kMaxSimulcastStreams); } OpenH264VideoEncoder::~OpenH264VideoEncoder() { @@ -377,11 +411,14 @@ int32_t OpenH264VideoEncoder::InitEncode( encoded_images_.resize(number_of_streams); encoders_.resize(number_of_streams); pictures_.resize(number_of_streams); + svc_controllers_.resize(number_of_streams); + scalability_modes_.resize(number_of_streams); configurations_.resize(number_of_streams); tl0sync_limit_.resize(number_of_streams); - number_of_cores_ = settings.number_of_cores; max_payload_size_ = settings.max_payload_size; + number_of_cores_ = settings.number_of_cores; + encoder_thread_limit_ = settings.encoder_thread_limit; codec_ = *inst; // Code expects simulcastStream resolutions to be correct, make sure they are @@ -419,9 +456,11 @@ int32_t OpenH264VideoEncoder::InitEncode( configurations_[i].width = codec_.simulcastStream[idx].width; configurations_[i].height = codec_.simulcastStream[idx].height; configurations_[i].max_frame_rate = static_cast(codec_.maxFramerate); + configurations_[i].frame_dropping_on = codec_.GetFrameDropEnabled(); configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval; configurations_[i].num_temporal_layers = - codec_.simulcastStream[idx].numberOfTemporalLayers; + std::max(codec_.H264()->numberOfTemporalLayers, + codec_.simulcastStream[idx].numberOfTemporalLayers); // Create downscaled image buffers. if (i > 0) { @@ -460,9 +499,20 @@ int32_t OpenH264VideoEncoder::InitEncode( encoded_images_[i].set_size(0); tl0sync_limit_[i] = configurations_[i].num_temporal_layers; + scalability_modes_[i] = ScalabilityModeFromTemporalLayers( + configurations_[i].num_temporal_layers); + if (scalability_modes_[i].has_value()) { + svc_controllers_[i] = CreateScalabilityStructure(*scalability_modes_[i]); + if (svc_controllers_[i] == nullptr) { + RTC_LOG(LS_ERROR) << "Failed to create scalability structure"; + Release(); + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + } } - SimulcastRateAllocator init_allocator(codec_); + SimulcastRateAllocator init_allocator(env_, codec_); VideoBitrateAllocation allocation = init_allocator.Allocate(VideoBitrateAllocationParameters( DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate)); @@ -484,6 +534,8 @@ int32_t OpenH264VideoEncoder::Release() { encoded_images_.clear(); pictures_.clear(); tl0sync_limit_.clear(); + svc_controllers_.clear(); + scalability_modes_.clear(); return WEBRTC_VIDEO_CODEC_OK; } @@ -553,29 +605,29 @@ int32_t OpenH264VideoEncoder::Encode( return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - rtc::scoped_refptr frame_buffer = + rtc::scoped_refptr frame_buffer = input_frame.video_frame_buffer()->ToI420(); + if (!frame_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert " + << VideoFrameBufferTypeToString( + input_frame.video_frame_buffer()->type()) + << " image to I420. Can't encode frame."; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + RTC_CHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420 || + frame_buffer->type() == VideoFrameBuffer::Type::kI420A); - bool send_key_frame = false; + bool is_keyframe_needed = false; for (size_t i = 0; i < configurations_.size(); ++i) { if (configurations_[i].key_frame_request && configurations_[i].sending) { - send_key_frame = true; + // This is legacy behavior, generating a keyframe on all layers + // when generating one for a layer that became active for the first time + // or after being disabled. + is_keyframe_needed = true; break; } } - if (!send_key_frame && frame_types) { - for (size_t i = 0; i < configurations_.size(); ++i) { - const size_t simulcast_idx = - static_cast(configurations_[i].simulcast_idx); - if (configurations_[i].sending && simulcast_idx < frame_types->size() && - (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) { - send_key_frame = true; - break; - } - } - } - RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width()); RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height()); @@ -614,21 +666,29 @@ int32_t OpenH264VideoEncoder::Encode( pictures_[i].iStride[0], pictures_[i].pData[1], pictures_[i].iStride[1], pictures_[i].pData[2], pictures_[i].iStride[2], configurations_[i].width, - configurations_[i].height, libyuv::kFilterBilinear); + configurations_[i].height, libyuv::kFilterBox); } if (!configurations_[i].sending) { continue; } - if (frame_types != nullptr) { + if (frame_types != nullptr && i < frame_types->size()) { // Skip frame? if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) { continue; } } + // Send a key frame either when this layer is configured to require one + // or we have explicitly been asked to. + const size_t simulcast_idx = + static_cast(configurations_[i].simulcast_idx); + bool send_key_frame = + is_keyframe_needed || + (frame_types && simulcast_idx < frame_types->size() && + (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey); if (send_key_frame) { // API doc says ForceIntraFrame(false) does nothing, but calling this - // function forces a key frame regardless of the |bIDR| argument's value. + // function forces a key frame regardless of the `bIDR` argument's value. // (If every frame is a key frame we get lag/delays.) encoders_[i]->ForceIntraFrame(true); configurations_[i].key_frame_request = false; @@ -637,6 +697,12 @@ int32_t OpenH264VideoEncoder::Encode( SFrameBSInfo info; memset(&info, 0, sizeof(SFrameBSInfo)); + std::vector layer_frames; + if (svc_controllers_[i]) { + layer_frames = svc_controllers_[i]->NextFrameConfig(send_key_frame); + RTC_CHECK_EQ(layer_frames.size(), 1); + } + // Encode! int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info); if (enc_ret != 0) { @@ -649,16 +715,17 @@ int32_t OpenH264VideoEncoder::Encode( encoded_images_[i]._encodedWidth = configurations_[i].width; encoded_images_[i]._encodedHeight = configurations_[i].height; - encoded_images_[i].SetRtpTimestamp(input_frame.timestamp()); + encoded_images_[i].SetRtpTimestamp(input_frame.rtp_timestamp()); + encoded_images_[i].SetColorSpace(input_frame.color_space()); encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType); - encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx); + encoded_images_[i].SetSimulcastIndex(configurations_[i].simulcast_idx); // Split encoded image up into fragments. This also updates - // |encoded_image_|. + // `encoded_image_`. RtpFragmentize(&encoded_images_[i], &info); // Encoder can skip frames to save bandwidth in which case - // |encoded_images_[i]._length| == 0. + // `encoded_images_[i]._length` == 0. if (encoded_images_[i].size() > 0) { // Parse QP. h264_bitstream_parser_.ParseBitstream(encoded_images_[i]); @@ -679,6 +746,25 @@ int32_t OpenH264VideoEncoder::Encode( codec_specific.codecSpecific.H264.temporal_idx = tid; codec_specific.codecSpecific.H264.base_layer_sync = tid > 0 && tid < tl0sync_limit_[i]; + if (svc_controllers_[i]) { + if (encoded_images_[i]._frameType == VideoFrameType::kVideoFrameKey) { + // Reset the ScalableVideoController on key frame + // to reset the expected dependency structure. + layer_frames = + svc_controllers_[i]->NextFrameConfig(/* restart= */ true); + RTC_CHECK_EQ(layer_frames.size(), 1); + RTC_DCHECK_EQ(layer_frames[0].TemporalId(), 0); + RTC_DCHECK_EQ(layer_frames[0].IsKeyframe(), true); + } + + if (layer_frames[0].TemporalId() != tid) { + RTC_LOG(LS_WARNING) + << "Encoder produced a frame with temporal id " << tid + << ", expected " << layer_frames[0].TemporalId() << "."; + continue; + } + encoded_images_[i].SetTemporalIndex(tid); + } if (codec_specific.codecSpecific.H264.base_layer_sync) { tl0sync_limit_[i] = tid; } @@ -686,6 +772,15 @@ int32_t OpenH264VideoEncoder::Encode( tl0sync_limit_[i] = configurations_[i].num_temporal_layers; } } + if (svc_controllers_[i]) { + codec_specific.generic_frame_info = + svc_controllers_[i]->OnEncodeDone(layer_frames[0]); + if (send_key_frame && codec_specific.generic_frame_info.has_value()) { + codec_specific.template_structure = + svc_controllers_[i]->DependencyStructure(); + } + codec_specific.scalability_mode = scalability_modes_[i]; + } encoded_image_callback_->OnEncodedImage(encoded_images_[i], &codec_specific); } @@ -721,8 +816,8 @@ SEncParamExt OpenH264VideoEncoder::CreateEncoderParams(size_t i) const { // The following parameters are extension parameters (they're in SEncParamExt, // not in SEncParamBase). encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on; - // |uiIntraPeriod| - multiple of GOP size - // |keyFrameInterval| - number of frames + // `uiIntraPeriod` - multiple of GOP size + // `keyFrameInterval` - number of frames encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval; // Reuse SPS id if possible. This helps to avoid reset of chromium HW decoder // on each key-frame. @@ -735,8 +830,9 @@ SEncParamExt OpenH264VideoEncoder::CreateEncoderParams(size_t i) const { // 0: auto (dynamic imp. internal encoder) // 1: single thread (default value) // >1: number of threads - encoder_params.iMultipleThreadIdc = NumberOfThreads( - encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_); + encoder_params.iMultipleThreadIdc = + NumberOfThreads(encoder_thread_limit_, encoder_params.iPicWidth, + encoder_params.iPicHeight, number_of_cores_); // The base spatial layer 0 is the only one we use. encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth; encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight; @@ -747,7 +843,13 @@ SEncParamExt OpenH264VideoEncoder::CreateEncoderParams(size_t i) const { encoder_params.iMaxBitrate; encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers; if (encoder_params.iTemporalLayerNum > 1) { - encoder_params.iNumRefFrame = 1; + // iNumRefFrame specifies total number of reference buffers to allocate. + // For N temporal layers we need at least (N - 1) buffers to store last + // encoded frames of all reference temporal layers. + // Note that there is no API in OpenH264 encoder to specify exact set of + // references to be used to prediction of a given frame. Encoder can + // theoretically use all available reference buffers. + encoder_params.iNumRefFrame = encoder_params.iTemporalLayerNum - 1; } RTC_LOG(LS_INFO) << "OpenH264 version is " << OPENH264_MAJOR << "." << OPENH264_MINOR; @@ -816,9 +918,22 @@ void OpenH264VideoEncoder::LayerConfig::SetStreamState(bool send_stream) { namespace sora { std::unique_ptr CreateOpenH264VideoEncoder( - const cricket::VideoCodec& codec, + const webrtc::SdpVideoFormat& format, std::string openh264) { - return webrtc::OpenH264VideoEncoder::Create(codec, std::move(openh264)); + webrtc::H264EncoderSettings settings; + if (auto it = format.parameters.find(cricket::kH264FmtpPacketizationMode); + it != format.parameters.end()) { + if (it->second == "0") { + settings.packetization_mode = + webrtc::H264PacketizationMode::SingleNalUnit; + } else if (it->second == "1") { + settings.packetization_mode = + webrtc::H264PacketizationMode::NonInterleaved; + } + } + + return absl::make_unique( + webrtc::CreateEnvironment(), settings, std::move(openh264)); } } // namespace sora diff --git a/src/sora_peer_connection_factory.cpp b/src/sora_peer_connection_factory.cpp index 4ab8922e..643aacbe 100644 --- a/src/sora_peer_connection_factory.cpp +++ b/src/sora_peer_connection_factory.cpp @@ -14,6 +14,7 @@ class PeerConnectionFactoryWithContext : public webrtc::PeerConnectionFactory { PeerConnectionFactoryWithContext( webrtc::PeerConnectionFactoryDependencies dependencies) : PeerConnectionFactoryWithContext( + // SDK の外部から webrtc::Environment を設定したくなるまで、ここで初期化する webrtc::ConnectionContext::Create(webrtc::CreateEnvironment(), &dependencies), &dependencies) {} diff --git a/src/sora_signaling.cpp b/src/sora_signaling.cpp index eb1e2daa..4cc8dd3f 100644 --- a/src/sora_signaling.cpp +++ b/src/sora_signaling.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "sora/data_channel.h" #include "sora/rtc_ssl_verifier.h" @@ -314,6 +316,10 @@ void SoraSignaling::DoSendConnect(bool redirect) { m["video"].as_object()["h264_params"] = config_.video_h264_params; } + if (!config_.video_h265_params.is_null()) { + m["video"].as_object()["h265_params"] = config_.video_h265_params; + } + // オプションの設定が行われてなければ単に true を設定 if (m["video"].as_object().empty()) { m["video"] = true; @@ -441,7 +447,7 @@ void SoraSignaling::DoSendUpdate(const std::string& sdp, std::string type) { } } -class RawCryptString : public rtc::CryptStringImpl { +class RawCryptString : public rtc::revive::CryptStringImpl { public: RawCryptString(const std::string& str) : str_(str) {} size_t GetLength() const override { return str_.size(); } @@ -514,11 +520,11 @@ SoraSignaling::CreatePeerConnection(boost::json::value jconfig) { dependencies.allocator->set_flags(rtc_config.port_allocator_config.flags); RTC_LOG(LS_INFO) << "Set Proxy: type=" - << rtc::ProxyToString(rtc::PROXY_HTTPS) + << rtc::revive::ProxyToString(rtc::revive::PROXY_HTTPS) << " url=" << config_.proxy_url << " username=" << config_.proxy_username; - rtc::ProxyInfo pi; - pi.type = rtc::PROXY_HTTPS; + rtc::revive::ProxyInfo pi; + pi.type = rtc::revive::PROXY_HTTPS; URLParts parts; if (!URLParts::Parse(config_.proxy_url, parts)) { RTC_LOG(LS_ERROR) << "Failed to parse: proxy_url=" << config_.proxy_url; @@ -529,7 +535,7 @@ SoraSignaling::CreatePeerConnection(boost::json::value jconfig) { pi.username = config_.proxy_username; } if (!config_.proxy_password.empty()) { - pi.password = rtc::CryptString(RawCryptString(config_.proxy_password)); + pi.password = rtc::revive::CryptString(RawCryptString(config_.proxy_password)); } std::string proxy_agent = "Sora C++ SDK"; if (!config_.proxy_agent.empty()) { @@ -1172,6 +1178,9 @@ void SoraSignaling::DoConnect() { } else { ws.reset(new Websocket(*config_.io_context)); } + if (config_.user_agent != boost::none) { + ws->SetUserAgent(*config_.user_agent); + } ws->Connect(url, std::bind(&SoraSignaling::OnConnect, shared_from_this(), std::placeholders::_1, url, ws)); connecting_wss_.push_back(ws); diff --git a/src/sora_video_decoder_factory.cpp b/src/sora_video_decoder_factory.cpp index f2da1614..f817a720 100644 --- a/src/sora_video_decoder_factory.cpp +++ b/src/sora_video_decoder_factory.cpp @@ -2,6 +2,7 @@ // WebRTC #include +#include #include #include #include @@ -31,10 +32,6 @@ #include "sora/hwenc_vpl/vpl_video_decoder.h" #endif -#if defined(USE_JETSON_ENCODER) -#include "sora/hwenc_jetson/jetson_video_decoder.h" -#endif - #include "default_video_formats.h" namespace sora { @@ -67,8 +64,8 @@ SoraVideoDecoderFactory::GetSupportedFormats() const { return r; } -std::unique_ptr -SoraVideoDecoderFactory::CreateVideoDecoder( +std::unique_ptr SoraVideoDecoderFactory::Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) { webrtc::VideoCodecType specified_codec = webrtc::PayloadStringToCodecType(format.name); @@ -84,10 +81,10 @@ SoraVideoDecoderFactory::CreateVideoDecoder( std::vector supported_formats = formats_[n++]; if (enc.factory != nullptr) { - create_video_decoder = - [factory = enc.factory.get()](const webrtc::SdpVideoFormat& format) { - return factory->CreateVideoDecoder(format); - }; + create_video_decoder = [factory = enc.factory.get(), + env](const webrtc::SdpVideoFormat& format) { + return factory->Create(env, format); + }; } else if (enc.create_video_decoder != nullptr) { create_video_decoder = enc.create_video_decoder; } @@ -159,6 +156,17 @@ SoraVideoDecoderFactoryConfig GetDefaultVideoDecoderFactoryConfig( cuda_context, CudaVideoCodec::H264)); })); } + if (NvCodecVideoDecoder::IsSupported(cuda_context, + sora::CudaVideoCodec::H265)) { + config.decoders.insert( + config.decoders.begin(), + VideoDecoderConfig(webrtc::kVideoCodecH265, + [cuda_context = cuda_context](auto format) { + return std::unique_ptr( + absl::make_unique( + cuda_context, CudaVideoCodec::H265)); + })); + } #endif #if USE_VPL_ENCODER @@ -215,45 +223,16 @@ SoraVideoDecoderFactoryConfig GetDefaultVideoDecoderFactoryConfig( } #endif -#if defined(USE_JETSON_ENCODER) - if (JetsonVideoDecoder::IsSupportedVP8()) { - config.decoders.insert( - config.decoders.begin(), - VideoDecoderConfig(webrtc::kVideoCodecVP8, [](auto format) { - return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecVP8)); - })); - } - if (JetsonVideoDecoder::IsSupportedAV1()) { - config.decoders.insert( - config.decoders.begin(), - VideoDecoderConfig(webrtc::kVideoCodecAV1, [](auto format) { - return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecAV1)); - })); - } - config.decoders.insert( - config.decoders.begin(), - VideoDecoderConfig(webrtc::kVideoCodecVP9, [](auto format) { - return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecVP9)); - })); - config.decoders.insert( - config.decoders.begin(), - VideoDecoderConfig(webrtc::kVideoCodecH264, [](auto format) { - return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecH264)); - })); -#endif - return config; } SoraVideoDecoderFactoryConfig GetSoftwareOnlyVideoDecoderFactoryConfig() { + // SDK の外部から webrtc::Environment を設定したくなるまで、ここで初期化する + auto env = webrtc::CreateEnvironment(); SoraVideoDecoderFactoryConfig config; config.decoders.push_back(VideoDecoderConfig( webrtc::kVideoCodecVP8, - [](auto format) { return webrtc::VP8Decoder::Create(); })); + [env](auto format) { return webrtc::CreateVp8Decoder(env); })); config.decoders.push_back(VideoDecoderConfig( webrtc::kVideoCodecVP9, [](auto format) { return webrtc::VP9Decoder::Create(); })); diff --git a/src/sora_video_encoder_factory.cpp b/src/sora_video_encoder_factory.cpp index a388a3f9..a42592e6 100644 --- a/src/sora_video_encoder_factory.cpp +++ b/src/sora_video_encoder_factory.cpp @@ -3,6 +3,7 @@ // WebRTC #include #include +#include #include #include #include @@ -27,17 +28,13 @@ #endif #if defined(USE_NVCODEC_ENCODER) -#include "sora/hwenc_nvcodec/nvcodec_h264_encoder.h" +#include "sora/hwenc_nvcodec/nvcodec_video_encoder.h" #endif #if defined(USE_VPL_ENCODER) #include "sora/hwenc_vpl/vpl_video_encoder.h" #endif -#if defined(USE_JETSON_ENCODER) -#include "sora/hwenc_jetson/jetson_video_encoder.h" -#endif - #include "default_video_formats.h" #include "sora/aligned_encoder_adapter.h" #include "sora/i420_encoder_adapter.h" @@ -81,6 +78,7 @@ SoraVideoEncoderFactory::GetSupportedFormats() const { std::unique_ptr SoraVideoEncoderFactory::CreateInternalVideoEncoder( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format, int& alignment) { if (formats_.empty()) { @@ -92,43 +90,47 @@ SoraVideoEncoderFactory::CreateInternalVideoEncoder( int n = 0; for (auto& enc : config_.encoders) { - // 対応していないフォーマットを CreateVideoEncoder に渡した時の挙動は未定義なので - // 確実に対応してるフォーマットのみを CreateVideoEncoder に渡すようにする。 + // 対応していないフォーマットを Create に渡した時の挙動は未定義なので + // 確実に対応してるフォーマットのみを Create に渡すようにする。 std::function( - const webrtc::SdpVideoFormat&)> + const webrtc::Environment&, const webrtc::SdpVideoFormat&)> create_video_encoder; std::vector supported_formats = formats_[n++]; if (enc.factory != nullptr) { - create_video_encoder = - [factory = enc.factory.get()](const webrtc::SdpVideoFormat& format) { - return factory->CreateVideoEncoder(format); - }; + create_video_encoder = [factory = enc.factory.get()]( + const webrtc::Environment& env, + const webrtc::SdpVideoFormat& format) { + return factory->Create(env, format); + }; // Factory 経由で作ったエンコーダはアライメントが必要かどうか分からないので全部アライメントする alignment = 16; } else if (enc.create_video_encoder != nullptr) { - create_video_encoder = enc.create_video_encoder; + create_video_encoder = [&enc](const webrtc::Environment& env, + const webrtc::SdpVideoFormat& format) { + return enc.create_video_encoder(format); + }; alignment = enc.alignment; } for (const auto& f : supported_formats) { if (f.IsSameCodec(format)) { - return create_video_encoder(format); + return create_video_encoder(env, format); } } } return nullptr; } -std::unique_ptr -SoraVideoEncoderFactory::CreateVideoEncoder( +std::unique_ptr SoraVideoEncoderFactory::Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) { if (internal_encoder_factory_ != nullptr) { // サイマルキャストの場合はアダプタを噛ましつつ、無条件ですべてアライメントする std::unique_ptr encoder = std::make_unique( - internal_encoder_factory_.get(), format); + env, internal_encoder_factory_.get(), nullptr, format); if (config_.force_i420_conversion_for_simulcast_adapter) { encoder = std::make_unique(std::move(encoder)); @@ -140,7 +142,7 @@ SoraVideoEncoderFactory::CreateVideoEncoder( } int alignment = 0; - auto encoder = CreateInternalVideoEncoder(format, alignment); + auto encoder = CreateInternalVideoEncoder(env, format, alignment); if (encoder == nullptr) { return nullptr; } @@ -181,17 +183,27 @@ SoraVideoEncoderFactoryConfig GetDefaultVideoEncoderFactoryConfig( #endif #if defined(USE_NVCODEC_ENCODER) - if (NvCodecH264Encoder::IsSupported(cuda_context)) { - config.encoders.insert( - config.encoders.begin(), - VideoEncoderConfig( - webrtc::kVideoCodecH264, - [cuda_context = cuda_context]( - auto format) -> std::unique_ptr { - return NvCodecH264Encoder::Create( - cricket::CreateVideoCodec(format), cuda_context); - }, - 16)); + if (NvCodecVideoEncoder::IsSupported(cuda_context, CudaVideoCodec::H264)) { + config.encoders.insert(config.encoders.begin(), + VideoEncoderConfig( + webrtc::kVideoCodecH264, + [cuda_context = cuda_context](auto format) + -> std::unique_ptr { + return NvCodecVideoEncoder::Create( + cuda_context, CudaVideoCodec::H264); + }, + 16)); + } + if (NvCodecVideoEncoder::IsSupported(cuda_context, CudaVideoCodec::H265)) { + config.encoders.insert(config.encoders.begin(), + VideoEncoderConfig( + webrtc::kVideoCodecH265, + [cuda_context = cuda_context](auto format) + -> std::unique_ptr { + return NvCodecVideoEncoder::Create( + cuda_context, CudaVideoCodec::H265); + }, + 16)); } #endif @@ -254,75 +266,31 @@ SoraVideoEncoderFactoryConfig GetDefaultVideoEncoderFactoryConfig( } #endif -#if defined(USE_JETSON_ENCODER) - if (JetsonVideoEncoder::IsSupportedVP8()) { - config.encoders.insert(config.encoders.begin(), - VideoEncoderConfig( - webrtc::kVideoCodecVP8, - [](auto format) { - return std::unique_ptr( - absl::make_unique( - cricket::CreateVideoCodec(format))); - }, - 16)); - } - if (JetsonVideoEncoder::IsSupportedVP9()) { - config.encoders.insert(config.encoders.begin(), - VideoEncoderConfig( - webrtc::kVideoCodecVP9, - [](auto format) { - return std::unique_ptr( - absl::make_unique( - cricket::CreateVideoCodec(format))); - }, - 16)); - } - if (JetsonVideoEncoder::IsSupportedAV1()) { - config.encoders.insert(config.encoders.begin(), - VideoEncoderConfig( - webrtc::kVideoCodecAV1, - [](auto format) { - return std::unique_ptr( - absl::make_unique( - cricket::CreateVideoCodec(format))); - }, - 16)); - } - config.encoders.insert(config.encoders.begin(), - VideoEncoderConfig( - webrtc::kVideoCodecH264, - [](auto format) { - return std::unique_ptr( - absl::make_unique( - cricket::CreateVideoCodec(format))); - }, - 16)); -#endif - return config; } SoraVideoEncoderFactoryConfig GetSoftwareOnlyVideoEncoderFactoryConfig( std::optional openh264) { SoraVideoEncoderFactoryConfig config; - config.encoders.push_back(VideoEncoderConfig( - webrtc::kVideoCodecVP8, - [](auto format) { return webrtc::VP8Encoder::Create(); })); + config.encoders.push_back( + VideoEncoderConfig(webrtc::kVideoCodecVP8, [](auto format) { + return webrtc::CreateVp8Encoder(webrtc::CreateEnvironment()); + })); config.encoders.push_back( VideoEncoderConfig(webrtc::kVideoCodecVP9, [](auto format) { - return webrtc::VP9Encoder::Create(cricket::CreateVideoCodec(format)); + return webrtc::CreateVp9Encoder(webrtc::CreateEnvironment()); })); if (openh264) { config.encoders.push_back(VideoEncoderConfig( webrtc::kVideoCodecH264, [openh264 = *openh264](auto format) { - return CreateOpenH264VideoEncoder(cricket::CreateVideoCodec(format), - openh264); + return CreateOpenH264VideoEncoder(format, openh264); })); } #if !defined(__arm__) || defined(__aarch64__) || defined(__ARM_NEON__) - config.encoders.push_back(VideoEncoderConfig( - webrtc::kVideoCodecAV1, - [](auto format) { return webrtc::CreateLibaomAv1Encoder(); })); + config.encoders.push_back( + VideoEncoderConfig(webrtc::kVideoCodecAV1, [](auto format) { + return webrtc::CreateLibaomAv1Encoder(webrtc::CreateEnvironment()); + })); #endif return config; } diff --git a/src/version.cpp b/src/version.cpp index 18b135c1..cef3be3e 100644 --- a/src/version.cpp +++ b/src/version.cpp @@ -49,6 +49,8 @@ #endif +#define DEFAULT_USER_AGENT "Mozilla/5.0 (Sora C++ SDK/" SORA_CPP_SDK_VERSION ")" + namespace sora { std::string Version::GetClientName() { @@ -178,39 +180,6 @@ std::string Version::GetEnvironmentName() { break; } -#if defined(USE_JETSON_ENCODER) - // Jetson 系の場合、更に詳細な情報を取得する - - // nvidia-l4t-core のバージョンを拾う - // $ dpkg-query --show nvidia-l4t-core - // で取得できるが、外部コマンドは出来るだけ使いたくないので、 - // /var/lib/dpkg/status から該当行を探す - - std::string content; - { - std::stringstream ss; - std::ifstream fin("/var/lib/dpkg/status"); - ss << fin.rdbuf(); - content = ss.str(); - } - std::string l4t_core_version = "unknown"; - auto pos = content.find("Package: nvidia-l4t-core"); - if (pos != std::string::npos) { - const std::string VERSION = "Version: "; - auto pos2 = content.find(VERSION, pos); - if (pos2 != std::string::npos) { - pos2 += VERSION.size(); - auto pos3 = content.find("\n", pos2); - l4t_core_version = pos3 == std::string::npos - ? content.substr(pos2) - : content.substr(pos2, pos3 - pos2); - } - } - - info = " (nvidia-l4t-core " + l4t_core_version + ")"; - -#endif - #endif environment = "[" + arch + "] " + os + info; @@ -219,4 +188,8 @@ std::string Version::GetEnvironmentName() { return environment; } +http_header_value Version::GetDefaultUserAgent() { + return std::string(DEFAULT_USER_AGENT); +} + } // namespace sora diff --git a/src/websocket.cpp b/src/websocket.cpp index c6400337..5b60a404 100644 --- a/src/websocket.cpp +++ b/src/websocket.cpp @@ -14,6 +14,7 @@ #include #include "sora/ssl_verifier.h" +#include "sora/version.h" namespace sora { @@ -47,7 +48,8 @@ Websocket::Websocket(boost::asio::io_context& ioc) : ws_(new websocket_t(ioc)), resolver_(new boost::asio::ip::tcp::resolver(ioc)), strand_(ws_->get_executor()), - close_timeout_timer_(ioc) { + close_timeout_timer_(ioc), + user_agent_(Version::GetDefaultUserAgent()) { ws_->write_buffer_bytes(8192); } Websocket::Websocket(Websocket::ssl_tag, @@ -58,7 +60,8 @@ Websocket::Websocket(Websocket::ssl_tag, : resolver_(new boost::asio::ip::tcp::resolver(ioc)), strand_(ioc.get_executor()), close_timeout_timer_(ioc), - insecure_(insecure) { + insecure_(insecure), + user_agent_(Version::GetDefaultUserAgent()) { ssl_ctx_ = CreateSSLContext(client_cert, client_key); wss_.reset(new ssl_websocket_t(ioc, *ssl_ctx_)); InitWss(wss_.get(), insecure); @@ -66,7 +69,8 @@ Websocket::Websocket(Websocket::ssl_tag, Websocket::Websocket(boost::asio::ip::tcp::socket socket) : ws_(new websocket_t(std::move(socket))), strand_(ws_->get_executor()), - close_timeout_timer_(ws_->get_executor()) { + close_timeout_timer_(ws_->get_executor()), + user_agent_(Version::GetDefaultUserAgent()) { ws_->write_buffer_bytes(8192); } Websocket::Websocket(https_proxy_tag, @@ -84,7 +88,8 @@ Websocket::Websocket(https_proxy_tag, proxy_socket_(new boost::asio::ip::tcp::socket(ioc)), proxy_url_(std::move(proxy_url)), proxy_username_(std::move(proxy_username)), - proxy_password_(std::move(proxy_password)) { + proxy_password_(std::move(proxy_password)), + user_agent_(Version::GetDefaultUserAgent()) { ssl_ctx_ = CreateSSLContext(client_cert, client_key); } @@ -92,6 +97,10 @@ Websocket::~Websocket() { RTC_LOG(LS_INFO) << "Websocket::~Websocket this=" << (void*)this; } +void Websocket::SetUserAgent(http_header_value user_agent) { + user_agent_ = user_agent; +} + bool Websocket::IsSSL() const { return https_proxy_ || wss_ != nullptr; } @@ -161,6 +170,20 @@ void Websocket::Connect(const std::string& url, connect_callback_t on_connect) { } } + // ヘッダーの設定 + auto set_headers = [this](boost::beast::websocket::request_type& req) { + if (user_agent_ != boost::none) { + req.set(boost::beast::http::field::user_agent, *user_agent_); + } + }; + if (IsSSL()) { + wss_->set_option( + boost::beast::websocket::stream_base::decorator(set_headers)); + } else { + ws_->set_option( + boost::beast::websocket::stream_base::decorator(set_headers)); + } + on_connect_ = std::move(on_connect); // DNS ルックアップ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 808130ca..69f1f94c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -72,9 +72,6 @@ function (init_target target) ) elseif (APPLE) set_target_properties(${target} PROPERTIES CXX_VISIBILITY_PRESET hidden) - elseif (JETSON) - target_compile_definitions(${target} PRIVATE JETSON) - target_link_directories(${target} PRIVATE ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/tegra) endif() endfunction() diff --git a/test/android/app/build.gradle b/test/android/app/build.gradle index 55913fcf..cc159cf7 100644 --- a/test/android/app/build.gradle +++ b/test/android/app/build.gradle @@ -33,9 +33,11 @@ android { "-DWEBRTC_INCLUDE_DIR=$projectDir/../../../_install/android/release/webrtc/include", "-DWEBRTC_LIBRARY_DIR=$projectDir/../../../_install/android/release/webrtc/lib/arm64-v8a", "-DLIBCXX_INCLUDE_DIR=$projectDir/../../../_install/android/release/llvm/libcxx/include", - // "-DWEBRTC_INCLUDE_DIR=$projectDir/../../../_source/android/release/webrtc/src", - // "-DWEBRTC_LIBRARY_DIR=$projectDir/../../../_build/android/release/webrtc", - // "-DLIBCXX_INCLUDE_DIR=$projectDir/../../../_source/android/release/webrtc/src/buildtools/third_party/libc++/trunk/include", + "-DLLVM_DIR=$projectDir/../../../_install/android/release/llvm/clang", + // "-DWEBRTC_INCLUDE_DIR=$projectDir/../../../../webrtc-build/_source/android/webrtc/src", + // "-DWEBRTC_LIBRARY_DIR=$projectDir/../../../../webrtc-build/_build/android/release/webrtc/arm64-v8a", + // "-DLIBCXX_INCLUDE_DIR=$projectDir/../../../../webrtc-build/_source/android/webrtc/src/third_party/libc++/src/include", + // "-DLLVM_DIR=$projectDir/../../../../webrtc-build/_source/android/webrtc/src/third_party/llvm-build/Release+Asserts", "-DANDROID_STL=none", "-DANDROID_NATIVE_API_LEVEL=29", "-DANDROID_PLATFORM=29", @@ -63,7 +65,7 @@ android { externalNativeBuild { cmake { path file('src/main/cpp/CMakeLists.txt') - version '3.28.1' + version '3.29.6' } } buildFeatures { diff --git a/test/android/app/src/main/cpp/CMakeLists.txt b/test/android/app/src/main/cpp/CMakeLists.txt index f5ee5998..0c5ab49e 100644 --- a/test/android/app/src/main/cpp/CMakeLists.txt +++ b/test/android/app/src/main/cpp/CMakeLists.txt @@ -45,6 +45,16 @@ target_link_libraries(hello PRIVATE Sora::sora ${Blend2D_LIBRARY}) target_compile_definitions(hello PRIVATE BL_STATIC=1) target_include_directories(hello PRIVATE ${BLEND2D_ROOT_DIR}/include) +# libwebrtc が ELF フォーマットの新しい仕様を使ってリンクしているため、 +# Android NDK に用意されている標準のリンカーだと以下のようなリンクエラーになってしまう。 +# +# C/C++: ld.lld: error: /home/runner/work/sora-cpp-sdk/sora-cpp-sdk/_install/android/release/webrtc/lib/arm64-v8a/libwebrtc.a(jsep_ice_candidate.o):(.rodata+0x4): unknown relocation (315) against symbol typeinfo for webrtc::JsepIceCandidate +# +# なので libwebrtc と同じバージョンのリンカーを使ってリンクするために -B でリンカーのパスを指定する。 +# +# 新しい仕様の該当コミット: https://github.com/llvm/llvm-project/commit/04a906ec980e7bf49ffda0808766f51d08e8ae76 +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -B ${LLVM_DIR}/bin") + target_compile_options(hello PRIVATE "$<$:-nostdinc++>" diff --git a/test/android/app/src/main/cpp/jni_onload.cc b/test/android/app/src/main/cpp/jni_onload.cc index e6a88c43..c28813fc 100644 --- a/test/android/app/src/main/cpp/jni_onload.cc +++ b/test/android/app/src/main/cpp/jni_onload.cc @@ -12,7 +12,6 @@ #undef JNIEXPORT #define JNIEXPORT __attribute__((visibility("default"))) -#include "examples/unityplugin/class_reference_holder.h" #include "rtc_base/logging.h" #include "rtc_base/ssl_adapter.h" #include "sdk/android/native_api/jni/class_loader.h"