From 1812fad996bbdae70120924bb96c2780c606fbc7 Mon Sep 17 00:00:00 2001 From: melpon Date: Sat, 15 Jun 2024 01:13:52 +0900 Subject: [PATCH] =?UTF-8?q?Jetson=20=E3=82=92=20H265=20=E3=81=AB=E5=AF=BE?= =?UTF-8?q?=E5=BF=9C=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/c_cpp_properties.json | 39 ++++ CHANGES.md | 2 + CMakeLists.txt | 12 +- VERSION | 2 +- src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp | 29 --- src/main.cpp | 4 +- src/rtc/momo_video_decoder_factory.cpp | 32 ++- src/rtc/momo_video_encoder_factory.cpp | 68 ++++-- .../sora}/hwenc_jetson/jetson_buffer.h | 15 +- .../sora}/hwenc_jetson/jetson_jpeg_decoder.h | 20 +- .../hwenc_jetson/jetson_jpeg_decoder_pool.h | 20 +- .../sora}/hwenc_jetson/jetson_v4l2_capturer.h | 18 +- .../sora}/hwenc_jetson/jetson_video_decoder.h | 11 +- .../sora}/hwenc_jetson/jetson_video_encoder.h | 20 +- .../src}/hwenc_jetson/jetson_buffer.cpp | 24 ++- .../src}/hwenc_jetson/jetson_jpeg_decoder.cpp | 13 +- .../hwenc_jetson/jetson_jpeg_decoder_pool.cpp | 38 ++++ .../src/hwenc_jetson/jetson_util.h | 16 ++ .../hwenc_jetson/jetson_v4l2_capturer.cpp | 25 +-- .../hwenc_jetson/jetson_video_decoder.cpp | 47 ++--- .../hwenc_jetson/jetson_video_encoder.cpp | 193 +++++++++++++----- src/video_codec_info.h | 32 ++- 22 files changed, 462 insertions(+), 218 deletions(-) delete mode 100644 src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp rename src/{ => sora-cpp-sdk/include/sora}/hwenc_jetson/jetson_buffer.h (92%) rename src/{ => sora-cpp-sdk/include/sora}/hwenc_jetson/jetson_jpeg_decoder.h (64%) rename src/{ => sora-cpp-sdk/include/sora}/hwenc_jetson/jetson_jpeg_decoder_pool.h (50%) rename src/{ => sora-cpp-sdk/include/sora}/hwenc_jetson/jetson_v4l2_capturer.h (80%) rename src/{ => sora-cpp-sdk/include/sora}/hwenc_jetson/jetson_video_decoder.h (89%) rename src/{ => sora-cpp-sdk/include/sora}/hwenc_jetson/jetson_video_encoder.h (89%) rename src/{ => sora-cpp-sdk/src}/hwenc_jetson/jetson_buffer.cpp (92%) rename src/{ => sora-cpp-sdk/src}/hwenc_jetson/jetson_jpeg_decoder.cpp (75%) create mode 100644 src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp create mode 100644 src/sora-cpp-sdk/src/hwenc_jetson/jetson_util.h rename src/{ => sora-cpp-sdk/src}/hwenc_jetson/jetson_v4l2_capturer.cpp (96%) rename src/{ => sora-cpp-sdk/src}/hwenc_jetson/jetson_video_decoder.cpp (92%) rename src/{ => sora-cpp-sdk/src}/hwenc_jetson/jetson_video_encoder.cpp (83%) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 95463518..c3f1aa4e 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -71,6 +71,45 @@ "cStandard": "c11", "cppStandard": "c++17", "intelliSenseMode": "linux-clang-x64" + }, + { + "name": "Jetson", + "includePath": [ + "${workspaceFolder}/src", + "${workspaceFolder}/src/sora-cpp-sdk/include", + "${workspaceFolder}/third_party/NvCodec/include", + "${workspaceFolder}/third_party/NvCodec/NvCodec", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/abseil-cpp", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/boringssl/src/include", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/libyuv/include", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/zlib", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/boost/include", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/cli11/include", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/sdl2/include/SDL2", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/llvm/libcxx/include", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/rootfs/usr/src/jetson_multimedia_api/include", + "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/rootfs/usr/src/jetson_multimedia_api/include/libjpeg-8b", + "${workspaceFolder}/_build/ubuntu-22.04_armv8_jetson/release/momo" + ], + "defines": [ + "WEBRTC_POSIX", + "WEBRTC_LINUX", + "USE_JETSON_ENCODER", + "USE_SCREEN_CAPTURER", + "_LIBCPP_ABI_NAMESPACE=Cr", + "_LIBCPP_ABI_VERSION=2", + "_LIBCPP_DISABLE_AVAILABILITY", + "_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE", + "BOOST_NO_CXX98_FUNCTION_BASE", + "RTC_ENABLE_H265" + ], + // "compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/llvm/clang/bin/clang++", + "compilerPath": "/usr/bin/clang++-18", + "compilerArgs": ["-nostdinc++"], + "cStandard": "c11", + "cppStandard": "c++17", + "intelliSenseMode": "linux-clang-x64" } ], "version": 4 diff --git a/CHANGES.md b/CHANGES.md index 6ee41bdd..a59057f9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -52,6 +52,8 @@ - @melpon - [ADD] videoToolbox の H.265 ハードウェアエンコーダ/デコーダに対応する - @melpon +- [ADD] Jetson の H.265 ハードウェアエンコーダ/デコーダに対応する + - @melpon ## 2023.1.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 96c822cb..046a1746 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -509,12 +509,12 @@ elseif (TARGET_OS STREQUAL "linux") target_sources(momo PRIVATE - src/hwenc_jetson/jetson_buffer.cpp - src/hwenc_jetson/jetson_jpeg_decoder.cpp - src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp - src/hwenc_jetson/jetson_v4l2_capturer.cpp - src/hwenc_jetson/jetson_video_encoder.cpp - src/hwenc_jetson/jetson_video_decoder.cpp + src/sora-cpp-sdk/src/hwenc_jetson/jetson_buffer.cpp + src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder.cpp + src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp + src/sora-cpp-sdk/src/hwenc_jetson/jetson_v4l2_capturer.cpp + src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_encoder.cpp + src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_decoder.cpp ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvBufSurface.cpp ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvBuffer.cpp ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvElement.cpp diff --git a/VERSION b/VERSION index 9b6f5bbf..914ebef8 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ MOMO_VERSION=2023.1.0 -WEBRTC_BUILD_VERSION=m125.6422.2.1 +WEBRTC_BUILD_VERSION=m125.6422.2.2 BOOST_VERSION=1.85.0 CLI11_VERSION=v2.4.2 SDL2_VERSION=2.30.3 diff --git a/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp b/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp deleted file mode 100644 index 2050e3c8..00000000 --- a/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "jetson_jpeg_decoder_pool.h" - -// WebRTC -#include - -std::shared_ptr JetsonJpegDecoderPool::Pop() { - std::unique_ptr nv_decoder; - - // プールを使うとなぜか実行時にクラッシュすることがあるのでコメントアウト - // 多分 nvjpeg のバグ - { - //std::lock_guard lock(mtx_); - //if (decoder_queue_.size() == 0) { - nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec")); - //} else { - // nv_decoder = std::move(decoder_queue_.front()); - // decoder_queue_.pop(); - //} - } - - std::shared_ptr decoder( - new JetsonJpegDecoder(shared_from_this(), std::move(nv_decoder))); - return decoder; -} - -void JetsonJpegDecoderPool::Push(std::unique_ptr decoder) { - std::lock_guard lock(mtx_); - //decoder_queue_.push(std::move(decoder)); -} \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 63b2b0fa..f46d6767 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -18,7 +18,7 @@ #include "mac_helper/mac_capturer.h" #elif defined(__linux__) #if defined(USE_JETSON_ENCODER) -#include "hwenc_jetson/jetson_v4l2_capturer.h" +#include "sora/hwenc_jetson/jetson_v4l2_capturer.h" #elif defined(USE_NVCODEC_ENCODER) #include "sora/hwenc_nvcodec/nvcodec_v4l2_capturer.h" #elif defined(USE_V4L2_ENCODER) @@ -131,7 +131,7 @@ int main(int argc, char* argv[]) { #if defined(USE_JETSON_ENCODER) if (v4l2_config.use_native) { - return JetsonV4L2Capturer::Create(std::move(v4l2_config)); + return sora::JetsonV4L2Capturer::Create(std::move(v4l2_config)); } else { return sora::V4L2VideoCapturer::Create(std::move(v4l2_config)); } diff --git a/src/rtc/momo_video_decoder_factory.cpp b/src/rtc/momo_video_decoder_factory.cpp index c63d0cd0..f2456806 100644 --- a/src/rtc/momo_video_decoder_factory.cpp +++ b/src/rtc/momo_video_decoder_factory.cpp @@ -21,7 +21,7 @@ #endif #if defined(USE_JETSON_ENCODER) -#include "hwenc_jetson/jetson_video_decoder.h" +#include "sora/hwenc_jetson/jetson_video_decoder.h" #endif #if defined(USE_NVCODEC_ENCODER) @@ -119,7 +119,8 @@ MomoVideoDecoderFactory::GetSupportedFormats() const { } if (config_.h265_decoder == VideoCodecInfo::Type::Intel || - config_.h265_decoder == VideoCodecInfo::Type::NVIDIA) { + config_.h265_decoder == VideoCodecInfo::Type::NVIDIA || + config_.h265_decoder == VideoCodecInfo::Type::Jetson) { supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kH265CodecName)); } @@ -163,9 +164,9 @@ std::unique_ptr MomoVideoDecoderFactory::Create( #endif #if defined(USE_JETSON_ENCODER) if (config_.vp8_decoder == VideoCodecInfo::Type::Jetson && - JetsonVideoDecoder::IsSupportedVP8()) { + sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecVP8)) { return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecVP8)); + absl::make_unique(webrtc::kVideoCodecVP8)); } #endif @@ -189,9 +190,10 @@ std::unique_ptr MomoVideoDecoderFactory::Create( } #endif #if defined(USE_JETSON_ENCODER) - if (config_.vp9_decoder == VideoCodecInfo::Type::Jetson) { + if (config_.vp9_decoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecVP9)) { return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecVP9)); + absl::make_unique(webrtc::kVideoCodecVP9)); } #endif @@ -209,9 +211,9 @@ std::unique_ptr MomoVideoDecoderFactory::Create( #endif #if defined(USE_JETSON_ENCODER) if (config_.av1_decoder == VideoCodecInfo::Type::Jetson && - JetsonVideoDecoder::IsSupportedAV1()) { + sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecAV1)) { return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecAV1)); + absl::make_unique(webrtc::kVideoCodecAV1)); } #endif #if !defined(__arm__) || defined(__aarch64__) || defined(__ARM_NEON__) @@ -242,9 +244,10 @@ std::unique_ptr MomoVideoDecoderFactory::Create( } #endif #if defined(USE_JETSON_ENCODER) - if (config_.h264_decoder == VideoCodecInfo::Type::Jetson) { + if (config_.h264_decoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecH264)) { return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecH264)); + absl::make_unique(webrtc::kVideoCodecH264)); } #endif @@ -262,6 +265,15 @@ std::unique_ptr MomoVideoDecoderFactory::Create( return video_decoder_factory_->Create(env, format); } #endif + +#if defined(USE_JETSON_ENCODER) + if (config_.h265_decoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecH265)) { + return std::unique_ptr( + absl::make_unique(webrtc::kVideoCodecH265)); + } +#endif + #if defined(USE_NVCODEC_ENCODER) if (config_.h265_decoder == VideoCodecInfo::Type::NVIDIA) { return std::unique_ptr( diff --git a/src/rtc/momo_video_encoder_factory.cpp b/src/rtc/momo_video_encoder_factory.cpp index 4e840936..056eaa7a 100644 --- a/src/rtc/momo_video_encoder_factory.cpp +++ b/src/rtc/momo_video_encoder_factory.cpp @@ -27,7 +27,7 @@ #endif #if defined(USE_JETSON_ENCODER) -#include "hwenc_jetson/jetson_video_encoder.h" +#include "sora/hwenc_jetson/jetson_video_encoder.h" #endif #if defined(USE_NVCODEC_ENCODER) #include "sora/hwenc_nvcodec/nvcodec_video_encoder.h" @@ -63,7 +63,6 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { // VP8 if (config_.vp8_encoder == VideoCodecInfo::Type::Software || - config_.vp8_encoder == VideoCodecInfo::Type::Jetson || config_.vp8_encoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName)); } @@ -74,8 +73,7 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { webrtc::SupportedVP9Codecs(true)) { supported_codecs.push_back(format); } - } else if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson || - config_.vp9_encoder == VideoCodecInfo::Type::Intel) { + } else if (config_.vp9_encoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat( cricket::kVp9CodecName, {{webrtc::kVP9FmtpProfileId, @@ -85,7 +83,6 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { #if !defined(__arm__) || defined(__aarch64__) || defined(__ARM_NEON__) // AV1 if (config_.av1_encoder == VideoCodecInfo::Type::Software || - config_.av1_encoder == VideoCodecInfo::Type::Jetson || config_.av1_encoder == VideoCodecInfo::Type::Intel) { supported_codecs.push_back(webrtc::SdpVideoFormat( cricket::kAv1CodecName, webrtc::SdpVideoFormat::Parameters(), @@ -160,7 +157,7 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { } if (config_.h265_encoder == VideoCodecInfo::Type::VideoToolbox) { - // VideoToolbox の場合は video_encoder_factory_ から H264 を拾ってくる + // VideoToolbox の場合は video_encoder_factory_ から H265 を拾ってくる for (auto format : video_encoder_factory_->GetSupportedFormats()) { if (absl::EqualsIgnoreCase(format.name, cricket::kH265CodecName)) { supported_codecs.push_back(format); @@ -168,6 +165,36 @@ MomoVideoEncoderFactory::GetSupportedFormats() const { } } +#if defined(USE_JETSON_ENCODER) + if (config_.vp8_encoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP8)) { + supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName)); + } + if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP9)) { + for (const webrtc::SdpVideoFormat& format : + webrtc::SupportedVP9Codecs(true)) { + supported_codecs.push_back(format); + } + } + if (config_.av1_encoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecAV1)) { + supported_codecs.push_back(webrtc::SdpVideoFormat( + cricket::kAv1CodecName, webrtc::SdpVideoFormat::Parameters(), + webrtc::LibaomAv1EncoderSupportedScalabilityModes())); + } + if (config_.h264_encoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH264)) { + for (const webrtc::SdpVideoFormat& format : h264_codecs) { + supported_codecs.push_back(format); + } + } + if (config_.h265_encoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH265)) { + supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kH265CodecName)); + } +#endif + return supported_codecs; } @@ -220,10 +247,10 @@ std::unique_ptr MomoVideoEncoderFactory::Create( } #if defined(USE_JETSON_ENCODER) if (config_.vp8_encoder == VideoCodecInfo::Type::Jetson && - JetsonVideoEncoder::IsSupportedVP8()) { + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP8)) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique( + absl::make_unique( cricket::CreateVideoCodec(format))); }); } @@ -247,10 +274,10 @@ std::unique_ptr MomoVideoEncoderFactory::Create( } #if defined(USE_JETSON_ENCODER) if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson && - JetsonVideoEncoder::IsSupportedVP9()) { + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP9)) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique( + absl::make_unique( cricket::CreateVideoCodec(format))); }); } @@ -285,10 +312,10 @@ std::unique_ptr MomoVideoEncoderFactory::Create( #endif #if defined(USE_JETSON_ENCODER) if (config_.av1_encoder == VideoCodecInfo::Type::Jetson && - JetsonVideoEncoder::IsSupportedAV1()) { + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecAV1)) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique( + absl::make_unique( cricket::CreateVideoCodec(format))); }); } @@ -306,10 +333,11 @@ std::unique_ptr MomoVideoEncoderFactory::Create( #endif #if defined(USE_JETSON_ENCODER) - if (config_.h264_encoder == VideoCodecInfo::Type::Jetson) { + if (config_.h264_encoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH264)) { return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { return std::unique_ptr( - absl::make_unique( + absl::make_unique( cricket::CreateVideoCodec(format))); }); } @@ -357,6 +385,18 @@ std::unique_ptr MomoVideoEncoderFactory::Create( }); } #endif + +#if defined(USE_JETSON_ENCODER) + if (config_.h265_encoder == VideoCodecInfo::Type::Jetson && + sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH265)) { + return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + absl::make_unique( + cricket::CreateVideoCodec(format))); + }); + } +#endif + #if defined(USE_NVCODEC_ENCODER) if (config_.h265_encoder == VideoCodecInfo::Type::NVIDIA && sora::NvCodecVideoEncoder::IsSupported(config_.cuda_context, diff --git a/src/hwenc_jetson/jetson_buffer.h b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_buffer.h similarity index 92% rename from src/hwenc_jetson/jetson_buffer.h rename to src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_buffer.h index b2bc042f..3b41ecc8 100644 --- a/src/hwenc_jetson/jetson_buffer.h +++ b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_buffer.h @@ -1,5 +1,5 @@ -#ifndef JETSON_BUFFER_H_ -#define JETSON_BUFFER_H_ +#ifndef SORA_HWENC_JETSON_JETSON_BUFFER_H_ +#define SORA_HWENC_JETSON_JETSON_BUFFER_H_ #include #include @@ -10,12 +10,10 @@ #include #include -// Jetson Linux Multimedia API -#include -#include - #include "jetson_jpeg_decoder.h" +namespace sora { + class JetsonBuffer : public webrtc::VideoFrameBuffer { public: static rtc::scoped_refptr Create( @@ -77,4 +75,7 @@ class JetsonBuffer : public webrtc::VideoFrameBuffer { const std::unique_ptr data_; size_t length_; }; -#endif // JETSON_BUFFER_H_ + +} // namespace sora + +#endif diff --git a/src/hwenc_jetson/jetson_jpeg_decoder.h b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_jpeg_decoder.h similarity index 64% rename from src/hwenc_jetson/jetson_jpeg_decoder.h rename to src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_jpeg_decoder.h index a9e4af65..7c2c9702 100644 --- a/src/hwenc_jetson/jetson_jpeg_decoder.h +++ b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_jpeg_decoder.h @@ -1,19 +1,20 @@ -#ifndef JETSON_JPEG_DECODER_H_ -#define JETSON_JPEG_DECODER_H_ +#ifndef SORA_HWENC_JETSON_JETSON_JPEG_DECODER_H_ +#define SORA_HWENC_JETSON_JETSON_JPEG_DECODER_H_ #include -// Jetson Linux Multimedia API -#include - #include "jetson_jpeg_decoder_pool.h" +class NvJPEGDecoder; + +namespace sora { + class JetsonJpegDecoderPool; class JetsonJpegDecoder { public: JetsonJpegDecoder(std::shared_ptr pool, - std::unique_ptr decoder); + std::shared_ptr decoder); ~JetsonJpegDecoder(); int DecodeToFd(int& fd, @@ -25,6 +26,9 @@ class JetsonJpegDecoder { private: std::shared_ptr pool_; - std::unique_ptr decoder_; + std::shared_ptr decoder_; }; -#endif // JETSON_JPEG_DECODER_H_ \ No newline at end of file + +} // namespace sora + +#endif \ No newline at end of file diff --git a/src/hwenc_jetson/jetson_jpeg_decoder_pool.h b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_jpeg_decoder_pool.h similarity index 50% rename from src/hwenc_jetson/jetson_jpeg_decoder_pool.h rename to src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_jpeg_decoder_pool.h index 699502bf..328e6f1d 100644 --- a/src/hwenc_jetson/jetson_jpeg_decoder_pool.h +++ b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_jpeg_decoder_pool.h @@ -1,25 +1,29 @@ -#ifndef JETSON_JPEG_DECODER_POOL_H_ -#define JETSON_JPEG_DECODER_POOL_H_ +#ifndef SORA_HWENC_JETSON_JETSON_JPEG_DECODER_POOL_H_ +#define SORA_HWENC_JETSON_JETSON_JPEG_DECODER_POOL_H_ #include #include #include -// Jetson Linux Multimedia API -#include - #include "jetson_jpeg_decoder.h" +class NvJPEGDecoder; + +namespace sora { + class JetsonJpegDecoder; class JetsonJpegDecoderPool : public std::enable_shared_from_this { public: std::shared_ptr Pop(); - void Push(std::unique_ptr decoder); + void Push(std::shared_ptr decoder); private: std::mutex mtx_; - std::queue> decoder_queue_; + std::queue> decoder_queue_; }; -#endif // JETSON_JPEG_DECODER_POOL_H_ \ No newline at end of file + +} // namespace sora + +#endif diff --git a/src/hwenc_jetson/jetson_v4l2_capturer.h b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_v4l2_capturer.h similarity index 80% rename from src/hwenc_jetson/jetson_v4l2_capturer.h rename to src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_v4l2_capturer.h index e1637dc1..fc01594d 100644 --- a/src/hwenc_jetson/jetson_v4l2_capturer.h +++ b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_v4l2_capturer.h @@ -1,5 +1,5 @@ -#ifndef HWENC_JETSON_JETSON_V4L2_CAPTURER_H_ -#define HWENC_JETSON_JETSON_V4L2_CAPTURER_H_ +#ifndef SORA_HWENC_JETSON_JETSON_V4L2_CAPTURER_H_ +#define SORA_HWENC_JETSON_JETSON_V4L2_CAPTURER_H_ #include #include @@ -19,11 +19,13 @@ #include "sora/scalable_track_source.h" #include "sora/v4l2/v4l2_video_capturer.h" -class JetsonV4L2Capturer : public sora::ScalableVideoTrackSource { +namespace sora { + +class JetsonV4L2Capturer : public ScalableVideoTrackSource { public: static rtc::scoped_refptr Create( - const sora::V4L2VideoCapturerConfig& config); - JetsonV4L2Capturer(const sora::V4L2VideoCapturerConfig& config); + const V4L2VideoCapturerConfig& config); + JetsonV4L2Capturer(const V4L2VideoCapturerConfig& config); ~JetsonV4L2Capturer(); private: @@ -32,7 +34,7 @@ class JetsonV4L2Capturer : public sora::ScalableVideoTrackSource { int32_t Init(const char* deviceUniqueId, const std::string& specifiedVideoDevice); - int32_t StartCapture(const sora::V4L2VideoCapturerConfig& config); + int32_t StartCapture(const V4L2VideoCapturerConfig& config); int32_t StopCapture(); bool AllocateVideoBuffers(); @@ -55,7 +57,7 @@ class JetsonV4L2Capturer : public sora::ScalableVideoTrackSource { private: static rtc::scoped_refptr Create( webrtc::VideoCaptureModule::DeviceInfo* device_info, - const sora::V4L2VideoCapturerConfig& config, + const V4L2VideoCapturerConfig& config, size_t capture_device_index); bool FindDevice(const char* deviceUniqueIdUTF8, const std::string& device); @@ -76,4 +78,6 @@ class JetsonV4L2Capturer : public sora::ScalableVideoTrackSource { std::shared_ptr jpeg_decoder_pool_; }; +} // namespace sora + #endif diff --git a/src/hwenc_jetson/jetson_video_decoder.h b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_video_decoder.h similarity index 89% rename from src/hwenc_jetson/jetson_video_decoder.h rename to src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_video_decoder.h index c175bda1..e887b217 100644 --- a/src/hwenc_jetson/jetson_video_decoder.h +++ b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_video_decoder.h @@ -9,8 +9,8 @@ * */ -#ifndef HWENC_JETSON_JETSON_VIDEO_DECODER_H_ -#define HWENC_JETSON_JETSON_VIDEO_DECODER_H_ +#ifndef SORA_HWENC_JETSON_JETSON_VIDEO_DECODER_H_ +#define SORA_HWENC_JETSON_JETSON_VIDEO_DECODER_H_ // WebRTC #include @@ -21,13 +21,14 @@ struct v4l2_crop; class NvV4l2Element; class NvVideoDecoder; +namespace sora { + class JetsonVideoDecoder : public webrtc::VideoDecoder { public: JetsonVideoDecoder(webrtc::VideoCodecType codec); ~JetsonVideoDecoder() override; - static bool IsSupportedVP8(); - static bool IsSupportedAV1(); + static bool IsSupported(webrtc::VideoCodecType codec); bool Configure(const Settings& settings) override; @@ -61,4 +62,6 @@ class JetsonVideoDecoder : public webrtc::VideoDecoder { std::shared_ptr capture_crop_; }; +} // namespace sora + #endif diff --git a/src/hwenc_jetson/jetson_video_encoder.h b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_video_encoder.h similarity index 89% rename from src/hwenc_jetson/jetson_video_encoder.h rename to src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_video_encoder.h index d71913e2..1c732edb 100644 --- a/src/hwenc_jetson/jetson_video_encoder.h +++ b/src/sora-cpp-sdk/include/sora/hwenc_jetson/jetson_video_encoder.h @@ -9,8 +9,8 @@ * */ -#ifndef HWENC_JETSON_JETSON_VIDEO_ENCODER_H_ -#define HWENC_JETSON_JETSON_VIDEO_ENCODER_H_ +#ifndef SORA_HWENC_JETSON_JETSON_VIDEO_ENCODER_H_ +#define SORA_HWENC_JETSON_JETSON_VIDEO_ENCODER_H_ #include #include @@ -37,14 +37,14 @@ class NvV4l2Element; class NvVideoEncoder; struct v4l2_ctrl_videoenc_outputbuf_metadata_; +namespace sora { + class JetsonVideoEncoder : public webrtc::VideoEncoder { public: - explicit JetsonVideoEncoder(const cricket::VideoCodec& codec); + explicit JetsonVideoEncoder(const cricket::Codec& codec); ~JetsonVideoEncoder() override; - static bool IsSupportedVP8(); - static bool IsSupportedVP9(); - static bool IsSupportedAV1(); + static bool IsSupported(webrtc::VideoCodecType codec); int32_t InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, @@ -123,6 +123,7 @@ class JetsonVideoEncoder : public webrtc::VideoEncoder { int32_t height_; bool use_native_; bool use_dmabuff_; + int dmabuff_fd_[CONVERTER_CAPTURE_NUM]; webrtc::GofInfoVP9 gof_; size_t gof_idx_; @@ -131,8 +132,15 @@ class JetsonVideoEncoder : public webrtc::VideoEncoder { webrtc::Mutex frame_params_lock_; std::queue> frame_params_; + std::mutex enc0_buffer_mtx_; + std::condition_variable enc0_buffer_cond_; + std::queue* enc0_buffer_queue_; int output_plane_fd_[32]; webrtc::EncodedImage encoded_image_; + webrtc::ScalabilityMode scalability_mode_; + std::vector obu_seq_header_; }; +} // namespace sora + #endif diff --git a/src/hwenc_jetson/jetson_buffer.cpp b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_buffer.cpp similarity index 92% rename from src/hwenc_jetson/jetson_buffer.cpp rename to src/sora-cpp-sdk/src/hwenc_jetson/jetson_buffer.cpp index a1a92aae..d1e41417 100644 --- a/src/hwenc_jetson/jetson_buffer.cpp +++ b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_buffer.cpp @@ -1,4 +1,4 @@ -#include "jetson_buffer.h" +#include "sora/hwenc_jetson/jetson_buffer.h" // Linux #include @@ -8,6 +8,12 @@ #include #include +// Jetson Linux Multimedia API +#include +#include + +namespace sora { + static const int kBufferAlignment = 64; rtc::scoped_refptr JetsonBuffer::Create( @@ -67,12 +73,13 @@ rtc::scoped_refptr JetsonBuffer::ToI420() { NvBufSurface* dst_surf = 0; - if (NvBufSurfaceAllocate(&dst_surf, 1, &input_params) == -1) { + if (NvBufSurfaceAllocate( + &dst_surf, + 1, /* NvUtils では複数のバッファーを同時に初期化できるため、バッファーの数を指定する */ + &input_params) == -1) { RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufSurfaceAllocate"; - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufferCreateEx"; return scaled_buffer; } - NvBufSurfaceParams params = dst_surf->surfaceList[0]; NvBufSurfTransformRect src_rect, dest_rect; @@ -86,6 +93,7 @@ rtc::scoped_refptr JetsonBuffer::ToI420() { dest_rect.height = buffer_height; NvBufSurfTransformParams trans_params; + memset(&trans_params, 0, sizeof(trans_params)); trans_params.transform_flag = NVBUFSURF_TRANSFORM_FILTER; trans_params.transform_flip = NvBufSurfTransform_None; trans_params.transform_filter = NvBufSurfTransformInter_Algo3; @@ -130,9 +138,9 @@ rtc::scoped_refptr JetsonBuffer::ToI420() { } for (int i = 0; i < height; i++) { memcpy(dest_addr + width * i, - (uint8_t*)data_addr + - dst_surf->surfaceList->planeParams.pitch[plane] * i, - width); + (uint8_t*)data_addr + + dst_surf->surfaceList->planeParams.pitch[plane] * i, + width); } } NvBufSurfaceUnMap(dst_surf, index, plane); @@ -231,3 +239,5 @@ JetsonBuffer::JetsonBuffer(webrtc::VideoType video_type, data_(static_cast(webrtc::AlignedMalloc( webrtc::CalcBufferSize(video_type, raw_width, raw_height), kBufferAlignment))) {} + +} // namespace sora diff --git a/src/hwenc_jetson/jetson_jpeg_decoder.cpp b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder.cpp similarity index 75% rename from src/hwenc_jetson/jetson_jpeg_decoder.cpp rename to src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder.cpp index ea25f698..bfd048bd 100644 --- a/src/hwenc_jetson/jetson_jpeg_decoder.cpp +++ b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder.cpp @@ -1,8 +1,13 @@ -#include "jetson_jpeg_decoder.h" +#include "sora/hwenc_jetson/jetson_jpeg_decoder.h" + +// Jetson Linux Multimedia API +#include + +namespace sora { JetsonJpegDecoder::JetsonJpegDecoder( std::shared_ptr pool, - std::unique_ptr decoder) + std::shared_ptr decoder) : pool_(pool), decoder_(std::move(decoder)) {} JetsonJpegDecoder::~JetsonJpegDecoder() { @@ -16,4 +21,6 @@ int JetsonJpegDecoder::DecodeToFd(int& fd, uint32_t& width, uint32_t& height) { return decoder_->decodeToFd(fd, in_buf, in_buf_size, pixfmt, width, height); -} \ No newline at end of file +} + +} // namespace sora diff --git a/src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp new file mode 100644 index 00000000..f0b33176 --- /dev/null +++ b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp @@ -0,0 +1,38 @@ +#include "sora/hwenc_jetson/jetson_jpeg_decoder_pool.h" + +// WebRTC +#include + +// Jetson Linux Multimedia API +#include + +namespace sora { + +std::shared_ptr JetsonJpegDecoderPool::Pop() { + std::shared_ptr nv_decoder; + + // JetPack 5.1.2 で同じフレームが送信され続ける問題が発生したため、キューを無効化した + // JetPack 5.1.1 では正常に動作していた + // momo で同様の問題に対応した際の PR: https://github.com/shiguredo/momo/pull/297/ + // { + // std::lock_guard lock(mtx_); + // if (decoder_queue_.size() == 0) { + // nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec")); + // } else { + // nv_decoder = std::move(decoder_queue_.front()); + // decoder_queue_.pop(); + // } + // } + nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec")); + + std::shared_ptr decoder( + new JetsonJpegDecoder(shared_from_this(), std::move(nv_decoder))); + return decoder; +} + +void JetsonJpegDecoderPool::Push(std::shared_ptr decoder) { + std::lock_guard lock(mtx_); + // decoder_queue_.push(std::move(decoder)); +} + +} // namespace sora diff --git a/src/sora-cpp-sdk/src/hwenc_jetson/jetson_util.h b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_util.h new file mode 100644 index 00000000..cd4debd9 --- /dev/null +++ b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_util.h @@ -0,0 +1,16 @@ +#ifndef SORA_HWENC_JETSON_JETSON_UTIL_H_ +#define SORA_HWENC_JETSON_JETSON_UTIL_H_ + +namespace sora { + +static int VideoCodecToV4L2Format(webrtc::VideoCodecType codec) { + return codec == webrtc::kVideoCodecVP8 ? V4L2_PIX_FMT_VP8 + : codec == webrtc::kVideoCodecVP9 ? V4L2_PIX_FMT_VP9 + : codec == webrtc::kVideoCodecAV1 ? V4L2_PIX_FMT_AV1 + : codec == webrtc::kVideoCodecH264 ? V4L2_PIX_FMT_H264 + : codec == webrtc::kVideoCodecH265 ? V4L2_PIX_FMT_H265 + : 0; +} + +} // namespace sora +#endif diff --git a/src/hwenc_jetson/jetson_v4l2_capturer.cpp b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_v4l2_capturer.cpp similarity index 96% rename from src/hwenc_jetson/jetson_v4l2_capturer.cpp rename to src/sora-cpp-sdk/src/hwenc_jetson/jetson_v4l2_capturer.cpp index 845993a1..3aaf687d 100644 --- a/src/hwenc_jetson/jetson_v4l2_capturer.cpp +++ b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_v4l2_capturer.cpp @@ -1,4 +1,4 @@ -#include "jetson_v4l2_capturer.h" +#include "sora/hwenc_jetson/jetson_v4l2_capturer.h" // C #include @@ -31,12 +31,14 @@ // L4T Multimedia API #include -#include "jetson_buffer.h" +#include "sora/hwenc_jetson/jetson_buffer.h" #define MJPEG_EOS_SEARCH_SIZE 4096 +namespace sora { + rtc::scoped_refptr JetsonV4L2Capturer::Create( - const sora::V4L2VideoCapturerConfig& config) { + const V4L2VideoCapturerConfig& config) { rtc::scoped_refptr capturer; std::unique_ptr device_info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -77,7 +79,7 @@ void JetsonV4L2Capturer::LogDeviceList( rtc::scoped_refptr JetsonV4L2Capturer::Create( webrtc::VideoCaptureModule::DeviceInfo* device_info, - const sora::V4L2VideoCapturerConfig& config, + const V4L2VideoCapturerConfig& config, size_t capture_device_index) { char device_name[256]; char unique_name[256]; @@ -103,7 +105,7 @@ rtc::scoped_refptr JetsonV4L2Capturer::Create( return v4l2_capturer; } -JetsonV4L2Capturer::JetsonV4L2Capturer(const sora::V4L2VideoCapturerConfig& config) +JetsonV4L2Capturer::JetsonV4L2Capturer(const V4L2VideoCapturerConfig& config) : ScalableVideoTrackSource(config), _deviceFd(-1), _buffersAllocatedByDevice(-1), @@ -176,7 +178,7 @@ JetsonV4L2Capturer::~JetsonV4L2Capturer() { } int32_t JetsonV4L2Capturer::StartCapture( - const sora::V4L2VideoCapturerConfig& config) { + const V4L2VideoCapturerConfig& config) { if (_captureStarted) { if (config.width == _currentWidth && config.height == _currentHeight) { return 0; @@ -570,26 +572,17 @@ void JetsonV4L2Capturer::OnCaptured(v4l2_buffer* buf) { Search for EOF to get exact size */ if (eosSearchSize > bytesused) eosSearchSize = bytesused; - bool found = false; for (unsigned int i = 0; i < eosSearchSize; i++) { p = data + bytesused; if ((*(p - 2) == 0xff) && (*(p - 1) == 0xd9)) { - found = true; break; } bytesused--; } - if (!found) { - RTC_LOG(LS_WARNING) << __FUNCTION__ - << " Invalid JPEG buffer frame skipped"; - return; - } std::shared_ptr decoder = jpeg_decoder_pool_->Pop(); int fd = 0; uint32_t width, height, pixfmt; - RTC_LOG(LS_INFO) << "data: " << (void*)data << " (" << (int)data[0] << "," - << (int)data[1] << ") bytesused: " << bytesused; if (decoder->DecodeToFd(fd, data, bytesused, pixfmt, width, height) < 0) { RTC_LOG(LS_ERROR) << "decodeToFd Failed"; return; @@ -617,3 +610,5 @@ void JetsonV4L2Capturer::OnCaptured(v4l2_buffer* buf) { .build()); } } + +} // namespace sora \ No newline at end of file diff --git a/src/hwenc_jetson/jetson_video_decoder.cpp b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_decoder.cpp similarity index 92% rename from src/hwenc_jetson/jetson_video_decoder.cpp rename to src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_decoder.cpp index 45eb64b7..b10fe04c 100644 --- a/src/hwenc_jetson/jetson_video_decoder.cpp +++ b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_decoder.cpp @@ -9,7 +9,7 @@ * */ -#include "jetson_video_decoder.h" +#include "sora/hwenc_jetson/jetson_video_decoder.h" #include @@ -29,6 +29,8 @@ #include #include +#include "jetson_util.h" + #define INIT_ERROR(cond, desc) \ if (cond) { \ RTC_LOG(LS_ERROR) << __FUNCTION__ << desc; \ @@ -37,12 +39,10 @@ } #define CHUNK_SIZE 4000000 +namespace sora { + JetsonVideoDecoder::JetsonVideoDecoder(webrtc::VideoCodecType codec) - : input_format_(codec == webrtc::kVideoCodecVP8 ? V4L2_PIX_FMT_VP8 - : codec == webrtc::kVideoCodecVP9 ? V4L2_PIX_FMT_VP9 - : codec == webrtc::kVideoCodecH264 ? V4L2_PIX_FMT_H264 - : codec == webrtc::kVideoCodecAV1 ? V4L2_PIX_FMT_AV1 - : 0), + : input_format_(VideoCodecToV4L2Format(codec)), decoder_(nullptr), decode_complete_callback_(nullptr), buffer_pool_(false, 300 /* max_number_of_buffers*/), @@ -54,20 +54,10 @@ JetsonVideoDecoder::~JetsonVideoDecoder() { Release(); } -bool JetsonVideoDecoder::IsSupportedVP8() { - //SuppressErrors sup; - - auto decoder = NvVideoDecoder::createVideoDecoder("dec0"); - auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_VP8, CHUNK_SIZE); - delete decoder; - return ret >= 0; -} - -bool JetsonVideoDecoder::IsSupportedAV1() { - //SuppressErrors sup; - +bool JetsonVideoDecoder::IsSupported(webrtc::VideoCodecType codec) { auto decoder = NvVideoDecoder::createVideoDecoder("dec0"); - auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_AV1, CHUNK_SIZE); + auto ret = + decoder->setOutputPlaneFormat(VideoCodecToV4L2Format(codec), CHUNK_SIZE); delete decoder; return ret >= 0; } @@ -101,10 +91,10 @@ int32_t JetsonVideoDecoder::Decode(const webrtc::EncodedImage& input_image, memset(planes, 0, sizeof(planes)); v4l2_buf.m.planes = planes; - // RTC_LOG(LS_INFO) << __FUNCTION__ << " output_plane.getNumBuffers: " - // << decoder_->output_plane.getNumBuffers() - // << " output_plane.getNumQueuedBuffers: " - // << decoder_->output_plane.getNumQueuedBuffers(); + RTC_LOG(LS_INFO) << __FUNCTION__ << " output_plane.getNumBuffers: " + << decoder_->output_plane.getNumBuffers() + << " output_plane.getNumQueuedBuffers: " + << decoder_->output_plane.getNumQueuedBuffers(); if (decoder_->output_plane.getNumQueuedBuffers() == decoder_->output_plane.getNumBuffers()) { @@ -134,8 +124,9 @@ int32_t JetsonVideoDecoder::Decode(const webrtc::EncodedImage& input_image, return WEBRTC_VIDEO_CODEC_ERROR; } - // RTC_LOG(LS_INFO) << __FUNCTION__ << " timestamp:" << input_image.RtpTimestamp() - // << " bytesused:" << buffer->planes[0].bytesused; + RTC_LOG(LS_INFO) << __FUNCTION__ + << " timestamp:" << input_image.RtpTimestamp() + << " bytesused:" << buffer->planes[0].bytesused; return WEBRTC_VIDEO_CODEC_OK; } @@ -282,6 +273,7 @@ void JetsonVideoDecoder::CaptureLoop() { } NvBuffer* buffer; + while (1) { struct v4l2_buffer v4l2_buf; struct v4l2_plane planes[MAX_PLANES]; @@ -318,6 +310,7 @@ void JetsonVideoDecoder::CaptureLoop() { transform_params.flag = NVBUFSURF_TRANSFORM_FILTER; transform_params.flip = NvBufSurfTransform_None; transform_params.filter = NvBufSurfTransformInter_Algo3; + // 何が来ても YUV420 に変換する ret = NvBufSurf::NvTransform(&transform_params, buffer->planes[0].fd, dst_dma_fd_); @@ -353,8 +346,8 @@ void JetsonVideoDecoder::CaptureLoop() { } else { break; } - NvBufSurface* dst_surf = 0; + NvBufSurface* dst_surf = 0; if (NvBufSurfaceFromFd(dst_dma_fd_, (void**)(&dst_surf)) == -1) { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to NvBufSurfaceFromFd"; break; @@ -458,3 +451,5 @@ int JetsonVideoDecoder::SetCapture() { return WEBRTC_VIDEO_CODEC_OK; } + +} // namespace sora diff --git a/src/hwenc_jetson/jetson_video_encoder.cpp b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_encoder.cpp similarity index 83% rename from src/hwenc_jetson/jetson_video_encoder.cpp rename to src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_encoder.cpp index 4934fd75..72ab1dd4 100644 --- a/src/hwenc_jetson/jetson_video_encoder.cpp +++ b/src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_encoder.cpp @@ -9,13 +9,15 @@ * */ -#include "jetson_video_encoder.h" +#include "sora/hwenc_jetson/jetson_video_encoder.h" #include #include // WebRTC #include +#include +#include #include #include #include @@ -23,16 +25,19 @@ #include #include #include -#include -#include -#include + +// libyuv +#include +#include +#include // L4T Multimedia API #include #include #include -#include "jetson_buffer.h" +#include "jetson_util.h" +#include "sora/hwenc_jetson/jetson_buffer.h" #define H264HWENC_HEADER_DEBUG 0 #define INIT_ERROR(cond, desc) \ @@ -42,6 +47,36 @@ return WEBRTC_VIDEO_CODEC_ERROR; \ } +static std::string hex_dump(const uint8_t* buf, size_t len) { + std::stringstream ss; + + for (size_t i = 0; i < len; ++i) { + // 行の先頭にオフセットを表示 + if (i % 16 == 0) { + ss << std::setw(8) << std::setfill('0') << std::hex << i << ": "; + } + + // 値を16進数で表示 + ss << std::setw(2) << std::setfill('0') << std::hex << (int)buf[i] << " "; + + // 16バイトごとに改行 + if ((i + 1) % 16 == 0 || i == len - 1) { + ss << "\n"; + } + } + + return ss.str(); +} + +static void save_to_file(const std::string& filename, + const uint8_t* buf, + size_t size) { + std::ofstream file(filename, std::ios::binary); + file.write((const char*)buf, size); +} + +namespace sora { + JetsonVideoEncoder::JetsonVideoEncoder(const cricket::VideoCodec& codec) : callback_(nullptr), encoder_(nullptr), @@ -75,34 +110,12 @@ JetsonVideoEncoder::~JetsonVideoEncoder() { // int old_log_level; //}; -bool JetsonVideoEncoder::IsSupportedVP8() { - //SuppressErrors sup; - - auto encoder = NvVideoEncoder::createVideoEncoder("enc0"); - auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_VP8, 1024, 768, - 2 * 1024 * 1024); - delete encoder; - - return ret >= 0; -} - -bool JetsonVideoEncoder::IsSupportedVP9() { +bool JetsonVideoEncoder::IsSupported(webrtc::VideoCodecType codec) { //SuppressErrors sup; auto encoder = NvVideoEncoder::createVideoEncoder("enc0"); - auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_VP9, 1024, 768, - 2 * 1024 * 1024); - delete encoder; - - return ret >= 0; -} - -bool JetsonVideoEncoder::IsSupportedAV1() { - //SuppressErrors sup; - - auto encoder = NvVideoEncoder::createVideoEncoder("enc0"); - auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, 1024, 768, - 2 * 1024 * 1024); + auto ret = encoder->setCapturePlaneFormat(VideoCodecToV4L2Format(codec), 1024, + 768, 2 * 1024 * 1024); delete encoder; return ret >= 0; @@ -126,6 +139,9 @@ int32_t JetsonVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, target_bitrate_bps_ = codec_settings->startBitrate * 1000; if (codec_settings->codecType == webrtc::kVideoCodecH264) { key_frame_interval_ = codec_settings->H264().keyFrameInterval; + } else if (codec_settings->codecType == webrtc::kVideoCodecH265) { + // key_frame_interval_ = codec_settings->H265().keyFrameInterval; + key_frame_interval_ = 3000; } else if (codec_settings->codecType == webrtc::kVideoCodecVP8) { key_frame_interval_ = codec_settings->VP8().keyFrameInterval; } else if (codec_settings->codecType == webrtc::kVideoCodecVP9) { @@ -152,6 +168,12 @@ int32_t JetsonVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, RTC_LOG(LS_INFO) << "InitEncode scalability_mode:" << (int)*scalability_mode; svc_controller_ = webrtc::CreateScalabilityStructure(*scalability_mode); + scalability_mode_ = *scalability_mode; + + // codec_settings->AV1() にはキーフレームの設定が無いけれども、 + // key_frame_interval_ 自体は何らかの値で初期化しておかないと + // 不定値になってしまうので、適当な値を入れておく + key_frame_interval_ = 3000; } framerate_ = codec_settings->maxFramerate; @@ -187,19 +209,9 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { encoder_ = NvVideoEncoder::createVideoEncoder("enc0"); INIT_ERROR(!encoder_, "Failed to createVideoEncoder"); - if (codec_.codecType == webrtc::kVideoCodecH264) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, width_, height_, - 2 * 1024 * 1024); - } else if (codec_.codecType == webrtc::kVideoCodecVP8) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_VP8, width_, height_, - 2 * 1024 * 1024); - } else if (codec_.codecType == webrtc::kVideoCodecVP9) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_VP9, width_, height_, - 2 * 1024 * 1024); - } else if (codec_.codecType == webrtc::kVideoCodecAV1) { - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, width_, height_, - 2 * 1024 * 1024); - } + ret = + encoder_->setCapturePlaneFormat(VideoCodecToV4L2Format(codec_.codecType), + width_, height_, 2 * 1024 * 1024); INIT_ERROR(ret < 0, "Failed to encoder setCapturePlaneFormat"); ret = encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, width_, height_); @@ -224,6 +236,25 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { // V4L2_ENC_HW_PRESET_ULTRAFAST が推奨値だけど MEDIUM でも Nano, AGX では OK // NX は V4L2_ENC_HW_PRESET_FAST でないとフレームレートがでない + ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_FAST); + INIT_ERROR(ret < 0, "Failed to setHWPresetType"); + } else if (codec_.codecType == webrtc::kVideoCodecH265) { + // H264 の設定を真似する + ret = encoder_->setProfile(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN); + INIT_ERROR(ret < 0, "Failed to setProfile"); + + ret = encoder_->setLevel(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1); + INIT_ERROR(ret < 0, "Failed to setLevel"); + + ret = encoder_->setNumBFrames(0); + INIT_ERROR(ret < 0, "Failed to setNumBFrames"); + + ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); + INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); + + ret = encoder_->setInsertVuiEnabled(true); + INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); + ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_FAST); INIT_ERROR(ret < 0, "Failed to setHWPresetType"); } else if (codec_.codecType == webrtc::kVideoCodecVP8) { @@ -515,6 +546,11 @@ webrtc::VideoEncoder::EncoderInfo JetsonVideoEncoder::GetEncoderInfo() const { static const int kHighH264QpThreshold = 40; info.scaling_settings = VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); + } else if (codec_.codecType == webrtc::kVideoCodecH265) { + static const int kLowH265QpThreshold = 34; + static const int kHighH265QpThreshold = 40; + info.scaling_settings = VideoEncoder::ScalingSettings(kLowH265QpThreshold, + kHighH265QpThreshold); } else if (codec_.codecType == webrtc::kVideoCodecVP8) { static const int kLowVp8QpThreshold = 29; static const int kHighVp8QpThreshold = 95; @@ -758,6 +794,10 @@ int32_t JetsonVideoEncoder::Encode( RTC_LOG(LS_ERROR) << "Failed to NvBufSurfaceSyncForDevice"; return WEBRTC_VIDEO_CODEC_ERROR; } + + // ここで NvBufSurfaceDestroy が必要かなと思ったが、以下のサンプル・コードを確認したところ不要そうだった + // 参照: jetson_multimedia_api/samples/01_video_encode/video_encode_main.cpp + // NvBufSurfaceDestroy(surf); } if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { @@ -790,9 +830,9 @@ int32_t JetsonVideoEncoder::SendFrame( encoded_image_.rotation_ = params->rotation; encoded_image_.qp_ = enc_metadata->AvgQP; if (enc_metadata->KeyFrame) { - encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey; + encoded_image_.SetFrameType(webrtc::VideoFrameType::kVideoFrameKey); } else { - encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameDelta; + encoded_image_.SetFrameType(webrtc::VideoFrameType::kVideoFrameDelta); } webrtc::CodecSpecificInfo codec_specific; @@ -804,6 +844,13 @@ int32_t JetsonVideoEncoder::SendFrame( codec_specific.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; + } else if (codec_.codecType == webrtc::kVideoCodecH265) { + auto encoded_image_buffer = + webrtc::EncodedImageBuffer::Create(buffer, size); + encoded_image_.SetEncodedData(encoded_image_buffer); + + codec_specific.codecSpecific.H265.packetization_mode = + webrtc::H265PacketizationMode::NonInterleaved; } else if (codec_.codecType == webrtc::kVideoCodecAV1 || codec_.codecType == webrtc::kVideoCodecVP9 || codec_.codecType == webrtc::kVideoCodecVP8) { @@ -816,8 +863,47 @@ int32_t JetsonVideoEncoder::SendFrame( buffer += 12; size -= 12; - auto encoded_image_buffer = - webrtc::EncodedImageBuffer::Create(buffer, size); + rtc::scoped_refptr encoded_image_buffer; + + if (codec_.codecType == webrtc::kVideoCodecAV1) { + // JetPack 5.1.1 以降、AV1 のエンコードフレームにシーケンスヘッダー（OBU_SEQUENCE_HEADER）が含まれなくなってしまった。 + // キーフレームには必ずシーケンスヘッダーを設定しておかないと、途中から受信したデコーダーでデコードができなくなってしまう。 + // そのため、初回のシーケンスヘッダーを保存しておき、キーフレームの前に挿入することで対応する。 + + // buffer の最初の 2 バイトは 0x12 0x00 で、これは OBU_TEMPORAL_DELIMITER なのでこれは無視して、その次の OBU を見る + if (buffer[2] == 0x0a) { + // 0x0a は OBU_SEQUENCE_HEADER なので、この OBU を保存しておく + // obu_size は本当は LEB128 だけど、128 バイト以上になることはなさそうなので、そのまま 1 バイトで取得する + int obu_size = buffer[3]; + obu_seq_header_.resize(obu_size + 2); + memcpy(obu_seq_header_.data(), buffer + 2, obu_size + 2); + } + + // キーフレームとしてマークされているのにシーケンスヘッダーが入っていない場合、 + // フレームの前にシーケンスヘッダーを入れる + if (enc_metadata->KeyFrame && buffer[2] != 0x0a) { + std::vector new_buffer; + new_buffer.resize(obu_seq_header_.size() + size); + uint8_t* p = new_buffer.data(); + // OBU_TEMPORAL_DELIMITER + memcpy(p, buffer, 2); + p += 2; + // OBU_SEQUENCE_HEADER + memcpy(p, obu_seq_header_.data(), obu_seq_header_.size()); + p += obu_seq_header_.size(); + // OBU_FRAME + memcpy(p, buffer + 2, size - 2); + + // RTC_LOG(LS_ERROR) << "\n" << hex_dump(new_buffer.data(), 64); + // save_to_file("keyframe2.obu", new_buffer.data(), new_buffer.size()); + encoded_image_buffer = webrtc::EncodedImageBuffer::Create( + new_buffer.data(), new_buffer.size()); + } else { + encoded_image_buffer = webrtc::EncodedImageBuffer::Create(buffer, size); + } + } else { + encoded_image_buffer = webrtc::EncodedImageBuffer::Create(buffer, size); + } encoded_image_.SetEncodedData(encoded_image_buffer); if (codec_.codecType == webrtc::kVideoCodecVP8) { @@ -851,17 +937,12 @@ int32_t JetsonVideoEncoder::SendFrame( codec_specific.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); } } else if (codec_.codecType == webrtc::kVideoCodecAV1) { - bool is_key = buffer[2] == 0x0a; - // v4l2_ctrl_videoenc_outputbuf_metadata.KeyFrame が効いていない - // キーフレームの時には OBU_SEQUENCE_HEADER が入っているために 0x0a になるためこれを使う - // キーフレームではない時には OBU_FRAME が入っていて 0x32 になっている - if (is_key) { - encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey; - } - + bool is_key = + encoded_image_._frameType == webrtc::VideoFrameType::kVideoFrameKey; std::vector layer_frames = svc_controller_->NextFrameConfig(is_key); codec_specific.end_of_picture = true; + codec_specific.scalability_mode = scalability_mode_; codec_specific.generic_frame_info = svc_controller_->OnEncodeDone(layer_frames[0]); if (is_key && codec_specific.generic_frame_info) { @@ -887,3 +968,5 @@ int32_t JetsonVideoEncoder::SendFrame( bitrate_adjuster_->Update(size); return WEBRTC_VIDEO_CODEC_OK; } + +} // namespace sora diff --git a/src/video_codec_info.h b/src/video_codec_info.h index 3c115721..30ed3b95 100644 --- a/src/video_codec_info.h +++ b/src/video_codec_info.h @@ -17,8 +17,8 @@ #endif #if defined(USE_JETSON_ENCODER) -#include "hwenc_jetson/jetson_video_decoder.h" -#include "hwenc_jetson/jetson_video_encoder.h" +#include "sora/hwenc_jetson/jetson_video_decoder.h" +#include "sora/hwenc_jetson/jetson_video_encoder.h" #endif struct VideoCodecInfo { @@ -277,22 +277,34 @@ struct VideoCodecInfo { #endif #if defined(USE_JETSON_ENCODER) - info.h264_encoders.push_back(Type::Jetson); - info.h264_decoders.push_back(Type::Jetson); - if (JetsonVideoEncoder::IsSupportedVP8()) { + if (sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH264)) { + info.h264_encoders.push_back(Type::Jetson); + } + if (sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecH264)) { + info.h264_decoders.push_back(Type::Jetson); + } + if (sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH265)) { + info.h265_encoders.push_back(Type::Jetson); + } + if (sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecH265)) { + info.h265_decoders.push_back(Type::Jetson); + } + if (sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP8)) { info.vp8_encoders.push_back(Type::Jetson); } - if (JetsonVideoDecoder::IsSupportedVP8()) { + if (sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecVP8)) { info.vp8_decoders.push_back(Type::Jetson); } - if (JetsonVideoEncoder::IsSupportedVP9()) { + if (sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP9)) { info.vp9_encoders.push_back(Type::Jetson); } - info.vp9_decoders.push_back(Type::Jetson); - if (JetsonVideoEncoder::IsSupportedAV1()) { + if (sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecVP9)) { + info.vp9_decoders.push_back(Type::Jetson); + } + if (sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecAV1)) { info.av1_encoders.push_back(Type::Jetson); } - if (JetsonVideoDecoder::IsSupportedAV1()) { + if (sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecAV1)) { info.av1_decoders.push_back(Type::Jetson); } #endif