Jetson を H265 に対応する

shiguredo · Jun 14, 2024 · 1812fad · 1812fad
1 parent 4a1655f
commit 1812fad
Show file tree

Hide file tree

Showing 22 changed files with 462 additions and 218 deletions.
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json
@@ -71,6 +71,45 @@
             "cStandard": "c11",
             "cppStandard": "c++17",
             "intelliSenseMode": "linux-clang-x64"
+        },
+        {
+            "name": "Jetson",
+            "includePath": [
+                "${workspaceFolder}/src",
+                "${workspaceFolder}/src/sora-cpp-sdk/include",
+                "${workspaceFolder}/third_party/NvCodec/include",
+                "${workspaceFolder}/third_party/NvCodec/NvCodec",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/abseil-cpp",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/boringssl/src/include",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/libyuv/include",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/webrtc/include/third_party/zlib",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/boost/include",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/cli11/include",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/sdl2/include/SDL2",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/llvm/libcxx/include",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/rootfs/usr/src/jetson_multimedia_api/include",
+                "${workspaceFolder}/_install/ubuntu-22.04_armv8_jetson/release/rootfs/usr/src/jetson_multimedia_api/include/libjpeg-8b",
+                "${workspaceFolder}/_build/ubuntu-22.04_armv8_jetson/release/momo"
+            ],
+            "defines": [
+                "WEBRTC_POSIX",
+                "WEBRTC_LINUX",
+                "USE_JETSON_ENCODER",
+                "USE_SCREEN_CAPTURER",
+                "_LIBCPP_ABI_NAMESPACE=Cr",
+                "_LIBCPP_ABI_VERSION=2",
+                "_LIBCPP_DISABLE_AVAILABILITY",
+                "_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE",
+                "BOOST_NO_CXX98_FUNCTION_BASE",
+                "RTC_ENABLE_H265"
+            ],
+            // "compilerPath": "${workspaceFolder}/_install/ubuntu-20.04_x86_64/release/llvm/clang/bin/clang++",
+            "compilerPath": "/usr/bin/clang++-18",
+            "compilerArgs": ["-nostdinc++"],
+            "cStandard": "c11",
+            "cppStandard": "c++17",
+            "intelliSenseMode": "linux-clang-x64"
         }
     ],
     "version": 4

diff --git a/CHANGES.md b/CHANGES.md
@@ -52,6 +52,8 @@
   - @melpon
 - [ADD] videoToolbox の H.265 ハードウェアエンコーダ/デコーダに対応する
   - @melpon
+- [ADD] Jetson の H.265 ハードウェアエンコーダ/デコーダに対応する
+  - @melpon
 
 ## 2023.1.0
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -509,12 +509,12 @@ elseif (TARGET_OS STREQUAL "linux")
 
         target_sources(momo
           PRIVATE
-            src/hwenc_jetson/jetson_buffer.cpp
-            src/hwenc_jetson/jetson_jpeg_decoder.cpp
-            src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp
-            src/hwenc_jetson/jetson_v4l2_capturer.cpp
-            src/hwenc_jetson/jetson_video_encoder.cpp
-            src/hwenc_jetson/jetson_video_decoder.cpp
+            src/sora-cpp-sdk/src/hwenc_jetson/jetson_buffer.cpp
+            src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder.cpp
+            src/sora-cpp-sdk/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp
+            src/sora-cpp-sdk/src/hwenc_jetson/jetson_v4l2_capturer.cpp
+            src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_encoder.cpp
+            src/sora-cpp-sdk/src/hwenc_jetson/jetson_video_decoder.cpp
             ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvBufSurface.cpp
             ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvBuffer.cpp
             ${CMAKE_SYSROOT}/usr/src/jetson_multimedia_api/samples/common/classes/NvElement.cpp

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 MOMO_VERSION=2023.1.0
-WEBRTC_BUILD_VERSION=m125.6422.2.1
+WEBRTC_BUILD_VERSION=m125.6422.2.2
 BOOST_VERSION=1.85.0
 CLI11_VERSION=v2.4.2
 SDL2_VERSION=2.30.3

diff --git a/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp b/src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp
diff --git a/src/main.cpp b/src/main.cpp
@@ -18,7 +18,7 @@
 #include "mac_helper/mac_capturer.h"
 #elif defined(__linux__)
 #if defined(USE_JETSON_ENCODER)
-#include "hwenc_jetson/jetson_v4l2_capturer.h"
+#include "sora/hwenc_jetson/jetson_v4l2_capturer.h"
 #elif defined(USE_NVCODEC_ENCODER)
 #include "sora/hwenc_nvcodec/nvcodec_v4l2_capturer.h"
 #elif defined(USE_V4L2_ENCODER)
@@ -131,7 +131,7 @@ int main(int argc, char* argv[]) {
 
 #if defined(USE_JETSON_ENCODER)
     if (v4l2_config.use_native) {
-      return JetsonV4L2Capturer::Create(std::move(v4l2_config));
+      return sora::JetsonV4L2Capturer::Create(std::move(v4l2_config));
     } else {
       return sora::V4L2VideoCapturer::Create(std::move(v4l2_config));
     }

diff --git a/src/rtc/momo_video_decoder_factory.cpp b/src/rtc/momo_video_decoder_factory.cpp
@@ -21,7 +21,7 @@
 #endif
 
 #if defined(USE_JETSON_ENCODER)
-#include "hwenc_jetson/jetson_video_decoder.h"
+#include "sora/hwenc_jetson/jetson_video_decoder.h"
 #endif
 
 #if defined(USE_NVCODEC_ENCODER)
@@ -119,7 +119,8 @@ MomoVideoDecoderFactory::GetSupportedFormats() const {
   }
 
   if (config_.h265_decoder == VideoCodecInfo::Type::Intel ||
-      config_.h265_decoder == VideoCodecInfo::Type::NVIDIA) {
+      config_.h265_decoder == VideoCodecInfo::Type::NVIDIA ||
+      config_.h265_decoder == VideoCodecInfo::Type::Jetson) {
     supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kH265CodecName));
   }
 
@@ -163,9 +164,9 @@ std::unique_ptr<webrtc::VideoDecoder> MomoVideoDecoderFactory::Create(
 #endif
 #if defined(USE_JETSON_ENCODER)
     if (config_.vp8_decoder == VideoCodecInfo::Type::Jetson &&
-        JetsonVideoDecoder::IsSupportedVP8()) {
+        sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecVP8)) {
       return std::unique_ptr<webrtc::VideoDecoder>(
-          absl::make_unique<JetsonVideoDecoder>(webrtc::kVideoCodecVP8));
+          absl::make_unique<sora::JetsonVideoDecoder>(webrtc::kVideoCodecVP8));
     }
 #endif
 
@@ -189,9 +190,10 @@ std::unique_ptr<webrtc::VideoDecoder> MomoVideoDecoderFactory::Create(
     }
 #endif
 #if defined(USE_JETSON_ENCODER)
-    if (config_.vp9_decoder == VideoCodecInfo::Type::Jetson) {
+    if (config_.vp9_decoder == VideoCodecInfo::Type::Jetson &&
+        sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecVP9)) {
       return std::unique_ptr<webrtc::VideoDecoder>(
-          absl::make_unique<JetsonVideoDecoder>(webrtc::kVideoCodecVP9));
+          absl::make_unique<sora::JetsonVideoDecoder>(webrtc::kVideoCodecVP9));
     }
 #endif
 
@@ -209,9 +211,9 @@ std::unique_ptr<webrtc::VideoDecoder> MomoVideoDecoderFactory::Create(
 #endif
 #if defined(USE_JETSON_ENCODER)
     if (config_.av1_decoder == VideoCodecInfo::Type::Jetson &&
-        JetsonVideoDecoder::IsSupportedAV1()) {
+        sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecAV1)) {
       return std::unique_ptr<webrtc::VideoDecoder>(
-          absl::make_unique<JetsonVideoDecoder>(webrtc::kVideoCodecAV1));
+          absl::make_unique<sora::JetsonVideoDecoder>(webrtc::kVideoCodecAV1));
     }
 #endif
 #if !defined(__arm__) || defined(__aarch64__) || defined(__ARM_NEON__)
@@ -242,9 +244,10 @@ std::unique_ptr<webrtc::VideoDecoder> MomoVideoDecoderFactory::Create(
     }
 #endif
 #if defined(USE_JETSON_ENCODER)
-    if (config_.h264_decoder == VideoCodecInfo::Type::Jetson) {
+    if (config_.h264_decoder == VideoCodecInfo::Type::Jetson &&
+        sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecH264)) {
       return std::unique_ptr<webrtc::VideoDecoder>(
-          absl::make_unique<JetsonVideoDecoder>(webrtc::kVideoCodecH264));
+          absl::make_unique<sora::JetsonVideoDecoder>(webrtc::kVideoCodecH264));
     }
 #endif
 
@@ -262,6 +265,15 @@ std::unique_ptr<webrtc::VideoDecoder> MomoVideoDecoderFactory::Create(
       return video_decoder_factory_->Create(env, format);
     }
 #endif
+
+#if defined(USE_JETSON_ENCODER)
+    if (config_.h265_decoder == VideoCodecInfo::Type::Jetson &&
+        sora::JetsonVideoDecoder::IsSupported(webrtc::kVideoCodecH265)) {
+      return std::unique_ptr<webrtc::VideoDecoder>(
+          absl::make_unique<sora::JetsonVideoDecoder>(webrtc::kVideoCodecH265));
+    }
+#endif
+
 #if defined(USE_NVCODEC_ENCODER)
     if (config_.h265_decoder == VideoCodecInfo::Type::NVIDIA) {
       return std::unique_ptr<webrtc::VideoDecoder>(

diff --git a/src/rtc/momo_video_encoder_factory.cpp b/src/rtc/momo_video_encoder_factory.cpp
@@ -27,7 +27,7 @@
 #endif
 
 #if defined(USE_JETSON_ENCODER)
-#include "hwenc_jetson/jetson_video_encoder.h"
+#include "sora/hwenc_jetson/jetson_video_encoder.h"
 #endif
 #if defined(USE_NVCODEC_ENCODER)
 #include "sora/hwenc_nvcodec/nvcodec_video_encoder.h"
@@ -63,7 +63,6 @@ MomoVideoEncoderFactory::GetSupportedFormats() const {
 
   // VP8
   if (config_.vp8_encoder == VideoCodecInfo::Type::Software ||
-      config_.vp8_encoder == VideoCodecInfo::Type::Jetson ||
       config_.vp8_encoder == VideoCodecInfo::Type::Intel) {
     supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
   }
@@ -74,8 +73,7 @@ MomoVideoEncoderFactory::GetSupportedFormats() const {
          webrtc::SupportedVP9Codecs(true)) {
       supported_codecs.push_back(format);
     }
-  } else if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson ||
-             config_.vp9_encoder == VideoCodecInfo::Type::Intel) {
+  } else if (config_.vp9_encoder == VideoCodecInfo::Type::Intel) {
     supported_codecs.push_back(webrtc::SdpVideoFormat(
         cricket::kVp9CodecName,
         {{webrtc::kVP9FmtpProfileId,
@@ -85,7 +83,6 @@ MomoVideoEncoderFactory::GetSupportedFormats() const {
 #if !defined(__arm__) || defined(__aarch64__) || defined(__ARM_NEON__)
   // AV1
   if (config_.av1_encoder == VideoCodecInfo::Type::Software ||
-      config_.av1_encoder == VideoCodecInfo::Type::Jetson ||
       config_.av1_encoder == VideoCodecInfo::Type::Intel) {
     supported_codecs.push_back(webrtc::SdpVideoFormat(
         cricket::kAv1CodecName, webrtc::SdpVideoFormat::Parameters(),
@@ -160,14 +157,44 @@ MomoVideoEncoderFactory::GetSupportedFormats() const {
   }
 
   if (config_.h265_encoder == VideoCodecInfo::Type::VideoToolbox) {
-    // VideoToolbox の場合は video_encoder_factory_ から H264 を拾ってくる
+    // VideoToolbox の場合は video_encoder_factory_ から H265 を拾ってくる
     for (auto format : video_encoder_factory_->GetSupportedFormats()) {
       if (absl::EqualsIgnoreCase(format.name, cricket::kH265CodecName)) {
         supported_codecs.push_back(format);
       }
     }
   }
 
+#if defined(USE_JETSON_ENCODER)
+  if (config_.vp8_encoder == VideoCodecInfo::Type::Jetson &&
+      sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP8)) {
+    supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
+  }
+  if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson &&
+      sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP9)) {
+    for (const webrtc::SdpVideoFormat& format :
+         webrtc::SupportedVP9Codecs(true)) {
+      supported_codecs.push_back(format);
+    }
+  }
+  if (config_.av1_encoder == VideoCodecInfo::Type::Jetson &&
+      sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecAV1)) {
+    supported_codecs.push_back(webrtc::SdpVideoFormat(
+        cricket::kAv1CodecName, webrtc::SdpVideoFormat::Parameters(),
+        webrtc::LibaomAv1EncoderSupportedScalabilityModes()));
+  }
+  if (config_.h264_encoder == VideoCodecInfo::Type::Jetson &&
+      sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH264)) {
+    for (const webrtc::SdpVideoFormat& format : h264_codecs) {
+      supported_codecs.push_back(format);
+    }
+  }
+  if (config_.h265_encoder == VideoCodecInfo::Type::Jetson &&
+      sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH265)) {
+    supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kH265CodecName));
+  }
+#endif
+
   return supported_codecs;
 }
 
@@ -220,10 +247,10 @@ std::unique_ptr<webrtc::VideoEncoder> MomoVideoEncoderFactory::Create(
     }
 #if defined(USE_JETSON_ENCODER)
     if (config_.vp8_encoder == VideoCodecInfo::Type::Jetson &&
-        JetsonVideoEncoder::IsSupportedVP8()) {
+        sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP8)) {
       return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) {
         return std::unique_ptr<webrtc::VideoEncoder>(
-            absl::make_unique<JetsonVideoEncoder>(
+            absl::make_unique<sora::JetsonVideoEncoder>(
                 cricket::CreateVideoCodec(format)));
       });
     }
@@ -247,10 +274,10 @@ std::unique_ptr<webrtc::VideoEncoder> MomoVideoEncoderFactory::Create(
     }
 #if defined(USE_JETSON_ENCODER)
     if (config_.vp9_encoder == VideoCodecInfo::Type::Jetson &&
-        JetsonVideoEncoder::IsSupportedVP9()) {
+        sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecVP9)) {
       return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) {
         return std::unique_ptr<webrtc::VideoEncoder>(
-            absl::make_unique<JetsonVideoEncoder>(
+            absl::make_unique<sora::JetsonVideoEncoder>(
                 cricket::CreateVideoCodec(format)));
       });
     }
@@ -285,10 +312,10 @@ std::unique_ptr<webrtc::VideoEncoder> MomoVideoEncoderFactory::Create(
 #endif
 #if defined(USE_JETSON_ENCODER)
     if (config_.av1_encoder == VideoCodecInfo::Type::Jetson &&
-        JetsonVideoEncoder::IsSupportedAV1()) {
+        sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecAV1)) {
       return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) {
         return std::unique_ptr<webrtc::VideoEncoder>(
-            absl::make_unique<JetsonVideoEncoder>(
+            absl::make_unique<sora::JetsonVideoEncoder>(
                 cricket::CreateVideoCodec(format)));
       });
     }
@@ -306,10 +333,11 @@ std::unique_ptr<webrtc::VideoEncoder> MomoVideoEncoderFactory::Create(
 #endif
 
 #if defined(USE_JETSON_ENCODER)
-    if (config_.h264_encoder == VideoCodecInfo::Type::Jetson) {
+    if (config_.h264_encoder == VideoCodecInfo::Type::Jetson &&
+        sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH264)) {
       return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) {
         return std::unique_ptr<webrtc::VideoEncoder>(
-            absl::make_unique<JetsonVideoEncoder>(
+            absl::make_unique<sora::JetsonVideoEncoder>(
                 cricket::CreateVideoCodec(format)));
       });
     }
@@ -357,6 +385,18 @@ std::unique_ptr<webrtc::VideoEncoder> MomoVideoEncoderFactory::Create(
                            });
     }
 #endif
+
+#if defined(USE_JETSON_ENCODER)
+    if (config_.h265_encoder == VideoCodecInfo::Type::Jetson &&
+        sora::JetsonVideoEncoder::IsSupported(webrtc::kVideoCodecH265)) {
+      return WithSimulcast(format, [](const webrtc::SdpVideoFormat& format) {
+        return std::unique_ptr<webrtc::VideoEncoder>(
+            absl::make_unique<sora::JetsonVideoEncoder>(
+                cricket::CreateVideoCodec(format)));
+      });
+    }
+#endif
+
 #if defined(USE_NVCODEC_ENCODER)
     if (config_.h265_encoder == VideoCodecInfo::Type::NVIDIA &&
         sora::NvCodecVideoEncoder::IsSupported(config_.cuda_context,

diff --git a/src/hwenc_jetson/jetson_buffer.h → ...include/sora/hwenc_jetson/jetson_buffer.h b/src/hwenc_jetson/jetson_buffer.h → ...include/sora/hwenc_jetson/jetson_buffer.h
@@ -1,5 +1,5 @@
-#ifndef JETSON_BUFFER_H_
-#define JETSON_BUFFER_H_
+#ifndef SORA_HWENC_JETSON_JETSON_BUFFER_H_
+#define SORA_HWENC_JETSON_JETSON_BUFFER_H_
 
 #include <linux/videodev2.h>
 #include <memory>
@@ -10,12 +10,10 @@
 #include <common_video/libyuv/include/webrtc_libyuv.h>
 #include <rtc_base/memory/aligned_malloc.h>
 
-// Jetson Linux Multimedia API
-#include <nvbufsurface.h>
-#include <nvbufsurftransform.h>
-
 #include "jetson_jpeg_decoder.h"
 
+namespace sora {
+
 class JetsonBuffer : public webrtc::VideoFrameBuffer {
  public:
   static rtc::scoped_refptr<JetsonBuffer> Create(
@@ -77,4 +75,7 @@ class JetsonBuffer : public webrtc::VideoFrameBuffer {
   const std::unique_ptr<uint8_t, webrtc::AlignedFreeDeleter> data_;
   size_t length_;
 };
-#endif  // JETSON_BUFFER_H_
+
+}  // namespace sora
+
+#endif