From 3f0025a5037e5b55de1f2cc337a4c813cc3d3d37 Mon Sep 17 00:00:00 2001 From: victorpasoshnikov <133087930+victorpasoshnikov@users.noreply.github.com> Date: Mon, 31 Jul 2023 22:09:35 +0300 Subject: [PATCH] [UWP] Fix wrong ROI coordinates for rendering (#1044) b/287515452 Change-Id: I1c0bf98e2abb5de0c6e4448b92502df88cd03cb5 (cherry picked from commit b3f339ff0a6a0191a664bcc0f4252b20b13c8523) --- .../xb1/shared/gpu_base_video_decoder.cc | 563 ++++++++++++++++++ 1 file changed, 563 insertions(+) create mode 100644 starboard/xb1/shared/gpu_base_video_decoder.cc diff --git a/starboard/xb1/shared/gpu_base_video_decoder.cc b/starboard/xb1/shared/gpu_base_video_decoder.cc new file mode 100644 index 000000000000..32749b16278b --- /dev/null +++ b/starboard/xb1/shared/gpu_base_video_decoder.cc @@ -0,0 +1,563 @@ +// Copyright 2021 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "starboard/xb1/shared/gpu_base_video_decoder.h" + +#include +#include + +#include "starboard/shared/uwp/application_uwp.h" +#include "starboard/shared/uwp/async_utils.h" +#include "starboard/shared/uwp/decoder_utils.h" +#include "starboard/shared/uwp/extended_resources_manager.h" +#include "starboard/shared/win32/decode_target_internal.h" +#include "starboard/shared/win32/error_utils.h" +#include "starboard/thread.h" +#include "third_party/angle/include/EGL/egl.h" +#include "third_party/angle/include/EGL/eglext.h" +#include "third_party/angle/include/GLES2/gl2.h" +#include "third_party/angle/include/GLES2/gl2ext.h" +#include "third_party/angle/include/angle_hdr.h" +#if defined(INTERNAL_BUILD) +#include "third_party/internal/libav1_xb1/libav1/d3dx12.h" +#endif // defined(INTERNAL_BUILD) + +namespace starboard { +namespace xb1 { +namespace shared { + +namespace { + +using Microsoft::WRL::ComPtr; +using starboard::shared::starboard::player::JobThread; +using starboard::shared::starboard::player::filter::VideoFrame; +using starboard::shared::uwp::ApplicationUwp; +using starboard::shared::uwp::ExtendedResourcesManager; +using starboard::shared::uwp::FindByTimestamp; +using starboard::shared::uwp::RemoveByTimestamp; +using starboard::shared::uwp::UpdateHdrColorMetadataToCurrentDisplay; +using Windows::Graphics::Display::Core::HdmiDisplayInformation; + +// Limit the number of pending buffers. +constexpr int kMaxNumberOfPendingBuffers = 8; +// Limit the cached presenting images. +constexpr int kNumberOfCachedPresentingImage = 3; + +const char kDecoderThreadName[] = "gpu_video_decoder_thread"; + +} // namespace + +class GpuVideoDecoderBase::GPUDecodeTargetPrivate + : public SbDecodeTargetPrivate { + public: + GPUDecodeTargetPrivate( + void* egl_display, + void* egl_config, + const scoped_refptr& image) + : egl_display_(egl_display), egl_config_(egl_config), image_(image) { + SB_DCHECK(egl_display_); + SB_DCHECK(egl_config); + + if (image->bit_depth() == 8) { + info.format = kSbDecodeTargetFormat3PlaneYUVI420; + } else { + SB_DCHECK(image->bit_depth() == 10); +#if SB_API_VERSION >= 14 + info.format = image->is_compacted() + ? kSbDecodeTargetFormat3Plane10BitYUVI420Compact + : kSbDecodeTargetFormat3Plane10BitYUVI420; +#else // SB_API_VERSION >= 14 + SB_DCHECK(!image->is_compacted()); + info.format = kSbDecodeTargetFormat3Plane10BitYUVI420; +#endif // SB_API_VERSION >= 14 + } + info.is_opaque = true; + info.width = image->width(); + info.height = image->height(); + + GLuint gl_textures_yuv[kNumberOfPlanes] = {}; + glGenTextures(kNumberOfPlanes, gl_textures_yuv); + SB_DCHECK(glGetError() == GL_NO_ERROR); + + for (unsigned int i = 0; i < kNumberOfPlanes; ++i) { + const int stride = image->stride(i); + const int subsampling = i > 0; + const int width = info.width >> subsampling; + const int height = info.height >> subsampling; + EGLint texture_attributes[] = {EGL_WIDTH, + static_cast(stride), + EGL_HEIGHT, + static_cast(height), + EGL_TEXTURE_TARGET, + EGL_TEXTURE_2D, + EGL_TEXTURE_FORMAT, + EGL_TEXTURE_RGBA, + EGL_NONE}; + surfaces_[i] = eglCreatePbufferFromClientBuffer( + egl_display_, EGL_D3D_TEXTURE_ANGLE, image->texture(i).Get(), + egl_config, texture_attributes); + SB_DCHECK(glGetError() == GL_NO_ERROR && surfaces_[i] != EGL_NO_SURFACE); + glBindTexture(GL_TEXTURE_2D, gl_textures_yuv[i]); + SB_DCHECK(glGetError() == GL_NO_ERROR); + bool result = + eglBindTexImage(egl_display_, surfaces_[i], EGL_BACK_BUFFER); + SB_DCHECK(result); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + D3D11_TEXTURE2D_DESC tex_desc = {}; + image->texture(i)->GetDesc(&tex_desc); + + SbDecodeTargetInfoPlane* plane = &info.planes[i]; + plane->width = + image->is_compacted() ? tex_desc.Width * 3 : tex_desc.Width; + plane->height = tex_desc.Height; + plane->content_region.left = image->texture_corner_left(i); + plane->content_region.top = plane->height - image->texture_corner_top(i); + plane->content_region.bottom = plane->content_region.top - height; + plane->content_region.right = plane->content_region.left + width; + plane->texture = gl_textures_yuv[i]; + plane->gl_texture_target = GL_TEXTURE_2D; + plane->gl_texture_format = GL_RED_EXT; + } + } + + ~GPUDecodeTargetPrivate() override { + for (unsigned int i = 0; i < kNumberOfPlanes; ++i) { + glDeleteTextures(1, &(info.planes[i].texture)); + eglReleaseTexImage(egl_display_, surfaces_[i], EGL_BACK_BUFFER); + eglDestroySurface(egl_display_, surfaces_[i]); + } + } + + SbTime timestamp() { return image_->timestamp(); } + + void ReleaseImage() { + // Release the codec resource, while the D3D textures are still safe to use. + SB_DCHECK(image_); + image_ = nullptr; + } + + private: + // Hold the codec resource until it's not needed in render pipeline to prevent + // it being reused and overwritten. + scoped_refptr image_; + // EGLSurface is defined as void* in "third_party/angle/include/EGL/egl.h". + // Use void* directly here to avoid `egl.h` being included broadly. + void* surfaces_[kNumberOfPlanes]; + void* egl_display_; + void* egl_config_; +}; + +GpuVideoDecoderBase::GpuVideoDecoderBase( + SbDecodeTargetGraphicsContextProvider* + decode_target_graphics_context_provider, + const VideoStreamInfo& video_stream_info, + bool is_hdr_video, + const ComPtr& d3d12_device, + void* d3d12_queue) + : decode_target_context_runner_(decode_target_graphics_context_provider), + is_hdr_video_(is_hdr_video), + d3d12_device_(d3d12_device), + d3d12_queue_(d3d12_queue) { + SB_DCHECK(d3d12_device_); + SB_DCHECK(d3d12_queue_); + + egl_display_ = eglGetDisplay(EGL_DEFAULT_DISPLAY); + EGLint attribute_list[] = {EGL_SURFACE_TYPE, // this must be first + EGL_WINDOW_BIT | EGL_PBUFFER_BIT, + EGL_RED_SIZE, + 8, + EGL_GREEN_SIZE, + 8, + EGL_BLUE_SIZE, + 8, + EGL_ALPHA_SIZE, + 8, + EGL_BIND_TO_TEXTURE_RGBA, + EGL_TRUE, + EGL_RENDERABLE_TYPE, + EGL_OPENGL_ES2_BIT, + EGL_NONE}; + SB_DCHECK(egl_display_); + EGLint num_configs; + int result = eglChooseConfig(egl_display_, attribute_list, &egl_config_, 1, + &num_configs); + SB_DCHECK(result); + + ComPtr d3d11_device = + starboard::shared::uwp::GetDirectX11Device(egl_display_); + HRESULT hr = d3d11_device.As(&d3d11_device_); + SB_DCHECK(SUCCEEDED(hr)); + + if (is_hdr_video_) { + UpdateHdrMetadata(video_stream_info.color_metadata); + } + frame_width_ = video_stream_info.frame_width; + frame_height_ = video_stream_info.frame_height; +} + +GpuVideoDecoderBase::~GpuVideoDecoderBase() { + // Reset() should be already called before ~GpuVideoDecoderBase(). + SB_DCHECK(!decoder_thread_); + SB_DCHECK(pending_inputs_.empty()); + SB_DCHECK(written_inputs_.empty()); + SB_DCHECK(output_queue_.empty()); + SB_DCHECK(decoder_behavior_.load() == kDecodingStopped); + // All presenting decode targets should be released. + SB_DCHECK(presenting_decode_targets_.empty()); + + if (ApplicationUwp::Get()->IsHdrSupported() && IsHdrAngleModeEnabled()) { + SetHdrAngleModeEnabled(false); + auto hdmi_display_info = HdmiDisplayInformation::GetForCurrentView(); + starboard::shared::uwp::WaitForComplete( + hdmi_display_info->SetDefaultDisplayModeAsync()); + } +} + +void GpuVideoDecoderBase::Initialize(const DecoderStatusCB& decoder_status_cb, + const ErrorCB& error_cb) { + SB_DCHECK(decoder_status_cb); + SB_DCHECK(!decoder_status_cb_); + SB_DCHECK(error_cb); + SB_DCHECK(!error_cb_); + decoder_status_cb_ = decoder_status_cb; + error_cb_ = error_cb; +} + +size_t GpuVideoDecoderBase::GetMaxNumberOfCachedFrames() const { + return GetMaxNumberOfCachedFramesInternal() - + number_of_presenting_decode_targets_; +} + +void GpuVideoDecoderBase::WriteInputBuffers(const InputBuffers& input_buffers) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(decoder_status_cb_); + SB_DCHECK(input_buffers.size() == 1); + SB_DCHECK(input_buffers[0]); + SB_DCHECK(decoder_behavior_.load() != kResettingDecoder); + SB_DCHECK(decoder_behavior_.load() != kEndingStream); + + if (error_occured_.load()) { + return; + } + + const auto& input_buffer = input_buffers[0]; + + if (!decoder_thread_) { + decoder_thread_.reset(new JobThread(kDecoderThreadName)); + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::InitializeCodecIfNeededInternal, this)); + } + + bool needs_more_input = false; + { + ScopedLock pending_inputs_lock(pending_inputs_mutex_); + pending_inputs_.push_back(input_buffer); + needs_more_input = pending_inputs_.size() < kMaxNumberOfPendingBuffers; + } + decoder_behavior_.store(kDecodingFrames); + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::DecodeOneBuffer, this)); + if (needs_more_input) { + decoder_status_cb_(kNeedMoreInput, nullptr); + } +} + +void GpuVideoDecoderBase::WriteEndOfStream() { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(decoder_status_cb_); + SB_DCHECK(decoder_behavior_.load() != kResettingDecoder); + SB_DCHECK(decoder_behavior_.load() != kEndingStream); + + if (error_occured_.load()) { + return; + } + + if (decoder_thread_) { + SB_DCHECK(decoder_behavior_.load() == kDecodingFrames); + decoder_behavior_.store(kEndingStream); + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::DecodeEndOfStream, this)); + return; + } + + SB_DCHECK(decoder_behavior_.load() == kDecodingStopped); + decoder_status_cb_(kBufferFull, VideoFrame::CreateEOSFrame()); +} + +void GpuVideoDecoderBase::Reset() { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(decoder_status_cb_); + decoder_behavior_.store(kResettingDecoder); + if (decoder_thread_) { + // Release stored frames to free frame buffers. + decoder_status_cb_(kReleaseAllFrames, nullptr); + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::DrainDecoder, this)); + decoder_thread_.reset(); + } + pending_inputs_.clear(); + written_inputs_.clear(); + // Release all frames after decoder thread is destroyed. + decoder_status_cb_(kReleaseAllFrames, nullptr); + { + ScopedLock output_queue_lock(output_queue_mutex_); + output_queue_.clear(); + } + error_occured_.store(false); + decoder_behavior_.store(kDecodingStopped); + is_drain_decoder_called_ = false; +} + +SbDecodeTarget GpuVideoDecoderBase::GetCurrentDecodeTarget() { + scoped_refptr image = nullptr; + { + ScopedLock output_queue_lock(output_queue_mutex_); + if (!output_queue_.empty()) { + image = output_queue_.front(); + } + } + + if (!image && presenting_decode_targets_.empty()) { + return kSbDecodeTargetInvalid; + } + + if (presenting_decode_targets_.empty() || + (image && + image->timestamp() != presenting_decode_targets_.back()->timestamp())) { + // Create the new decode target and update hdr meta data. + if (is_hdr_video_) { + UpdateHdrMetadata(image->color_metadata()); + } + presenting_decode_targets_.push_back( + new GPUDecodeTargetPrivate(egl_display_, egl_config_, image)); + number_of_presenting_decode_targets_++; + } + + // Increment the refcount for the returned decode target. + presenting_decode_targets_.back()->AddRef(); + + // There's a data synchronization issue (b/180532476) between decoder and + // render pipelines. If we release the decode target immediately after it's + // released on render pipeline (the underllying resources may be reused by the + // libvpx/av1 decoder), there's a chance that the rendering frame gets + // overwritten. The root cause is still unclear and under investigating. So, + // as a workaround, we retain the decode target for longer time until it's no + // longer used by the renderer thread. + while (presenting_decode_targets_.size() > kNumberOfCachedPresentingImage && + presenting_decode_targets_.front()->refcount == 1) { + SbDecodeTargetRelease(presenting_decode_targets_.front()); + presenting_decode_targets_.pop_front(); + number_of_presenting_decode_targets_--; + } + return presenting_decode_targets_.back(); +} + +bool GpuVideoDecoderBase::BelongsToDecoderThread() const { + return decoder_thread_->job_queue()->BelongsToCurrentThread(); +} + +void GpuVideoDecoderBase::OnOutputRetrieved( + const scoped_refptr& image) { + SB_DCHECK(decoder_thread_); + SB_DCHECK(decoder_status_cb_); + SB_DCHECK(image); + + if (decoder_behavior_.load() == kResettingDecoder || error_occured_) { + return; + } + + if (!BelongsToDecoderThread()) { + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::OnOutputRetrieved, this, image)); + return; + } + + SbTime timestamp = image->timestamp(); + const auto iter = FindByTimestamp(written_inputs_, timestamp); + SB_DCHECK(iter != written_inputs_.cend()); + if (is_hdr_video_) { + image->AttachColorMetadata((*iter)->video_stream_info().color_metadata); + } + written_inputs_.erase(iter); + + scoped_refptr frame(new VideoFrameImpl( + timestamp, std::bind(&GpuVideoDecoderBase::DeleteVideoFrame, this, + std::placeholders::_1))); + decoder_status_cb_( + decoder_behavior_.load() == kEndingStream ? kBufferFull : kNeedMoreInput, + frame); + + if (!frame->HasOneRef()) { + ScopedLock output_queue_lock(output_queue_mutex_); + output_queue_.push_back(image); + } +} + +void GpuVideoDecoderBase::OnDecoderDrained() { + SB_DCHECK(decoder_thread_); + SB_DCHECK(decoder_status_cb_); + SB_DCHECK(decoder_behavior_.load() == kEndingStream || + decoder_behavior_.load() == kResettingDecoder); + + if (decoder_behavior_.load() == kResettingDecoder || error_occured_) { + return; + } + + if (!BelongsToDecoderThread()) { + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::OnDecoderDrained, this)); + return; + } + + SB_DCHECK(written_inputs_.empty()); + if (decoder_behavior_.load() == kEndingStream) { + decoder_status_cb_(kBufferFull, VideoFrame::CreateEOSFrame()); + } + decoder_behavior_.store(kDecodingStopped); +} + +void GpuVideoDecoderBase::ClearCachedImages() { + SB_DCHECK(output_queue_.empty()); + // All cached images should be released. + decode_target_context_runner_.RunOnGlesContext( + std::bind(&GpuVideoDecoderBase::ClearPresentingDecodeTargets, this)); +} + +void GpuVideoDecoderBase::ReportError(const SbPlayerError error, + const std::string& error_message) { + SB_DCHECK(error_cb_); + if (!error_occured_.exchange(true)) { + SB_LOG(ERROR) << error_message << " (error code: " << error << " )"; + error_cb_(error, error_message); + } +} + +bool GpuVideoDecoderBase::IsCacheFull() { + SB_DCHECK(decoder_thread_); + SB_DCHECK(BelongsToDecoderThread()); + + ScopedLock output_queue_lock(output_queue_mutex_); + return written_inputs_.size() + output_queue_.size() + + number_of_presenting_decode_targets_ >= + GetMaxNumberOfCachedFramesInternal(); +} + +void GpuVideoDecoderBase::DecodeOneBuffer() { + SB_DCHECK(decoder_thread_); + SB_DCHECK(BelongsToDecoderThread()); + + if (decoder_behavior_.load() == kResettingDecoder || error_occured_) { + return; + } + + if (IsCacheFull()) { + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::DecodeOneBuffer, this), + kSbTimeMillisecond); + return; + } + + { + ScopedLock pending_inputs_lock(pending_inputs_mutex_); + SB_DCHECK(!pending_inputs_.empty()); + written_inputs_.push_back(pending_inputs_.front()); + pending_inputs_.pop_front(); + } + DecodeInternal(written_inputs_.back()); +} + +void GpuVideoDecoderBase::DecodeEndOfStream() { + SB_DCHECK(decoder_thread_); + SB_DCHECK(BelongsToDecoderThread()); + SB_DCHECK(decoder_status_cb_); + + if (decoder_behavior_.load() == kResettingDecoder || error_occured_) { + return; + } + + { + ScopedLock pending_inputs_lock(pending_inputs_mutex_); + if (!pending_inputs_.empty()) { + decoder_thread_->job_queue()->Schedule( + std::bind(&GpuVideoDecoderBase::DecodeEndOfStream, this), + kSbTimeMillisecond); + return; + } + } + DrainDecoder(); +} + +void GpuVideoDecoderBase::DrainDecoder() { + SB_DCHECK(BelongsToDecoderThread()); + // Use |is_drain_decoder_called_| to prevent calling DrainDecoderInternal() + // twice. Theoretically, if Reset() is called during DecodeEndOfStream() + // executing, DrainDecoderInternal() could be called twice, one from + // DecodeEndOfStream(), and one from Reset(). + if (!is_drain_decoder_called_) { + is_drain_decoder_called_ = true; + DrainDecoderInternal(); + } +} + +void GpuVideoDecoderBase::DeleteVideoFrame(const VideoFrame* video_frame) { + ScopedLock output_queue_lock(output_queue_mutex_); + RemoveByTimestamp(&output_queue_, video_frame->timestamp()); +} + +void GpuVideoDecoderBase::UpdateHdrMetadata( + const SbMediaColorMetadata& color_metadata) { + SB_DCHECK(is_hdr_video_); + if (!ApplicationUwp::Get()->IsHdrSupported()) { + ReportError(kSbPlayerErrorCapabilityChanged, + "HDR sink lost while HDR video playing."); + return; + } + if (!needs_hdr_metadata_update_) { + return; + } + needs_hdr_metadata_update_ = false; + if (!IsHdrAngleModeEnabled()) { + SetHdrAngleModeEnabled(true); + } + if (memcmp(&color_metadata, &last_presented_color_metadata_, + sizeof(color_metadata)) != 0) { + last_presented_color_metadata_ = color_metadata; + UpdateHdrColorMetadataToCurrentDisplay(color_metadata); + } +} + +void GpuVideoDecoderBase::ClearPresentingDecodeTargets() { + // Delete all unused decode targets. + while (!presenting_decode_targets_.empty() && + presenting_decode_targets_.front()->refcount == 1) { + SbDecodeTargetRelease(presenting_decode_targets_.front()); + presenting_decode_targets_.pop_front(); + } + // The remaining decode targets are still used by the render pipeline. Force + // to release the underlying image to release the codec resources and + // decrement the refcount. + for (auto it = presenting_decode_targets_.begin(); + it != presenting_decode_targets_.end(); ++it) { + (*it)->ReleaseImage(); + SbDecodeTargetRelease(*it); + } + presenting_decode_targets_.clear(); + number_of_presenting_decode_targets_ = 0; +} + +} // namespace shared +} // namespace xb1 +} // namespace starboard