From 507638ca8a47ed089c244a8da6a15246f3979d6f Mon Sep 17 00:00:00 2001 From: Bo-Rong Chen Date: Thu, 14 Nov 2024 14:13:55 -0800 Subject: [PATCH] [media] Support progressive playback This PR brings the progressive demuxer from Cobalt C25 to support progressive videos. b/322033277 --- media/base/decoder_buffer.h | 1 + media/filters/demuxer_manager.cc | 23 +- media/filters/demuxer_manager.h | 4 + media/filters/ffmpeg_demuxer.cc | 1 + media/starboard/BUILD.gn | 25 + .../starboard/progressive/avc_access_unit.cc | 316 +++++ media/starboard/progressive/avc_access_unit.h | 81 ++ media/starboard/progressive/avc_parser.cc | 502 +++++++ media/starboard/progressive/avc_parser.h | 83 ++ .../progressive/data_source_reader.cc | 120 ++ .../progressive/data_source_reader.h | 69 + .../progressive/demuxer_extension_wrapper.cc | 1111 ++++++++++++++++ .../progressive/demuxer_extension_wrapper.h | 249 ++++ media/starboard/progressive/endian_util.h | 141 ++ .../progressive/mock_data_source_reader.h | 36 + media/starboard/progressive/mp4_map.cc | 1167 +++++++++++++++++ media/starboard/progressive/mp4_map.h | 218 +++ .../starboard/progressive/mp4_map_unittest.cc | 1121 ++++++++++++++++ media/starboard/progressive/mp4_parser.cc | 665 ++++++++++ media/starboard/progressive/mp4_parser.h | 113 ++ .../progressive/progressive_demuxer.cc | 566 ++++++++ .../progressive/progressive_demuxer.h | 204 +++ .../progressive/progressive_parser.cc | 60 + .../progressive/progressive_parser.h | 79 ++ media/starboard/progressive/rbsp_stream.cc | 215 +++ media/starboard/progressive/rbsp_stream.h | 71 + .../progressive/rbsp_stream_unittest.cc | 635 +++++++++ .../platform/media/web_media_player_impl.cc | 7 + 28 files changed, 7882 insertions(+), 1 deletion(-) create mode 100644 media/starboard/progressive/avc_access_unit.cc create mode 100644 media/starboard/progressive/avc_access_unit.h create mode 100644 media/starboard/progressive/avc_parser.cc create mode 100644 media/starboard/progressive/avc_parser.h create mode 100644 media/starboard/progressive/data_source_reader.cc create mode 100644 media/starboard/progressive/data_source_reader.h create mode 100644 media/starboard/progressive/demuxer_extension_wrapper.cc create mode 100644 media/starboard/progressive/demuxer_extension_wrapper.h create mode 100644 media/starboard/progressive/endian_util.h create mode 100644 media/starboard/progressive/mock_data_source_reader.h create mode 100644 media/starboard/progressive/mp4_map.cc create mode 100644 media/starboard/progressive/mp4_map.h create mode 100644 media/starboard/progressive/mp4_map_unittest.cc create mode 100644 media/starboard/progressive/mp4_parser.cc create mode 100644 media/starboard/progressive/mp4_parser.h create mode 100644 media/starboard/progressive/progressive_demuxer.cc create mode 100644 media/starboard/progressive/progressive_demuxer.h create mode 100644 media/starboard/progressive/progressive_parser.cc create mode 100644 media/starboard/progressive/progressive_parser.h create mode 100644 media/starboard/progressive/rbsp_stream.cc create mode 100644 media/starboard/progressive/rbsp_stream.h create mode 100644 media/starboard/progressive/rbsp_stream_unittest.cc diff --git a/media/base/decoder_buffer.h b/media/base/decoder_buffer.h index 27a87eb204e5..c8d9057bfda7 100644 --- a/media/base/decoder_buffer.h +++ b/media/base/decoder_buffer.h @@ -266,6 +266,7 @@ class MEDIA_EXPORT DecoderBuffer // If there's no data in this buffer, it represents end of stream. #if BUILDFLAG(USE_STARBOARD_MEDIA) bool end_of_stream() const { return !data_; } + void shrink_to(size_t size) { DCHECK_LE(size, size_); size_ = size; } #else // BUILDFLAG(USE_STARBOARD_MEDIA) bool end_of_stream() const { return !read_only_mapping_.IsValid() && !writable_mapping_.IsValid() && diff --git a/media/filters/demuxer_manager.cc b/media/filters/demuxer_manager.cc index 25db5bc3878e..06c17677ee80 100644 --- a/media/filters/demuxer_manager.cc +++ b/media/filters/demuxer_manager.cc @@ -25,6 +25,10 @@ #include "media/filters/manifest_demuxer.h" #endif // BUILDFLAG(ENABLE_HLS_DEMUXER) +#if BUILDFLAG(USE_STARBOARD_MEDIA) +#include "media/starboard/progressive/progressive_demuxer.h" +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + namespace media { namespace { @@ -152,6 +156,7 @@ DemuxerManager::DemuxerManager( enable_instant_source_buffer_gc_(enable_instant_source_buffer_gc), demuxer_override_(std::move(demuxer_override)) { DCHECK(client_); + LOG(ERROR) << "Cobalt " << __func__; } DemuxerManager::~DemuxerManager() { @@ -372,6 +377,7 @@ PipelineStatus DemuxerManager::CreateDemuxer( DataSource::Preload preload, bool has_poster, DemuxerManager::DemuxerCreatedCB on_demuxer_created) { + LOG(ERROR) << "Cobalt " << __func__; // TODO(crbug/1377053) return a better error if (!client_) { return DEMUXER_ERROR_COULD_NOT_OPEN; @@ -407,12 +413,18 @@ PipelineStatus DemuxerManager::CreateDemuxer( // run in the demuxer override case? SetDemuxer(std::move(demuxer_override_)); } else if (!load_media_source) { -#if BUILDFLAG(ENABLE_FFMPEG) +#if BUILDFLAG(USE_STARBOARD_MEDIA) + LOG(ERROR) << "Cobalt " << __func__; + SetDemuxer(CreateProgressiveDemuxer()); +#elif BUILDFLAG(ENABLE_FFMPEG) + LOG(ERROR) << "Cobalt " << __func__; SetDemuxer(CreateFFmpegDemuxer()); #else + LOG(ERROR) << "Cobalt " << __func__; return DEMUXER_ERROR_COULD_NOT_OPEN; #endif } else { + LOG(ERROR) << "Cobalt " << __func__; DCHECK(!HasDataSource()); SetDemuxer(CreateChunkDemuxer()); is_static = false; @@ -596,6 +608,15 @@ std::unique_ptr DemuxerManager::CreateMediaUrlDemuxer( } #endif // BUILDFLAG(IS_ANDROID) +#if BUILDFLAG(USE_STARBOARD_MEDIA) +std::unique_ptr DemuxerManager::CreateProgressiveDemuxer() { + DCHECK(data_source_); + return std::make_unique( + media_task_runner_, data_source_.get(), + media_log_.get()); +} +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + void DemuxerManager::SetDemuxer(std::unique_ptr demuxer) { DCHECK(!demuxer_); CHECK(demuxer); diff --git a/media/filters/demuxer_manager.h b/media/filters/demuxer_manager.h index 435bce8989bf..79e678cff2de 100644 --- a/media/filters/demuxer_manager.h +++ b/media/filters/demuxer_manager.h @@ -183,6 +183,10 @@ class MEDIA_EXPORT DemuxerManager { std::unique_ptr CreateMediaUrlDemuxer(bool hls_content); #endif // BUILDFLAG(IS_ANDROID) +#if BUILDFLAG(USE_STARBOARD_MEDIA) + std::unique_ptr CreateProgressiveDemuxer(); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + void SetDemuxer(std::unique_ptr demuxer); // Memory pressure listener specifically for when using ChunkDemuxer. diff --git a/media/filters/ffmpeg_demuxer.cc b/media/filters/ffmpeg_demuxer.cc index d5ab57b0b41b..4663ffe4a9d9 100644 --- a/media/filters/ffmpeg_demuxer.cc +++ b/media/filters/ffmpeg_demuxer.cc @@ -924,6 +924,7 @@ FFmpegDemuxer::FFmpegDemuxer( DCHECK(task_runner_.get()); DCHECK(data_source_); DCHECK(media_tracks_updated_cb_); + LOG(ERROR) << "Cobalt " << __func__; } FFmpegDemuxer::~FFmpegDemuxer() { diff --git a/media/starboard/BUILD.gn b/media/starboard/BUILD.gn index 9f860dfdb446..25b2514c721d 100644 --- a/media/starboard/BUILD.gn +++ b/media/starboard/BUILD.gn @@ -63,6 +63,24 @@ source_set("starboard") { "decoder_buffer_allocator.cc", "decoder_buffer_allocator.h", "decoder_buffer_memory_info.h", + "progressive/avc_access_unit.cc", + "progressive/avc_access_unit.h", + "progressive/avc_parser.cc", + "progressive/avc_parser.h", + "progressive/data_source_reader.cc", + "progressive/data_source_reader.h", + "progressive/demuxer_extension_wrapper.cc", + "progressive/demuxer_extension_wrapper.h", + "progressive/mp4_map.cc", + "progressive/mp4_map.h", + "progressive/mp4_parser.cc", + "progressive/mp4_parser.h", + "progressive/progressive_demuxer.cc", + "progressive/progressive_demuxer.h", + "progressive/progressive_parser.cc", + "progressive/progressive_parser.h", + "progressive/rbsp_stream.cc", + "progressive/rbsp_stream.h", "sbplayer_bridge.cc", "sbplayer_bridge.h", "sbplayer_interface.cc", @@ -89,6 +107,10 @@ source_set("starboard") { "//starboard/common", ] + if (use_starboard_media) { + deps += [ "//third_party/abseil-cpp:absl" ] + } + configs += [ "//media:subcomponent_config" ] } @@ -98,6 +120,9 @@ source_set("unit_tests") { if (use_starboard_media) { sources += [ "bidirectional_fit_reuse_allocator_test.cc", + "progressive/mock_data_source_reader.h", + "progressive/mp4_map_unittest.cc", + "progressive/rbsp_stream_unittest.cc", "starboard_utils_test.cc", ] } diff --git a/media/starboard/progressive/avc_access_unit.cc b/media/starboard/progressive/avc_access_unit.cc new file mode 100644 index 000000000000..9cab0b52d9e0 --- /dev/null +++ b/media/starboard/progressive/avc_access_unit.cc @@ -0,0 +1,316 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/avc_access_unit.h" + +#include + +#include "media/base/decoder_buffer.h" +#include "media/base/timestamp_constants.h" +#include "media/starboard/progressive/endian_util.h" +#include "media/starboard/progressive/progressive_parser.h" + +namespace media { + +namespace { + +bool ReadBytes(uint64_t offset, + size_t size, + uint8_t* buffer, + DataSourceReader* reader) { + if (reader->BlockingRead(offset, size, buffer) != size) { + LOG(ERROR) << "unable to download AU"; + return false; + } + return true; +} + +// ==== EndOfStreamAU ================================================== + +class EndOfStreamAU : public AvcAccessUnit { + public: + EndOfStreamAU(Type type, TimeDelta timestamp) + : type_(type), timestamp_(timestamp), duration_(kInfiniteDuration) {} + + private: + bool IsEndOfStream() const override { return true; } + bool IsValid() const override { return true; } + bool Read(DataSourceReader* reader, DecoderBuffer* buffer) override { + NOTREACHED(); + return false; + } + Type GetType() const override { return type_; } + bool IsKeyframe() const override { + NOTREACHED(); + return false; + } + bool AddPrepend() const override { + NOTREACHED(); + return false; + } + size_t GetSize() const override { return 0; } + size_t GetMaxSize() const override { return 0; } + TimeDelta GetTimestamp() const override { return timestamp_; } + TimeDelta GetDuration() const override { return duration_; } + void SetDuration(TimeDelta duration) override { duration_ = duration; } + void SetTimestamp(TimeDelta timestamp) override { timestamp_ = timestamp; } + + Type type_; + TimeDelta timestamp_; + TimeDelta duration_; +}; + +// ==== AudioAU ======================================================= + +class AudioAU : public AvcAccessUnit { + public: + AudioAU(uint64_t offset, + size_t size, + size_t prepend_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser); + + private: + bool IsEndOfStream() const override { return false; } + bool IsValid() const override { + return offset_ != 0 && size_ != 0 && timestamp_ != kNoTimestamp; + } + bool Read(DataSourceReader* reader, DecoderBuffer* buffer) override; + Type GetType() const override { return DemuxerStream::AUDIO; } + bool IsKeyframe() const override { return is_keyframe_; } + bool AddPrepend() const override { return true; } + size_t GetSize() const override { return size_; } + size_t GetMaxSize() const override { return size_ + prepend_size_; } + TimeDelta GetTimestamp() const override { return timestamp_; } + TimeDelta GetDuration() const override { return duration_; } + void SetDuration(TimeDelta duration) override { duration_ = duration; } + void SetTimestamp(TimeDelta timestamp) override { timestamp_ = timestamp; } + + uint64_t offset_; + size_t size_; + size_t prepend_size_; + bool is_keyframe_; + TimeDelta timestamp_; + TimeDelta duration_; + ProgressiveParser* parser_; +}; + +AudioAU::AudioAU(uint64_t offset, + size_t size, + size_t prepend_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser) + : offset_(offset), + size_(size), + prepend_size_(prepend_size), + is_keyframe_(is_keyframe), + timestamp_(timestamp), + duration_(duration), + parser_(parser) {} + +bool AudioAU::Read(DataSourceReader* reader, DecoderBuffer* buffer) { + DCHECK_LE(size_ + prepend_size_, buffer->data_size()); + if (!ReadBytes(offset_, size_, buffer->writable_data() + prepend_size_, + reader)) { + return false; + } + + if (!parser_->Prepend(this, buffer)) { + LOG(ERROR) << "prepend fail"; + return false; + } + + return true; +} + +// ==== VideoAU ======================================================= + +class VideoAU : public AvcAccessUnit { + public: + VideoAU(uint64_t offset, + size_t size, + size_t prepend_size, + uint8_t length_of_nalu_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser); + + private: + bool IsEndOfStream() const override { return false; } + bool IsValid() const override { + return offset_ != 0 && size_ != 0 && timestamp_ != kNoTimestamp; + } + bool Read(DataSourceReader* reader, DecoderBuffer* buffer) override; + Type GetType() const override { return DemuxerStream::VIDEO; } + bool IsKeyframe() const override { return is_keyframe_; } + bool AddPrepend() const override { return is_keyframe_; } + size_t GetSize() const override { return size_; } + size_t GetMaxSize() const override { + // TODO : This code is a proof of concept. It should be fixed + // with more reasonable value once we have enough data. + return size_ + prepend_size_ + size_ / 1024 + 1024; + } + TimeDelta GetTimestamp() const override { return timestamp_; } + TimeDelta GetDuration() const override { return duration_; } + void SetDuration(TimeDelta duration) override { duration_ = duration; } + void SetTimestamp(TimeDelta timestamp) override { timestamp_ = timestamp; } + + uint64_t offset_; + size_t size_; + size_t prepend_size_; + uint8_t length_of_nalu_size_; + bool is_keyframe_; + TimeDelta timestamp_; + TimeDelta duration_; + ProgressiveParser* parser_; +}; + +VideoAU::VideoAU(uint64_t offset, + size_t size, + size_t prepend_size, + uint8_t length_of_nalu_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser) + : offset_(offset), + size_(size), + prepend_size_(prepend_size), + length_of_nalu_size_(length_of_nalu_size), + is_keyframe_(is_keyframe), + timestamp_(timestamp), + duration_(duration), + parser_(parser) { + CHECK_LE(length_of_nalu_size_, 4); + CHECK_NE(length_of_nalu_size_, 3); +} + +bool VideoAU::Read(DataSourceReader* reader, DecoderBuffer* buffer) { + size_t au_left = size_; // bytes left in the AU + uint64_t au_offset = offset_; // offset to read in the reader + size_t buf_left = buffer->data_size(); // bytes left in the buffer + // The current write position in the buffer + int64_t decoder_buffer_offset = prepend_size_; + + // The NALU is stored as [size][data][size][data].... We are going to + // transform it into [start code][data][start code][data].... + // The length of size is indicated by length_of_nalu_size_ + while (au_left >= length_of_nalu_size_ && buf_left >= kAnnexBStartCodeSize) { + uint8_t size_buf[4]; + uint32_t nal_size; + + // Store [start code] + memcpy(buffer->writable_data() + decoder_buffer_offset, kAnnexBStartCode, + kAnnexBStartCodeSize); + decoder_buffer_offset += kAnnexBStartCodeSize; + buf_left -= kAnnexBStartCodeSize; + + // Read [size] + if (!ReadBytes(au_offset, length_of_nalu_size_, size_buf, reader)) { + return false; + } + + au_offset += length_of_nalu_size_; + au_left -= length_of_nalu_size_; + + if (length_of_nalu_size_ == 1) { + nal_size = size_buf[0]; + } else if (length_of_nalu_size_ == 2) { + nal_size = endian_util::load_uint16_big_endian(size_buf); + } else { + DCHECK_EQ(length_of_nalu_size_, 4); + nal_size = endian_util::load_uint32_big_endian(size_buf); + } + + if (au_left < nal_size || buf_left < nal_size) { + break; + } + + // Read the [data] from reader into buf + if (!ReadBytes(au_offset, nal_size, + buffer->writable_data() + decoder_buffer_offset, reader)) { + return false; + } + + decoder_buffer_offset += nal_size; + au_offset += nal_size; + au_left -= nal_size; + buf_left -= nal_size; + } + + if (au_left != 0) { + LOG(ERROR) << "corrupted NALU"; + return false; + } + + size_ = decoder_buffer_offset; + buffer->shrink_to(size_); + + if (!parser_->Prepend(this, buffer)) { + LOG(ERROR) << "prepend fail"; + return false; + } + + return true; +} + +} // namespace + +// ==== AvcAccessUnit +// ================================================================ + +AvcAccessUnit::AvcAccessUnit() {} + +AvcAccessUnit::~AvcAccessUnit() {} + +// static +scoped_refptr AvcAccessUnit::CreateEndOfStreamAU( + DemuxerStream::Type type, + TimeDelta timestamp) { + return new EndOfStreamAU(type, timestamp); +} + +// static +scoped_refptr AvcAccessUnit::CreateAudioAU( + uint64_t offset, + size_t size, + size_t prepend_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser) { + return new AudioAU(offset, size, prepend_size, is_keyframe, timestamp, + duration, parser); +} + +// static +scoped_refptr AvcAccessUnit::CreateVideoAU( + uint64_t offset, + size_t size, + size_t prepend_size, + uint8_t length_of_nalu_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser) { + return new VideoAU(offset, size, prepend_size, length_of_nalu_size, + is_keyframe, timestamp, duration, parser); +} + +} // namespace media diff --git a/media/starboard/progressive/avc_access_unit.h b/media/starboard/progressive/avc_access_unit.h new file mode 100644 index 000000000000..086ddbb0fea2 --- /dev/null +++ b/media/starboard/progressive/avc_access_unit.h @@ -0,0 +1,81 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_AVC_ACCESS_UNIT_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_AVC_ACCESS_UNIT_H_ + +#include "base/memory/ref_counted.h" +#include "media/base/demuxer_stream.h" +#include "media/starboard/progressive/data_source_reader.h" + +namespace media { + +class ProgressiveParser; + +static const int kAnnexBStartCodeSize = 4; +static const uint8_t kAnnexBStartCode[] = {0, 0, 0, 1}; + +// The basic unit of currency between ProgressiveDemuxer and ProgressiveParser, +// the AvcAccessUnit defines all needed information for a given AvcAccessUnit +// (Frame) of encoded media data. +class AvcAccessUnit : public base::RefCountedThreadSafe { + public: + typedef base::TimeDelta TimeDelta; + typedef DemuxerStream::Type Type; + + static scoped_refptr CreateEndOfStreamAU(Type type, + TimeDelta timestamp); + static scoped_refptr CreateAudioAU(uint64_t offset, + size_t size, + size_t prepend_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser); + static scoped_refptr CreateVideoAU(uint64_t offset, + size_t size, + size_t prepend_size, + uint8_t length_of_nalu_size, + bool is_keyframe, + TimeDelta timestamp, + TimeDelta duration, + ProgressiveParser* parser); + + virtual bool IsEndOfStream() const = 0; + virtual bool IsValid() const = 0; + // Read an AU from reader to buffer and also do all the necessary operations + // like prepending head to make it ready to decode. + virtual bool Read(DataSourceReader* reader, DecoderBuffer* buffer) = 0; + virtual Type GetType() const = 0; + virtual bool IsKeyframe() const = 0; + virtual bool AddPrepend() const = 0; + // Get the size of this AU, it is always no larger than its max size. + virtual size_t GetSize() const = 0; + // Get the max required buffer of this AU + virtual size_t GetMaxSize() const = 0; + virtual TimeDelta GetTimestamp() const = 0; + virtual TimeDelta GetDuration() const = 0; + virtual void SetDuration(TimeDelta duration) = 0; + virtual void SetTimestamp(TimeDelta timestamp) = 0; + + protected: + friend class base::RefCountedThreadSafe; + + AvcAccessUnit(); + virtual ~AvcAccessUnit(); +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_AVC_ACCESS_UNIT_H_ diff --git a/media/starboard/progressive/avc_parser.cc b/media/starboard/progressive/avc_parser.cc new file mode 100644 index 000000000000..d121ee399508 --- /dev/null +++ b/media/starboard/progressive/avc_parser.cc @@ -0,0 +1,502 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/avc_parser.h" + +#include +#include + +#include "base/logging.h" +#include "base/strings/stringprintf.h" +#include "media/base/audio_codecs.h" +#include "media/base/decoder_buffer.h" +#include "media/base/encryption_scheme.h" +#include "media/base/media_util.h" +#include "media/base/video_codecs.h" +#include "media/base/video_color_space.h" +#include "media/base/video_transformation.h" +#include "media/base/video_types.h" +#include "media/formats/mp4/aac.h" +#include "media/starboard/progressive/avc_access_unit.h" +#include "media/starboard/progressive/endian_util.h" +#include "media/starboard/progressive/rbsp_stream.h" + +namespace media { + +// what's the smallest meaningful AVC config we can parse? +static const int kAVCConfigMinSize = 8; +// lower five bits of first byte in SPS should be 7 +static const uint8_t kSPSNALType = 7; + +AVCParser::AVCParser(scoped_refptr reader, + MediaLog* media_log) + : ProgressiveParser(reader), + media_log_(media_log), + nal_header_size_(0), + video_prepend_size_(0) { + DCHECK(media_log); +} + +AVCParser::~AVCParser() {} + +bool AVCParser::Prepend(scoped_refptr au, + scoped_refptr buffer) { + // sanity-check inputs + if (!au || !buffer) { + NOTREACHED() << "bad input to Prepend()"; + return false; + } + if (au->GetType() == DemuxerStream::VIDEO) { + if (au->AddPrepend()) { + if (buffer->data_size() <= video_prepend_size_) { + NOTREACHED() << "empty/undersized buffer to Prepend()"; + return false; + } + memcpy(buffer->writable_data(), video_prepend_, video_prepend_size_); + } + } else if (au->GetType() == DemuxerStream::AUDIO) { + if (audio_prepend_.size() < 6) { + // valid ADTS header not available + return false; + } + if (buffer->data_size() <= audio_prepend_.size()) { + NOTREACHED() << "empty/undersized buffer to Prepend()"; + return false; + } + // audio, need to copy ADTS header and then add buffer size + uint32_t buffer_size = au->GetSize() + audio_prepend_.size(); + // we can't express an AU size larger than 13 bits, something's bad here. + if (buffer_size & 0xffffe000) { + return false; + } + std::vector audio_prepend(audio_prepend_); + // OR size into buffer, byte 3 gets 2 MSb of 13-bit size + audio_prepend[3] |= (uint8_t)((buffer_size & 0x00001800) >> 11); + // byte 4 gets bits 10-3 of size + audio_prepend[4] = (uint8_t)((buffer_size & 0x000007f8) >> 3); + // byte 5 gets bits 2-0 of size + audio_prepend[5] |= (uint8_t)((buffer_size & 0x00000007) << 5); + memcpy(buffer->writable_data(), audio_prepend.data(), audio_prepend.size()); + } else { + NOTREACHED() << "unsupported demuxer stream type."; + return false; + } + + return true; +} + +bool AVCParser::DownloadAndParseAVCConfigRecord(uint64_t offset, + uint32_t size) { + if (size == 0) { + return false; + } + std::vector record_buffer(size); + int bytes_read = reader_->BlockingRead(offset, size, &record_buffer[0]); + DCHECK_LE(size, static_cast(std::numeric_limits::max())); + if (bytes_read < static_cast(size)) { + return false; + } + // ok, successfully downloaded the record, parse it + return ParseAVCConfigRecord(&record_buffer[0], size); +} + +// static +bool AVCParser::ParseSPS(const uint8_t* sps, + size_t sps_size, + SPSRecord* record_out) { + DCHECK(sps) << "no sps provided"; + DCHECK(record_out) << "no output structure provided"; + // first byte is NAL type id, check that it is SPS + if ((*sps & 0x1f) != kSPSNALType) { + LOG(ERROR) << "bad NAL type on SPS"; + return false; + } + // convert SPS NALU to RBSP stream + RBSPStream sps_rbsp(sps + 1, sps_size - 1); + uint8_t profile_idc = 0; + if (!sps_rbsp.ReadByte(&profile_idc)) { + LOG(ERROR) << "failure reading profile_idc from sps RBSP"; + return false; + } + // skip 3 constraint flags, 5 reserved bits, and level_idc (16 bits) + sps_rbsp.SkipBytes(2); + // ReadUEV/ReadSEV require a value to be passed by reference but + // there are many times in which we ignore this value. + uint32_t disposable_uev = 0; + int32_t disposable_sev = 0; + // seq_parameter_set_id + sps_rbsp.ReadUEV(&disposable_uev); + // skip profile-specific encoding information if there + if (profile_idc == 100 || profile_idc == 103 || profile_idc == 110 || + profile_idc == 122 || profile_idc == 244 || profile_idc == 44 || + profile_idc == 83 || profile_idc == 86 || profile_idc == 118) { + uint32_t chroma_format_idc = 0; + if (!sps_rbsp.ReadUEV(&chroma_format_idc)) { + LOG(WARNING) << "failure reading chroma_format_idc from sps RBSP"; + return false; + } + if (chroma_format_idc == 3) { + // separate_color_plane_flag + sps_rbsp.SkipBits(1); + } + // bit_depth_luma_minus8 + sps_rbsp.ReadUEV(&disposable_uev); + // bit_depth_chroma_minus8 + sps_rbsp.ReadUEV(&disposable_uev); + // qpprime_y_zero_transform_bypass_flag + sps_rbsp.SkipBits(1); + // seq_scaling_matrix_present_flag + uint8_t seq_scaling_matrix_present_flag = 0; + if (!sps_rbsp.ReadBit(&seq_scaling_matrix_present_flag)) { + LOG(ERROR) + << "failure reading seq_scaling_matrix_present_flag from sps RBSP"; + return false; + } + if (seq_scaling_matrix_present_flag) { + // seq_scaling_list_present_flag[] + sps_rbsp.SkipBits(chroma_format_idc != 3 ? 8 : 12); + } + } + // log2_max_frame_num_minus4 + sps_rbsp.ReadUEV(&disposable_uev); + // pic_order_cnt_type + uint32_t pic_order_cnt_type = 0; + if (!sps_rbsp.ReadUEV(&pic_order_cnt_type)) { + LOG(ERROR) << "failure reading pic_order_cnt_type from sps RBSP"; + return false; + } + if (pic_order_cnt_type == 0) { + // log2_max_pic_order_cnt_lsb_minus4 + sps_rbsp.ReadUEV(&disposable_uev); + } else if (pic_order_cnt_type == 1) { + // delta_pic_order_always_zero_flag + sps_rbsp.SkipBits(1); + // offset_for_non_ref_pic + sps_rbsp.ReadSEV(&disposable_sev); + // offset_for_top_to_bottom_field + sps_rbsp.ReadSEV(&disposable_sev); + // num_ref_frames_in_pic_order_cnt_cycle + uint32_t num_ref_frames_in_pic_order_cnt_cycle = 0; + if (!sps_rbsp.ReadUEV(&num_ref_frames_in_pic_order_cnt_cycle)) { + LOG(ERROR) + << "failure reading num_ref_frames_in_pic_order_cnt_cycle from sps"; + return false; + } + for (uint32_t i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; ++i) { + sps_rbsp.ReadSEV(&disposable_sev); + } + } + // number of reference frames used to decode + uint32_t num_ref_frames = 0; + if (!sps_rbsp.ReadUEV(&num_ref_frames)) { + LOG(ERROR) << "failure reading number of ref frames from sps RBSP"; + return false; + } + // gaps_in_frame_num_value_allowed_flag + sps_rbsp.SkipBits(1); + // width is calculated from pic_width_in_mbs_minus1 + uint32_t pic_width_in_mbs_minus1 = 0; + if (!sps_rbsp.ReadUEV(&pic_width_in_mbs_minus1)) { + LOG(WARNING) << "failure reading image width from sps RBSP"; + return false; + } + // 16 pxs per macroblock + uint32_t width = (pic_width_in_mbs_minus1 + 1) * 16; + // pic_height_in_map_units_minus1 + uint32_t pic_height_in_map_units_minus1 = 0; + if (!sps_rbsp.ReadUEV(&pic_height_in_map_units_minus1)) { + LOG(ERROR) + << "failure reading pic_height_in_map_uints_minus1 from sps RBSP"; + return false; + } + uint8_t frame_mbs_only_flag = 0; + if (!sps_rbsp.ReadBit(&frame_mbs_only_flag)) { + LOG(ERROR) << "failure reading frame_mbs_only_flag from sps RBSP"; + return false; + } + uint32_t height = (2 - static_cast(frame_mbs_only_flag)) * + (pic_height_in_map_units_minus1 + 1) * 16; + if (!frame_mbs_only_flag) { + sps_rbsp.SkipBits(1); + } + // direct_8x8_inference_flag + sps_rbsp.SkipBits(1); + // frame cropping flag + uint8_t frame_cropping_flag = 0; + if (!sps_rbsp.ReadBit(&frame_cropping_flag)) { + LOG(ERROR) << "failure reading frame_cropping_flag from sps RBSP"; + return false; + } + // distance in pixels from the associated edge of the media: + // + // <---coded_size---width---------------------> + // + // +------------------------------------------+ ^ + // | ^ | | + // | | | | + // | crop_top | | + // | | | | + // | v | height + // | +---------+ | | + // |<--crop_left-->| visible | | | + // | | rect |<--crop_right-->| | + // | +---------+ | | + // | ^ | | + // | | | | + // | crop_bottom | | + // | | | | + // | v | | + // +------------------------------------------+ v + // + uint32_t crop_left = 0; + uint32_t crop_right = 0; + uint32_t crop_top = 0; + uint32_t crop_bottom = 0; + // cropping values are stored divided by two + if (frame_cropping_flag) { + if (!sps_rbsp.ReadUEV(&crop_left)) { + LOG(ERROR) << "failure reading crop_left from sps RBSP"; + return false; + } + if (!sps_rbsp.ReadUEV(&crop_right)) { + LOG(ERROR) << "failure reading crop_right from sps RBSP"; + return false; + } + if (!sps_rbsp.ReadUEV(&crop_top)) { + LOG(ERROR) << "failure reading crop_top from sps RBSP"; + return false; + } + if (!sps_rbsp.ReadUEV(&crop_bottom)) { + LOG(ERROR) << "failure reading crop_bottom from sps RBSP"; + return false; + } + crop_left *= 2; + crop_right *= 2; + crop_top *= 2; + crop_bottom *= 2; + } + // remainder of SPS are values we can safely ignore, everything + // checks out, write output structure + int visible_width = width - (crop_left + crop_right); + int visible_height = height - (crop_top + crop_bottom); + record_out->coded_size = gfx::Size(width, height), + record_out->visible_rect = + gfx::Rect(crop_left, crop_top, visible_width, visible_height), + record_out->natural_size = gfx::Size(visible_width, visible_height); + record_out->num_ref_frames = num_ref_frames; + return true; +} + +bool AVCParser::ParseAVCConfigRecord(uint8_t* buffer, uint32_t size) { + if (size < kAVCConfigMinSize) { + LOG(ERROR) << base::StringPrintf("AVC config record bad size: %d", size); + return false; + } + + // get the NALU header size + nal_header_size_ = (buffer[4] & 0x03) + 1; + // validate size, needs to be 1, 2 or 4 bytes only + if (nal_header_size_ != 4 && nal_header_size_ != 2 && nal_header_size_ != 1) { + return false; + } + // AVCConfigRecords contain a variable number of SPS NALU + // (Sequence Parameter Set) (Network Abstraction Layer Units) + // from which we can extract width, height, and cropping info. + // That means we need at least 1 SPS NALU in this stream for extraction. + uint8_t number_of_sps_nalus = buffer[5] & 0x1f; + if (number_of_sps_nalus == 0) { + LOG(WARNING) << "got AVCConfigRecord without any SPS NALUs!"; + return false; + } + // iterate through SPS NALUs finding one of valid size for our purposes + // (this should usually be the first one), but also advancing through + // the ConfigRecord until we encounter the PPS sets + bool have_valid_sps = false; + int record_offset = 6; + size_t usable_sps_size = 0; + int usable_sps_offset = 0; + for (uint8_t i = 0; i < number_of_sps_nalus; i++) { + // make sure we haven't run out of record for the 2-byte size record + DCHECK_LE(size, static_cast(std::numeric_limits::max())); + if (record_offset + 2 > static_cast(size)) { + LOG(WARNING) << "ran out of AVCConfig record while parsing SPS size."; + return false; + } + // extract 2-byte size of this SPS + size_t sps_size = + endian_util::load_uint16_big_endian(buffer + record_offset); + // advance past the 2-byte size record + record_offset += 2; + // see if we jumped over record size + if (record_offset + sps_size > size) { + LOG(WARNING) << "ran out of AVCConfig record while parsing SPS blocks."; + return false; + } + if (!have_valid_sps) { + have_valid_sps = true; + // save size and offset for later copying and parsing + usable_sps_size = sps_size; + usable_sps_offset = record_offset; + // continue to iterate through sps records to get to pps which follow + } + record_offset += sps_size; + } + if (!have_valid_sps) { + LOG(WARNING) + << "unable to parse a suitable SPS. Perhaps increase max size?"; + return false; + } + // we don't strictly require a PPS, so we're even willing to accept that + // this could be the end of the bytestream, but if not the next byte should + // define the number of PPS objects in the record. Not sure if + // specific decoders could decode something without a PPS prepend but this + // doesn't break demuxing so we'll let them complain if that isn't going + // to work for them :) + size_t usable_pps_size = 0; + size_t usable_pps_offset = 0; + bool have_valid_pps = false; + DCHECK_LE(size, static_cast(std::numeric_limits::max())); + if (record_offset + 1 < static_cast(size)) { + uint8_t number_of_pps_nalus = buffer[record_offset]; + record_offset++; + for (uint8_t i = 0; i < number_of_pps_nalus; i++) { + // make sure we don't run out of room for 2-byte size record + DCHECK_LE(size, + static_cast(std::numeric_limits::max())); + if (record_offset + 2 >= static_cast(size)) { + LOG(WARNING) << "ran out of AVCConfig record while parsing PPS size."; + return false; + } + // extract 2-byte size of this PPS + size_t pps_size = + endian_util::load_uint16_big_endian(buffer + record_offset); + record_offset += 2; + // see if there's actually room for this record in the buffer + if (record_offset + pps_size > size) { + LOG(WARNING) + << "ran out of AVCConfig record while scanning PPS blocks."; + return false; + } + if (!have_valid_pps) { + have_valid_pps = true; + usable_pps_size = pps_size; + usable_pps_offset = record_offset; + break; + } + } + } + // now we parse the valid SPS we extracted from byte stream earlier. + SPSRecord sps_record; + if (!ParseSPS(buffer + usable_sps_offset, usable_sps_size, &sps_record)) { + LOG(WARNING) << "error parsing SPS"; + return false; + } + // we can now initialize our video decoder config + video_config_.Initialize(VideoCodec::kH264, + // profile is ignored currently + VideoCodecProfile::H264PROFILE_MAIN, + VideoDecoderConfig::AlphaMode::kIsOpaque, + VideoColorSpace::REC709(), VideoTransformation(), + sps_record.coded_size, sps_record.visible_rect, + sps_record.natural_size, EmptyExtraData(), + EncryptionScheme::kUnencrypted); + + return BuildAnnexBPrepend(buffer + usable_sps_offset, usable_sps_size, + buffer + usable_pps_offset, usable_pps_size); +} + +bool AVCParser::BuildAnnexBPrepend(uint8_t* sps, + uint32_t sps_size, + uint8_t* pps, + uint32_t pps_size) { + // We will need to attach the sps and pps (if provided) to each keyframe + // video packet, with the AnnexB start code in front of each. Start with + // sps size and start code + video_prepend_size_ = sps_size + kAnnexBStartCodeSize; + if (pps_size > 0) { + // Add pps and pps start code size if needed. + video_prepend_size_ += pps_size + kAnnexBStartCodeSize; + } + // this should be a very rare case for typical videos + if (video_prepend_size_ > kAnnexBPrependMaxSize) { + NOTREACHED() << base::StringPrintf("Bad AnnexB prepend size: %d", + video_prepend_size_); + return false; + } + // start code for sps comes first + memcpy(video_prepend_, kAnnexBStartCode, kAnnexBStartCodeSize); + // followed by sps body + memcpy(video_prepend_ + kAnnexBStartCodeSize, sps, sps_size); + int prepend_offset = kAnnexBStartCodeSize + sps_size; + if (pps_size > 0) { + // pps start code comes next + memcpy(video_prepend_ + prepend_offset, kAnnexBStartCode, + kAnnexBStartCodeSize); + prepend_offset += kAnnexBStartCodeSize; + // followed by pps + memcpy(video_prepend_ + prepend_offset, pps, pps_size); + prepend_offset += pps_size; + } + + // make sure we haven't wandered off into memory somewhere + DCHECK_EQ(prepend_offset, video_prepend_size_); + return true; +} + +void AVCParser::ParseAudioSpecificConfig(uint8_t b0, uint8_t b1) { + mp4::AAC aac; + std::vector aac_config(2); + + aac_config[0] = b0; + aac_config[1] = b1; + audio_prepend_.clear(); + + int adts_header_size; + if (!aac.Parse(aac_config, media_log_) || + !aac.ConvertEsdsToADTS(&audio_prepend_, &adts_header_size)) { + LOG(WARNING) << "Error in parsing AudioSpecificConfig."; + return; + } + + // Clear the length, it is 13 bits and stored as ******LL LLLLLLLL LLL***** + // in bytes 3 to 5. + audio_prepend_[3] &= 0xfc; + audio_prepend_[4] = 0; + audio_prepend_[5] &= 0x1f; + + const bool kSbrInMimetype = false; + audio_config_.Initialize( + AudioCodec::kAAC, kSampleFormatS16, aac.GetChannelLayout(kSbrInMimetype), + aac.GetOutputSamplesPerSecond(kSbrInMimetype), aac.codec_specific_data(), + EncryptionScheme::kUnencrypted, base::TimeDelta(), 0); + audio_config_.set_aac_extra_data(aac.codec_specific_data()); +} + +size_t AVCParser::CalculatePrependSize(DemuxerStream::Type type, + bool is_keyframe) { + size_t prepend_size = 0; + if (type == DemuxerStream::VIDEO) { + bool needs_prepend = is_keyframe; + if (needs_prepend) { + prepend_size = video_prepend_size_; + } + } else if (type == DemuxerStream::AUDIO) { + prepend_size = audio_prepend_.size(); + } else { + NOTREACHED() << "unsupported stream type"; + } + return prepend_size; +} + +} // namespace media diff --git a/media/starboard/progressive/avc_parser.h b/media/starboard/progressive/avc_parser.h new file mode 100644 index 000000000000..a23cfc41b682 --- /dev/null +++ b/media/starboard/progressive/avc_parser.h @@ -0,0 +1,83 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_AVC_PARSER_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_AVC_PARSER_H_ + +#include + +#include "media/base/media_log.h" +#include "media/starboard/progressive/progressive_parser.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +namespace media { + +// Typical size of an annexB prepend will be around 60 bytes. We make more room +// to ensure that only a very few videos will fail to play for lack of room +// in the prepend. +static const int kAnnexBPrependMaxSize = 1024; + +// while not an actual parser, provides shared functionality to both the +// mp4 and flv parsers which derive from it. Implements part of +// ProgressiveParser while leaving the rest for its children. +class AVCParser : public ProgressiveParser { + public: + explicit AVCParser(scoped_refptr reader, + MediaLog* media_log); + virtual ~AVCParser(); + + struct SPSRecord { + gfx::Size coded_size; + gfx::Rect visible_rect; + gfx::Size natural_size; + uint32_t num_ref_frames; + }; + static bool ParseSPS(const uint8_t* sps, + size_t sps_size, + SPSRecord* record_out); + + // GetNextAU we must pass on to FLV or MP4 children. + virtual scoped_refptr GetNextAU(DemuxerStream::Type type) = 0; + // Prepends are common to all AVC/AAC containers so we can do this one here. + bool Prepend(scoped_refptr au, + scoped_refptr buffer) override; + + protected: + virtual bool DownloadAndParseAVCConfigRecord(uint64_t offset, uint32_t size); + virtual bool ParseAVCConfigRecord(uint8_t* buffer, uint32_t size); + // pps_size can be 0. Returns false on unable to construct. + virtual bool BuildAnnexBPrepend(uint8_t* sps, + uint32_t sps_size, + uint8_t* pps, + uint32_t pps_size); + virtual void ParseAudioSpecificConfig(uint8_t b0, uint8_t b1); + virtual size_t CalculatePrependSize(DemuxerStream::Type type, + bool is_keyframe); + + MediaLog* media_log_; + uint8_t nal_header_size_; + // audio frames have a fixed-size small prepend that we attach to every + // audio buffer created by DownloadBuffer() + std::vector audio_prepend_; + // video frames have a variable-size prepend that we limit to a reasonable + // upper bound. We only need to attach it to keyframes, however, the rest + // of the frames need only an AnnexB start code. + uint8_t video_prepend_[kAnnexBPrependMaxSize]; + uint32_t video_prepend_size_; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_AVC_PARSER_H_ diff --git a/media/starboard/progressive/data_source_reader.cc b/media/starboard/progressive/data_source_reader.cc new file mode 100644 index 000000000000..9ea2d028dc3f --- /dev/null +++ b/media/starboard/progressive/data_source_reader.cc @@ -0,0 +1,120 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/data_source_reader.h" + +#include "starboard/types.h" + +namespace media { + +const int DataSourceReader::kReadError = DataSource::kReadError; + +DataSourceReader::DataSourceReader() + : data_source_(NULL), + blocking_read_event_(base::WaitableEvent::ResetPolicy::AUTOMATIC, + base::WaitableEvent::InitialState::NOT_SIGNALED), + file_size_(-1), + read_has_failed_(false), + last_bytes_read_(0) {} + +DataSourceReader::~DataSourceReader() {} + +void DataSourceReader::SetDataSource(DataSource* data_source) { + DCHECK(data_source); + data_source_ = data_source; +} + +// currently only single-threaded reads supported +int DataSourceReader::BlockingRead(int64_t position, int size, uint8_t* data) { + // read failures are unrecoverable, all subsequent reads will also fail + if (read_has_failed_) { + return kReadError; + } + + // check bounds of read at or past EOF + if (file_size_ >= 0 && position >= file_size_) { + return 0; + } + + int total_bytes_read = 0; + while (size > 0 && !read_has_failed_) { + { + base::AutoLock auto_lock(lock_); + if (!data_source_) { + break; + } + data_source_->Read( + position, size, data, + base::BindRepeating(&DataSourceReader::BlockingReadCompleted, this)); + } + + // wait for callback on read completion + blocking_read_event_.Wait(); + + if (last_bytes_read_ == DataSource::kReadError) { + // make all future reads fail + read_has_failed_ = true; + return kReadError; + } + + DCHECK_LE(last_bytes_read_, size); + if (last_bytes_read_ > size) { + // make all future reads fail + read_has_failed_ = true; + return kReadError; + } + + // Avoid entering an endless loop here. + if (last_bytes_read_ == 0) { + break; + } + + total_bytes_read += last_bytes_read_; + position += last_bytes_read_; + size -= last_bytes_read_; + data += last_bytes_read_; + } + + if (read_has_failed_) { + return kReadError; + } + return total_bytes_read; +} + +void DataSourceReader::Stop() { + if (data_source_) { + data_source_->Stop(); + + base::AutoLock auto_lock(lock_); + data_source_ = NULL; + } +} + +void DataSourceReader::BlockingReadCompleted(int bytes_read) { + last_bytes_read_ = bytes_read; + // wake up blocked thread + blocking_read_event_.Signal(); +} + +int64_t DataSourceReader::FileSize() { + if (file_size_ == -1) { + base::AutoLock auto_lock(lock_); + if (data_source_ && !data_source_->GetSize(&file_size_)) { + file_size_ = -1; + } + } + return file_size_; +} + +} // namespace media diff --git a/media/starboard/progressive/data_source_reader.h b/media/starboard/progressive/data_source_reader.h new file mode 100644 index 000000000000..0ad73a037032 --- /dev/null +++ b/media/starboard/progressive/data_source_reader.h @@ -0,0 +1,69 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_DATA_SOURCE_READER_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_DATA_SOURCE_READER_H_ + +#include "base/functional/bind.h" +#include "base/functional/callback.h" +#include "base/memory/ref_counted.h" +#include "base/synchronization/lock.h" +#include "base/synchronization/waitable_event.h" +#include "base/task/sequenced_task_runner.h" +#include "media/base/data_source.h" + +namespace media { + +// Allows sharing of a DataSource object between multiple objects on a single +// thread, and exposes a simple BlockingRead() method to block the thread until +// data is available or error. To avoid circular smart pointer references this +// object is also the sole owner of a pointer to DataSource. If we want to add +// asynchronous reading to this object it will need its own thread and a +// callback queue. +class DataSourceReader : public base::RefCountedThreadSafe { + public: + static const int kReadError; + + DataSourceReader(); + virtual void SetDataSource(DataSource* data_source); + + // Block the calling thread's task runner until read is complete. + // returns number of bytes read or kReadError on error. + // Currently only single-threaded support. + virtual int BlockingRead(int64_t position, int size, uint8_t* data); + + // returns size of file in bytes, or -1 if file size not known. If error will + // retry getting file size on subsequent calls to FileSize(). + virtual int64_t FileSize(); + + // abort any pending read, then stop the data source + virtual void Stop(); + + protected: + friend class base::RefCountedThreadSafe; + virtual ~DataSourceReader(); + // blocking read callback + virtual void BlockingReadCompleted(int bytes_read); + + base::Lock lock_; + DataSource* data_source_; + base::WaitableEvent blocking_read_event_; + int64_t file_size_; + bool read_has_failed_; + int last_bytes_read_; // protected implicitly by blocking_read_event_ +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_DATA_SOURCE_READER_H_ diff --git a/media/starboard/progressive/demuxer_extension_wrapper.cc b/media/starboard/progressive/demuxer_extension_wrapper.cc new file mode 100644 index 000000000000..7cb796c762ae --- /dev/null +++ b/media/starboard/progressive/demuxer_extension_wrapper.cc @@ -0,0 +1,1111 @@ +// Copyright 2022 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/demuxer_extension_wrapper.h" + +#include +#include +#include +#include + +#include "base/task/bind_post_task.h" +#include "base/time/time.h" +#include "media/base/audio_codecs.h" +#include "media/base/encryption_scheme.h" +#include "media/base/sample_format.h" +#include "media/base/video_types.h" +#include "media/filters/h264_to_annex_b_bitstream_converter.h" +#include "media/formats/mp4/box_definitions.h" +#include "media/starboard/starboard_utils.h" +#include "starboard/extension/demuxer.h" +#include "starboard/system.h" +#include "ui/gfx/color_space.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +namespace media { + +// Used to convert a lambda to a pure C function. +// |user_data| is a callback of type T, which takes a U*. +template +static void CallCB(U* u, void* user_data) { + (*static_cast(user_data))(u); +} + +// Converts AVCC h.264 frames to Annex B. This is necessary because the decoder +// expects packets in Annex B format. +class DemuxerExtensionWrapper::H264AnnexBConverter { + public: + // Creates an H264AnnexBConverter from the MP4 file's header data. + static std::unique_ptr Create(const uint8_t* extra_data, + size_t extra_data_size) { + if (!extra_data || extra_data_size == 0) { + LOG(ERROR) << "Invalid inputs to H264AnnexBConverter::Create."; + return nullptr; + } + mp4::AVCDecoderConfigurationRecord config; + std::unique_ptr converter( + new H264ToAnnexBBitstreamConverter); + if (!converter->ParseConfiguration( + extra_data, static_cast(extra_data_size), &config)) { + LOG(ERROR) << "Could not parse AVCC config."; + return nullptr; + } + return std::unique_ptr( + new H264AnnexBConverter(std::move(config), std::move(converter))); + } + + // Disallow copy and assign. + H264AnnexBConverter(const H264AnnexBConverter&) = delete; + H264AnnexBConverter& operator=(const H264AnnexBConverter&) = delete; + + ~H264AnnexBConverter() = default; + + // Attempts to convert the data in |data| from AVCC to AnnexB format, + // returning the data as a DecoderBuffer. Upon failure, the data will be + // returned unmodified in the DecoderBuffer. + scoped_refptr Convert(const uint8_t* data, size_t data_size) { + const auto* const config = config_.has_value() ? &*config_ : nullptr; + + std::vector rewritten( + converter_->CalculateNeededOutputBufferSize(data, data_size, config)); + + uint32_t rewritten_size = rewritten.size(); + if (rewritten.empty() || + !converter_->ConvertNalUnitStreamToByteStream( + data, data_size, config, rewritten.data(), &rewritten_size)) { + // TODO(b/231994311): Add the buffer's side_data here, for HDR10+ support. + return DecoderBuffer::CopyFrom(data, data_size); + } else { + // The data was successfully rewritten. + + // The SPS and PPS NALUs -- generated from the config -- should only be + // sent with the first real NALU. + config_ = absl::nullopt; + + // TODO(b/231994311): Add the buffer's side_data here, for HDR10+ support. + return DecoderBuffer::CopyFrom(rewritten.data(), rewritten.size()); + } + } + + private: + explicit H264AnnexBConverter( + mp4::AVCDecoderConfigurationRecord config, + std::unique_ptr converter) + : config_(std::move(config)), converter_(std::move(converter)) {} + + // This config data is only sent with the first NALU (as SPS and PPS NALUs). + absl::optional config_; + std::unique_ptr converter_; +}; + +DemuxerExtensionStream::DemuxerExtensionStream( + CobaltExtensionDemuxer* demuxer, + scoped_refptr task_runner, + CobaltExtensionDemuxerVideoDecoderConfig config) + : demuxer_(demuxer), task_runner_(std::move(task_runner)) { + CHECK(demuxer_); + CHECK(task_runner_); + std::vector extra_data; + if (config.extra_data_size > 0 && config.extra_data != nullptr) { + extra_data.assign(config.extra_data, + config.extra_data + config.extra_data_size); + } + + video_config_.emplace( + static_cast(config.codec), + static_cast(config.profile), + static_cast(config.alpha_mode), + VideoColorSpace( + config.color_space_primaries, config.color_space_transfer, + config.color_space_matrix, + static_cast(config.color_space_range_id)), + VideoTransformation(), gfx::Size(config.coded_width, config.coded_height), + gfx::Rect(config.visible_rect_x, config.visible_rect_y, + config.visible_rect_width, config.visible_rect_height), + gfx::Size(config.natural_width, config.natural_height), extra_data, + static_cast(config.encryption_scheme)); + + LOG_IF(ERROR, !video_config_->IsValidConfig()) + << "Video config is not valid!"; +} + +DemuxerExtensionStream::DemuxerExtensionStream( + CobaltExtensionDemuxer* demuxer, + scoped_refptr task_runner, + CobaltExtensionDemuxerAudioDecoderConfig config) + : demuxer_(demuxer), task_runner_(std::move(task_runner)) { + CHECK(demuxer_); + CHECK(task_runner_); + std::vector extra_data; + if (config.extra_data_size > 0 && config.extra_data != nullptr) { + extra_data.assign(config.extra_data, + config.extra_data + config.extra_data_size); + } + + audio_config_.emplace( + static_cast(config.codec), + static_cast(config.sample_format), + static_cast(config.channel_layout), + config.samples_per_second, extra_data, + static_cast(config.encryption_scheme)); + + LOG_IF(ERROR, !audio_config_->IsValidConfig()) + << "Audio config is not valid!"; +} + +void DemuxerExtensionStream::Read(uint32_t count, ReadCB read_cb) { + DCHECK(!read_cb.is_null()); + base::AutoLock auto_lock(lock_); + if (stopped_) { + LOG(INFO) << "Already stopped."; + std::vector> buffers; + buffers.push_back(DecoderBuffer::CreateEOSBuffer()); + std::move(read_cb).Run(DemuxerStream::kOk, buffers); + return; + } + + // Buffers are only queued when there are no pending reads. + CHECK(buffer_queue_.empty() || read_queue_.empty()); + + if (buffer_queue_.empty()) { + read_queue_.push_back(std::move(read_cb)); + return; + } + + // We already have a buffer queued. Send the oldest buffer back. + scoped_refptr buffer = buffer_queue_.front(); + if (!buffer->end_of_stream()) { + // Do not pop EOS buffers, so that subsequent read requests also get EOS. + total_buffer_size_ -= buffer->data_size(); + buffer_queue_.pop_front(); + } + + std::vector> buffers; + buffers.push_back(std::move(buffer)); + std::move(read_cb).Run(DemuxerStream::kOk, buffers); +} + +AudioDecoderConfig DemuxerExtensionStream::audio_decoder_config() { + DCHECK(audio_config_.has_value()); + return *audio_config_; +} + +VideoDecoderConfig DemuxerExtensionStream::video_decoder_config() { + DCHECK(video_config_.has_value()); + return *video_config_; +} + +DemuxerStream::Type DemuxerExtensionStream::type() const { + const uint8_t is_audio = static_cast(audio_config_.has_value()); + const uint8_t is_video = static_cast(video_config_.has_value()); + DCHECK((is_audio ^ is_video) == 1); + return is_audio ? Type::AUDIO : Type::VIDEO; +} + +Ranges DemuxerExtensionStream::GetBufferedRanges() { + return buffered_ranges_; +} + +void DemuxerExtensionStream::EnqueueBuffer( + scoped_refptr buffer) { + base::AutoLock auto_lock(lock_); + + if (stopped_) { + // It is possible due to pipelining -- both downstream and within the + // demuxer -- that several pipelined reads will be enqueuing packets on a + // stopped stream. These will be dropped. + LOG(WARNING) << "attempted to enqueue packet on stopped stream"; + return; + } + + if (buffer->end_of_stream()) { + LOG(INFO) << "Received EOS"; + } else if (buffer->timestamp() != kNoTimestamp) { + if (last_buffer_timestamp_ != kNoTimestamp && + last_buffer_timestamp_ < buffer->timestamp()) { + buffered_ranges_.Add(last_buffer_timestamp_, buffer->timestamp()); + } + last_buffer_timestamp_ = buffer->timestamp(); + } else { + LOG(WARNING) << "Bad timestamp info on enqueued buffer."; + } + + if (read_queue_.empty()) { + buffer_queue_.push_back(buffer); + if (!buffer->end_of_stream()) { + total_buffer_size_ += buffer->data_size(); + } + return; + } + + // A pending read implies that the buffer queue was empty; otherwise it should + // never have been added to the read queue in the first place. + CHECK_EQ(buffer_queue_.size(), 0); + ReadCB read_cb(std::move(read_queue_.front())); + read_queue_.pop_front(); + std::vector> buffers; + buffers.push_back(std::move(buffer)); + std::move(read_cb).Run(DemuxerStream::kOk, buffers); +} + +void DemuxerExtensionStream::FlushBuffers() { + base::AutoLock auto_lock(lock_); + buffer_queue_.clear(); + total_buffer_size_ = 0; + last_buffer_timestamp_ = kNoTimestamp; +} + +void DemuxerExtensionStream::Stop() { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + + base::AutoLock auto_lock(lock_); + buffer_queue_.clear(); + total_buffer_size_ = 0; + last_buffer_timestamp_ = kNoTimestamp; + // Fulfill any pending callbacks with EOS buffers set to end timestamp. + for (auto& read_cb : read_queue_) { + std::vector> buffers; + buffers.push_back(DecoderBuffer::CreateEOSBuffer()); + std::move(read_cb).Run(DemuxerStream::kOk, buffers); + } + read_queue_.clear(); + stopped_ = true; +} + +base::TimeDelta DemuxerExtensionStream::GetLastBufferTimestamp() const { + base::AutoLock auto_lock(lock_); + return last_buffer_timestamp_; +} + +size_t DemuxerExtensionStream::GetTotalBufferSize() const { + base::AutoLock auto_lock(lock_); + return total_buffer_size_; +} + +PositionalDataSource::PositionalDataSource( + scoped_refptr reader) + : reader_(std::move(reader)), position_(0) { + CHECK(reader_); +} + +PositionalDataSource::~PositionalDataSource() = default; + +void PositionalDataSource::Stop() { + reader_->Stop(); +} + +int PositionalDataSource::BlockingRead(uint8_t* data, int bytes_requested) { + const int bytes_read = + reader_->BlockingRead(position_, bytes_requested, data); + if (bytes_read != DataSourceReader::kReadError) { + position_ += bytes_read; + } + return bytes_read; +} + +void PositionalDataSource::SeekTo(int position) { + position_ = position; +} + +int64_t PositionalDataSource::GetPosition() const { + return position_; +} + +int64_t PositionalDataSource::GetSize() { + return reader_->FileSize(); +} + +// Functions for converting a PositionalDataSource to +// CobaltExtensionDemuxerDataSource. +static int CobaltExtensionDemuxerDataSource_BlockingReadRead( + uint8_t* data, + int bytes_requested, + void* user_data) { + return static_cast(user_data)->BlockingRead( + data, bytes_requested); +} + +static void CobaltExtensionDemuxerDataSource_SeekTo(int position, + void* user_data) { + static_cast(user_data)->SeekTo(position); +} + +static int64_t CobaltExtensionDemuxerDataSource_GetPosition(void* user_data) { + return static_cast(user_data)->GetPosition(); +} + +static int64_t CobaltExtensionDemuxerDataSource_GetSize(void* user_data) { + return static_cast(user_data)->GetSize(); +} + +std::unique_ptr DemuxerExtensionWrapper::Create( + DataSource* data_source, + scoped_refptr task_runner, + const CobaltExtensionDemuxerApi* demuxer_api) { + if (demuxer_api == nullptr) { + // Attempt to use the Cobalt extension. + demuxer_api = static_cast( + SbSystemGetExtension(kCobaltExtensionDemuxerApi)); + if (!demuxer_api || + strcmp(demuxer_api->name, kCobaltExtensionDemuxerApi) != 0) { + return nullptr; + } + } + + DCHECK(demuxer_api); + if (demuxer_api->version < 1) { + LOG(ERROR) << "Demuxer API version is too low: " << demuxer_api->version; + return nullptr; + } + + if (!data_source || !task_runner) { + LOG(ERROR) << "data_source and task_runner cannot be null."; + return nullptr; + } + + scoped_refptr reader = new DataSourceReader; + reader->SetDataSource(data_source); + + std::unique_ptr positional_data_source( + new PositionalDataSource(std::move(reader))); + + std::unique_ptr c_data_source( + new CobaltExtensionDemuxerDataSource{ + /*BlockingRead=*/&CobaltExtensionDemuxerDataSource_BlockingReadRead, + /*SeekTo=*/&CobaltExtensionDemuxerDataSource_SeekTo, + /*GetPosition=*/&CobaltExtensionDemuxerDataSource_GetPosition, + /*GetSize=*/&CobaltExtensionDemuxerDataSource_GetSize, + /*is_streaming=*/false, + /*user_data=*/positional_data_source.get()}); + + // TODO(b/231632632): Populate these vectors. + std::vector supported_audio_codecs; + std::vector supported_video_codecs; + + CobaltExtensionDemuxer* demuxer = demuxer_api->CreateDemuxer( + c_data_source.get(), supported_audio_codecs.data(), + supported_audio_codecs.size(), supported_video_codecs.data(), + supported_video_codecs.size()); + + if (!demuxer) { + LOG(ERROR) << "Failed to create a CobaltExtensionDemuxer."; + return nullptr; + } + + return std::unique_ptr(new DemuxerExtensionWrapper( + demuxer_api, demuxer, std::move(positional_data_source), + std::move(c_data_source), std::move(task_runner))); +} + +DemuxerExtensionWrapper::DemuxerExtensionWrapper( + const CobaltExtensionDemuxerApi* demuxer_api, + CobaltExtensionDemuxer* demuxer, + std::unique_ptr data_source, + std::unique_ptr c_data_source, + scoped_refptr task_runner) + : demuxer_api_(demuxer_api), + impl_(demuxer), + data_source_(std::move(data_source)), + c_data_source_(std::move(c_data_source)), + blocking_thread_("DemuxerExtensionWrapperBlockingThread"), + task_runner_(std::move(task_runner)) { + CHECK(demuxer_api_); + CHECK(impl_); + CHECK(data_source_); + CHECK(c_data_source_); + CHECK(task_runner_); +} + +DemuxerExtensionWrapper::~DemuxerExtensionWrapper() { + if (impl_) { + demuxer_api_->DestroyDemuxer(impl_); + } + // Explicitly stop |blocking_thread_| to ensure that it stops before the + // destruction of any other members. + blocking_thread_.Stop(); +} + +std::vector DemuxerExtensionWrapper::GetAllStreams() { + std::vector streams; + if (audio_stream_.has_value()) { + streams.push_back(&*audio_stream_); + } + if (video_stream_.has_value()) { + streams.push_back(&*video_stream_); + } + return streams; +} + +std::string DemuxerExtensionWrapper::GetDisplayName() const { + return "DemuxerExtensionWrapper"; +} +void DemuxerExtensionWrapper::Initialize(DemuxerHost* host, + PipelineStatusCallback status_cb) { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + host_ = host; + + // Start the blocking thread and have it download and parse the media config. + if (!blocking_thread_.Start()) { + LOG(ERROR) << "Unable to start blocking thread"; + std::move(status_cb).Run(DEMUXER_ERROR_COULD_NOT_PARSE); + return; + } + + // |status_cb| cannot be called until this function returns, so we post a task + // here. + blocking_thread_.task_runner()->PostTaskAndReplyWithResult( + FROM_HERE, base::BindOnce(impl_->Initialize, impl_->user_data), + base::BindOnce(&DemuxerExtensionWrapper::OnInitializeDone, + base::Unretained(this), std::move(status_cb))); +} + +void DemuxerExtensionWrapper::OnInitializeDone( + PipelineStatusCallback status_cb, + CobaltExtensionDemuxerStatus status) { + if (status == kCobaltExtensionDemuxerOk) { + // Set up the stream(s) on this end. + CobaltExtensionDemuxerAudioDecoderConfig audio_config = {}; + if (impl_->GetAudioConfig(&audio_config, impl_->user_data)) { + if (audio_config.encryption_scheme != + kCobaltExtensionDemuxerEncryptionSchemeUnencrypted) { + // TODO(b/232957482): Determine whether we need to handle this case. + LOG(ERROR) + << "Encrypted audio is not supported for progressive playback."; + std::move(status_cb).Run(DEMUXER_ERROR_NO_SUPPORTED_STREAMS); + return; + } + audio_stream_.emplace(impl_, task_runner_, std::move(audio_config)); + } + CobaltExtensionDemuxerVideoDecoderConfig video_config = {}; + if (impl_->GetVideoConfig(&video_config, impl_->user_data)) { + if (video_config.encryption_scheme != + kCobaltExtensionDemuxerEncryptionSchemeUnencrypted) { + // TODO(b/232957482): Determine whether we need to handle this case. + LOG(ERROR) + << "Encrypted video is not supported for progressive playback."; + std::move(status_cb).Run(DEMUXER_ERROR_NO_SUPPORTED_STREAMS); + return; + } + if (video_config.extra_data && video_config.extra_data_size > 0 && + video_config.codec == kCobaltExtensionDemuxerCodecH264) { + // This is probably an AVCC stream. We'll need to convert each packet + // from AVCC to AnnexB, so we create the converter based on the "extra + // data". This extra data will be passed in the form of SPS and PPS NALU + // packets in the AnnexB stream. + h264_converter_ = H264AnnexBConverter::Create( + video_config.extra_data, video_config.extra_data_size); + video_config.extra_data = nullptr; + video_config.extra_data_size = 0; + } + video_stream_.emplace(impl_, task_runner_, std::move(video_config)); + } + + if (!audio_stream_.has_value() && !video_stream_.has_value()) { + // Even though initialization seems to have succeeded, something is wrong + // if there are no streams. + LOG(ERROR) << "No streams are present"; + std::move(status_cb).Run(DEMUXER_ERROR_NO_SUPPORTED_STREAMS); + return; + } + + host_->SetDuration( + base::Microseconds(impl_->GetDuration(impl_->user_data))); + + // Begin downloading data. + Request(audio_stream_.has_value() ? DemuxerStream::AUDIO + : DemuxerStream::VIDEO); + } else { + LOG(ERROR) << "Initialization failed with status " << status; + } + std::move(status_cb).Run(static_cast(status)); +} + +void DemuxerExtensionWrapper::AbortPendingReads() {} + +void DemuxerExtensionWrapper::StartWaitingForSeek(base::TimeDelta seek_time) {} + +void DemuxerExtensionWrapper::CancelPendingSeek(base::TimeDelta seek_time) {} + +void DemuxerExtensionWrapper::Seek(base::TimeDelta time, + PipelineStatusCallback status_cb) { + // It's safe to use base::Unretained here because blocking_thread_ will be + // stopped in this class's destructor. + blocking_thread_.task_runner()->PostTask( + FROM_HERE, + base::BindOnce(&DemuxerExtensionWrapper::SeekTask, base::Unretained(this), + time, BindPostTaskToCurrentDefault(std::move(status_cb)))); +} + +// TODO(b/232984963): Determine whether it's OK to have reads and seeks on the +// same thread. +void DemuxerExtensionWrapper::SeekTask(base::TimeDelta time, + PipelineStatusCallback status_cb) { + CHECK(blocking_thread_.task_runner()->RunsTasksInCurrentSequence()); + + // clear any enqueued buffers on demuxer streams + if (video_stream_.has_value()) { + video_stream_->FlushBuffers(); + } + if (audio_stream_.has_value()) { + audio_stream_->FlushBuffers(); + } + + const CobaltExtensionDemuxerStatus status = + impl_->Seek(time.InMicroseconds(), impl_->user_data); + + if (status != kCobaltExtensionDemuxerOk) { + LOG(ERROR) << "Seek failed with status " << status; + std::move(status_cb).Run(PIPELINE_ERROR_READ); + return; + } + + // If all streams had finished downloading, we need to restart the request. + const bool issue_new_request = + (!video_stream_.has_value() || video_reached_eos_) && + (!audio_stream_.has_value() || audio_reached_eos_); + audio_reached_eos_ = false; + video_reached_eos_ = false; + flushing_ = true; + std::move(status_cb).Run(PIPELINE_OK); + + if (issue_new_request) { + IssueNextRequest(); + } +} + +Ranges DemuxerExtensionWrapper::GetBufferedRanges() { + DCHECK(audio_stream_.has_value() || video_stream_.has_value()); + + if (!audio_stream_.has_value()) { + return video_stream_->GetBufferedRanges(); + } + if (!video_stream_.has_value()) { + return audio_stream_->GetBufferedRanges(); + } + return video_stream_->GetBufferedRanges().IntersectionWith( + audio_stream_->GetBufferedRanges()); +} + +void DemuxerExtensionWrapper::Stop() { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + { + base::AutoLock lock(lock_for_stopped_); + stopped_ = true; + } + data_source_->Stop(); +} + +base::TimeDelta DemuxerExtensionWrapper::GetStartTime() const { + return base::Microseconds(impl_->GetStartTime(impl_->user_data)); +} + +base::Time DemuxerExtensionWrapper::GetTimelineOffset() const { + const base::TimeDelta reported_time = + base::Microseconds(impl_->GetTimelineOffset(impl_->user_data)); + return reported_time.is_zero() + ? base::Time() + : base::Time::FromDeltaSinceWindowsEpoch(reported_time); +} + +int64_t DemuxerExtensionWrapper::GetMemoryUsage() const { + NOTREACHED(); + return 0; +} + +void DemuxerExtensionWrapper::OnEnabledAudioTracksChanged( + const std::vector& track_ids, + base::TimeDelta curr_time, + TrackChangeCB change_completed_cb) { + NOTREACHED(); +} + +void DemuxerExtensionWrapper::OnSelectedVideoTrackChanged( + const std::vector& track_ids, + base::TimeDelta curr_time, + TrackChangeCB change_completed_cb) { + NOTREACHED(); +} + +void DemuxerExtensionWrapper::Request(DemuxerStream::Type type) { + static const auto kRequestDelay = base::Milliseconds(100); + + if (type == DemuxerStream::AUDIO) { + DCHECK(audio_stream_.has_value()); + } else { + DCHECK(video_stream_.has_value()); + } + + if (!blocking_thread_.task_runner()->RunsTasksInCurrentSequence()) { + blocking_thread_.task_runner()->PostTask( + FROM_HERE, base::BindRepeating(&DemuxerExtensionWrapper::Request, + base::Unretained(this), type)); + return; + } + + if (HasStopped()) { + return; + } + + const size_t total_buffer_size = + (audio_stream_.has_value() ? audio_stream_->GetTotalBufferSize() : 0) + + (video_stream_.has_value() ? video_stream_->GetTotalBufferSize() : 0); + + int progressive_budget = 0; + if (video_stream_.has_value()) { + const VideoDecoderConfig video_config = + video_stream_->video_decoder_config(); + // Only sdr video is supported in progressive mode. + // TODO(b/231994311): Figure out how to set this value properly. + constexpr int kBitDepth = 8; + progressive_budget = SbMediaGetProgressiveBufferBudget( + MediaVideoCodecToSbMediaVideoCodec(video_config.codec()), + video_config.visible_rect().size().width(), + video_config.visible_rect().size().height(), kBitDepth); + } else { + progressive_budget = SbMediaGetAudioBufferBudget(); + } + + if (total_buffer_size >= progressive_budget) { + // Retry after a delay. + blocking_thread_.task_runner()->PostDelayedTask( + FROM_HERE, + base::BindRepeating(&DemuxerExtensionWrapper::Request, + base::Unretained(this), type), + kRequestDelay); + return; + } + + scoped_refptr decoder_buffer; + bool called_cb = false; + auto read_cb = [this, type, &decoder_buffer, + &called_cb](CobaltExtensionDemuxerBuffer* buffer) { + called_cb = true; + if (!buffer) { + return; + } + + if (buffer->end_of_stream) { + decoder_buffer = DecoderBuffer::CreateEOSBuffer(); + return; + } + + if (h264_converter_ && type == DemuxerExtensionStream::VIDEO) { + // This converts from AVCC to AnnexB format for h.264 video. + decoder_buffer = + h264_converter_->Convert(buffer->data, buffer->data_size); + } else { + // TODO(b/231994311): Add the buffer's side_data here, for HDR10+ support. + decoder_buffer = DecoderBuffer::CopyFrom(buffer->data, buffer->data_size); + } + + decoder_buffer->set_timestamp(base::Microseconds(buffer->pts)); + decoder_buffer->set_duration(base::Microseconds(buffer->duration)); + decoder_buffer->set_is_key_frame(buffer->is_keyframe); + }; + impl_->Read(static_cast(type), + &CallCB, + &read_cb, impl_->user_data); + + if (!called_cb) { + LOG(ERROR) + << "Demuxer extension implementation did not call the read callback."; + host_->OnDemuxerError(PIPELINE_ERROR_READ); + return; + } + if (!decoder_buffer) { + LOG(ERROR) << "Received a null buffer from the demuxer."; + host_->OnDemuxerError(PIPELINE_ERROR_READ); + return; + } + + auto& stream = + (type == DemuxerStream::AUDIO) ? *audio_stream_ : *video_stream_; + bool& eos_status = + (type == DemuxerStream::AUDIO) ? audio_reached_eos_ : video_reached_eos_; + + eos_status = decoder_buffer->end_of_stream(); + stream.EnqueueBuffer(std::move(decoder_buffer)); + if (!eos_status) { + host_->OnBufferedTimeRangesChanged(GetBufferedRanges()); + } + + // If we reach this point, enqueueing the buffer was successful. + IssueNextRequest(); + return; +} + +void DemuxerExtensionWrapper::IssueNextRequest() { + { + base::AutoLock lock(lock_for_stopped_); + if (stopped_) { + LOG(INFO) << "Already stopped; request loop is stopping."; + return; + } + } + + DemuxerStream::Type type = DemuxerStream::UNKNOWN; + if (audio_reached_eos_ || video_reached_eos_) { + // If we have eos in one or both buffers, the decision is easy. + if ((audio_reached_eos_ && video_reached_eos_) || + (audio_reached_eos_ && !video_stream_.has_value()) || + (video_reached_eos_ && !audio_stream_.has_value())) { + LOG(INFO) << "All streams at EOS, request loop is stopping."; + return; + } + // Only one of two streams is at eos; download data for the stream NOT at + // eos. + type = audio_reached_eos_ ? DemuxerStream::VIDEO : DemuxerStream::AUDIO; + } else if (!audio_stream_.has_value() || !video_stream_.has_value()) { + // If only one stream is present and not at eos, just download that data. + type = + audio_stream_.has_value() ? DemuxerStream::AUDIO : DemuxerStream::VIDEO; + } else { + // Both streams are present, and neither is at eos. Priority order for + // figuring out what to download next. + const base::TimeDelta audio_stamp = audio_stream_->GetLastBufferTimestamp(); + const base::TimeDelta video_stamp = video_stream_->GetLastBufferTimestamp(); + // If the audio demuxer stream is empty, always fill it first. + if (audio_stamp == kNoTimestamp) { + type = DemuxerStream::AUDIO; + } else if (video_stamp == kNoTimestamp) { + // The video demuxer stream is empty; we need data for it. + type = DemuxerStream::VIDEO; + } else if (video_stamp < audio_stamp) { + // Video is earlier; fill it first. + type = DemuxerStream::VIDEO; + } else { + type = DemuxerStream::AUDIO; + } + } + + DCHECK_NE(type, DemuxerStream::UNKNOWN); + // We cannot call Request() directly even if this function is also run on + // |blocking_thread_| as otherwise it is possible that this function is + // running in a tight loop and seek/stop requests would have no chance to kick + // in. + blocking_thread_.task_runner()->PostTask( + FROM_HERE, base::BindRepeating(&DemuxerExtensionWrapper::Request, + base::Unretained(this), type)); +} + +bool DemuxerExtensionWrapper::HasStopped() { + base::AutoLock lock(lock_for_stopped_); + return stopped_; +} + +namespace { + +// Ensure that the demuxer extension's enums match up with the internal enums. +// This doesn't affect any code, but prevents compilation if there's a mismatch +// somewhere. +#define DEMUXER_EXTENSION_ENUM_EQ(a, b) \ + static_assert(static_cast(a) == static_cast(b), \ + "mismatching_" \ + "enums") + +// Pipeline status. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerOk, ::media::PIPELINE_OK); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorNetwork, + ::media::PIPELINE_ERROR_NETWORK); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorAbort, + ::media::PIPELINE_ERROR_ABORT); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorInitializationFailed, + ::media::PIPELINE_ERROR_INITIALIZATION_FAILED); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorRead, + ::media::PIPELINE_ERROR_READ); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorInvalidState, + ::media::PIPELINE_ERROR_INVALID_STATE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorCouldNotOpen, + ::media::DEMUXER_ERROR_COULD_NOT_OPEN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorCouldNotParse, + ::media::DEMUXER_ERROR_COULD_NOT_PARSE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerErrorNoSupportedStreams, + ::media::DEMUXER_ERROR_NO_SUPPORTED_STREAMS); + +// Audio codecs. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecUnknownAudio, + ::media::AudioCodec::kUnknown); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecAAC, + ::media::AudioCodec::kAAC); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecMP3, + ::media::AudioCodec::kMP3); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecPCM, + ::media::AudioCodec::kPCM); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecVorbis, + ::media::AudioCodec::kVorbis); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecFLAC, + ::media::AudioCodec::kFLAC); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecAMR_NB, + ::media::AudioCodec::kAMR_NB); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecAMR_WB, + ::media::AudioCodec::kAMR_WB); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecPCM_MULAW, + ::media::AudioCodec::kPCM_MULAW); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecGSM_MS, + ::media::AudioCodec::kGSM_MS); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecPCM_S16BE, + ::media::AudioCodec::kPCM_S16BE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecPCM_S24BE, + ::media::AudioCodec::kPCM_S24BE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecOpus, + ::media::AudioCodec::kOpus); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecEAC3, + ::media::AudioCodec::kEAC3); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecPCM_ALAW, + ::media::AudioCodec::kPCM_ALAW); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecALAC, + ::media::AudioCodec::kALAC); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecAC3, + ::media::AudioCodec::kAC3); + +// Video codecs. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecUnknownVideo, + ::media::VideoCodec::kUnknown); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecH264, + ::media::VideoCodec::kH264); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecVC1, + ::media::VideoCodec::kVC1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecMPEG2, + ::media::VideoCodec::kMPEG2); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecMPEG4, + ::media::VideoCodec::kMPEG4); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecTheora, + ::media::VideoCodec::kTheora); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecVP8, + ::media::VideoCodec::kVP8); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecVP9, + ::media::VideoCodec::kVP9); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecHEVC, + ::media::VideoCodec::kHEVC); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecDolbyVision, + ::media::VideoCodec::kDolbyVision); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerCodecAV1, + ::media::VideoCodec::kAV1); + +// Sample formats. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatUnknown, + ::media::kUnknownSampleFormat); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatU8, + ::media::kSampleFormatU8); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatS16, + ::media::kSampleFormatS16); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatS32, + ::media::kSampleFormatS32); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatF32, + ::media::kSampleFormatF32); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatPlanarS16, + ::media::kSampleFormatPlanarS16); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatPlanarF32, + ::media::kSampleFormatPlanarF32); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatPlanarS32, + ::media::kSampleFormatPlanarS32); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerSampleFormatS24, + ::media::kSampleFormatS24); + +// Channel layouts. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutNone, + ::media::CHANNEL_LAYOUT_NONE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutUnsupported, + ::media::CHANNEL_LAYOUT_UNSUPPORTED); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutMono, + ::media::CHANNEL_LAYOUT_MONO); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutStereo, + ::media::CHANNEL_LAYOUT_STEREO); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout2_1, + ::media::CHANNEL_LAYOUT_2_1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutSurround, + ::media::CHANNEL_LAYOUT_SURROUND); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout4_0, + ::media::CHANNEL_LAYOUT_4_0); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout2_2, + ::media::CHANNEL_LAYOUT_2_2); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutQuad, + ::media::CHANNEL_LAYOUT_QUAD); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout5_0, + ::media::CHANNEL_LAYOUT_5_0); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout5_1, + ::media::CHANNEL_LAYOUT_5_1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout5_0Back, + ::media::CHANNEL_LAYOUT_5_0_BACK); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout5_1Back, + ::media::CHANNEL_LAYOUT_5_1_BACK); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout7_0, + ::media::CHANNEL_LAYOUT_7_0); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout7_1, + ::media::CHANNEL_LAYOUT_7_1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout7_1Wide, + ::media::CHANNEL_LAYOUT_7_1_WIDE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutStereoDownmix, + ::media::CHANNEL_LAYOUT_STEREO_DOWNMIX); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout2point1, + ::media::CHANNEL_LAYOUT_2POINT1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout3_1, + ::media::CHANNEL_LAYOUT_3_1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout4_1, + ::media::CHANNEL_LAYOUT_4_1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout6_0, + ::media::CHANNEL_LAYOUT_6_0); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout6_0Front, + ::media::CHANNEL_LAYOUT_6_0_FRONT); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutHexagonal, + ::media::CHANNEL_LAYOUT_HEXAGONAL); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout6_1, + ::media::CHANNEL_LAYOUT_6_1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout6_1Back, + ::media::CHANNEL_LAYOUT_6_1_BACK); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout6_1Front, + ::media::CHANNEL_LAYOUT_6_1_FRONT); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout7_0Front, + ::media::CHANNEL_LAYOUT_7_0_FRONT); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout7_1WideBack, + ::media::CHANNEL_LAYOUT_7_1_WIDE_BACK); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutOctagonal, + ::media::CHANNEL_LAYOUT_OCTAGONAL); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutDiscrete, + ::media::CHANNEL_LAYOUT_DISCRETE); +DEMUXER_EXTENSION_ENUM_EQ( + kCobaltExtensionDemuxerChannelLayoutStereoAndKeyboardMic, + ::media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayout4_1QuadSide, + ::media::CHANNEL_LAYOUT_4_1_QUAD_SIDE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerChannelLayoutBitstream, + ::media::CHANNEL_LAYOUT_BITSTREAM); + +// Video codec profiles. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVideoCodecProfileUnknown, + ::media::VIDEO_CODEC_PROFILE_UNKNOWN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileMin, + ::media::H264PROFILE_MIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileBaseline, + ::media::H264PROFILE_BASELINE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileMain, + ::media::H264PROFILE_MAIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileExtended, + ::media::H264PROFILE_EXTENDED); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileHigh, + ::media::H264PROFILE_HIGH); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileHigh10Profile, + ::media::H264PROFILE_HIGH10PROFILE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileHigh422Profile, + ::media::H264PROFILE_HIGH422PROFILE); +DEMUXER_EXTENSION_ENUM_EQ( + kCobaltExtensionDemuxerH264ProfileHigh444PredictiveProfile, + ::media::H264PROFILE_HIGH444PREDICTIVEPROFILE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileScalableBaseline, + ::media::H264PROFILE_SCALABLEBASELINE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileScalableHigh, + ::media::H264PROFILE_SCALABLEHIGH); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileStereoHigh, + ::media::H264PROFILE_STEREOHIGH); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileMultiviewHigh, + ::media::H264PROFILE_MULTIVIEWHIGH); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerH264ProfileMax, + ::media::H264PROFILE_MAX); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp8ProfileMin, + ::media::VP8PROFILE_MIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp8ProfileAny, + ::media::VP8PROFILE_ANY); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp8ProfileMax, + ::media::VP8PROFILE_MAX); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp9ProfileMin, + ::media::VP9PROFILE_MIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp9ProfileProfile0, + ::media::VP9PROFILE_PROFILE0); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp9ProfileProfile1, + ::media::VP9PROFILE_PROFILE1); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp9ProfileProfile2, + ::media::VP9PROFILE_PROFILE2); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp9ProfileProfile3, + ::media::VP9PROFILE_PROFILE3); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerVp9ProfileMax, + ::media::VP9PROFILE_MAX); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerHevcProfileMin, + ::media::HEVCPROFILE_MIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerHevcProfileMain, + ::media::HEVCPROFILE_MAIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerHevcProfileMain10, + ::media::HEVCPROFILE_MAIN10); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerHevcProfileMainStillPicture, + ::media::HEVCPROFILE_MAIN_STILL_PICTURE); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerHevcProfileMax, + ::media::HEVCPROFILE_MAX); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerDolbyVisionProfile0, + ::media::DOLBYVISION_PROFILE0); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerDolbyVisionProfile4, + ::media::DOLBYVISION_PROFILE4); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerDolbyVisionProfile5, + ::media::DOLBYVISION_PROFILE5); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerDolbyVisionProfile7, + ::media::DOLBYVISION_PROFILE7); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerTheoraProfileMin, + ::media::THEORAPROFILE_MIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerTheoraProfileAny, + ::media::THEORAPROFILE_ANY); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerTheoraProfileMax, + ::media::THEORAPROFILE_MAX); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerAv1ProfileMin, + ::media::AV1PROFILE_MIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerAv1ProfileProfileMain, + ::media::AV1PROFILE_PROFILE_MAIN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerAv1ProfileProfileHigh, + ::media::AV1PROFILE_PROFILE_HIGH); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerAv1ProfileProfilePro, + ::media::AV1PROFILE_PROFILE_PRO); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerAv1ProfileMax, + ::media::AV1PROFILE_MAX); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerDolbyVisionProfile8, + ::media::DOLBYVISION_PROFILE8); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerDolbyVisionProfile9, + ::media::DOLBYVISION_PROFILE9); + +// Color range IDs. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerColorSpaceRangeIdInvalid, + gfx::ColorSpace::RangeID::INVALID); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerColorSpaceRangeIdLimited, + gfx::ColorSpace::RangeID::LIMITED); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerColorSpaceRangeIdFull, + gfx::ColorSpace::RangeID::FULL); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerColorSpaceRangeIdDerived, + gfx::ColorSpace::RangeID::DERIVED); + +// Alpha modes. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerHasAlpha, + ::media::VideoDecoderConfig::AlphaMode::kHasAlpha); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerIsOpaque, + ::media::VideoDecoderConfig::AlphaMode::kIsOpaque); + +// Demuxer stream types. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerStreamTypeUnknown, + ::media::DemuxerStream::Type::UNKNOWN); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerStreamTypeAudio, + ::media::DemuxerStream::Type::AUDIO); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerStreamTypeVideo, + ::media::DemuxerStream::Type::VIDEO); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerStreamTypeText, + ::media::DemuxerStream::Type::TEXT); + +// Encryption schemes. +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerEncryptionSchemeUnencrypted, + ::media::EncryptionScheme::kUnencrypted); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerEncryptionSchemeCenc, + ::media::EncryptionScheme::kCenc); +DEMUXER_EXTENSION_ENUM_EQ(kCobaltExtensionDemuxerEncryptionSchemeCbcs, + ::media::EncryptionScheme::kCbcs); + +#undef DEMUXER_EXTENSION_ENUM_EQ + +} // namespace + +} // namespace media diff --git a/media/starboard/progressive/demuxer_extension_wrapper.h b/media/starboard/progressive/demuxer_extension_wrapper.h new file mode 100644 index 000000000000..805e8d73f9de --- /dev/null +++ b/media/starboard/progressive/demuxer_extension_wrapper.h @@ -0,0 +1,249 @@ +// Copyright 2022 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Contains classes that wrap the Demuxer Cobalt Extension, providing an +// implementation of a Cobalt demuxer. The main API is DemuxerExtensionWrapper. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_DEMUXER_EXTENSION_WRAPPER_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_DEMUXER_EXTENSION_WRAPPER_H_ + +#include +#include +#include +#include + +#include "base/memory/scoped_refptr.h" +#include "base/sequence_checker.h" +#include "base/threading/thread.h" +#include "media/base/audio_decoder_config.h" +#include "media/base/decoder_buffer.h" +#include "media/base/demuxer.h" +#include "media/base/pipeline_status.h" +#include "media/base/ranges.h" +#include "media/base/video_decoder_config.h" +#include "media/starboard/progressive/data_source_reader.h" +#include "starboard/extension/demuxer.h" +#include "third_party/abseil-cpp/absl/types/optional.h" + +namespace media { + +// Represents an audio or video stream. Reads data via the demuxer Cobalt +// Extension. +class DemuxerExtensionStream : public DemuxerStream { + public: + // Represents a video stream. + explicit DemuxerExtensionStream( + CobaltExtensionDemuxer* demuxer, + scoped_refptr task_runner, + CobaltExtensionDemuxerVideoDecoderConfig config); + // Represents an audio stream. + explicit DemuxerExtensionStream( + CobaltExtensionDemuxer* demuxer, + scoped_refptr task_runner, + CobaltExtensionDemuxerAudioDecoderConfig config); + + // Disallow copy and assign. + DemuxerExtensionStream(const DemuxerExtensionStream&) = delete; + DemuxerExtensionStream& operator=(const DemuxerExtensionStream&) = delete; + + ~DemuxerExtensionStream() = default; + + // Functions used by DemuxerExtensionWrapper. + Ranges GetBufferedRanges(); + void EnqueueBuffer(scoped_refptr buffer); + void FlushBuffers(); + void Stop(); + base::TimeDelta GetLastBufferTimestamp() const; + size_t GetTotalBufferSize() const; + + // DemuxerStream implementation: + void Read(uint32_t count, ReadCB read_cb) override; + AudioDecoderConfig audio_decoder_config() override; + VideoDecoderConfig video_decoder_config() override; + Type type() const override; + + void EnableBitstreamConverter() override { NOTIMPLEMENTED(); } + + bool SupportsConfigChanges() override { return false; } + + private: + typedef std::deque> BufferQueue; + typedef std::deque ReadQueue; + + CobaltExtensionDemuxer* demuxer_ = nullptr; // Not owned. + std::optional video_config_; + std::optional audio_config_; + + // Protects everything below. + mutable base::Lock lock_; + // Keeps track of all time ranges this object has seen since creation. + // The demuxer uses these ranges to update the pipeline about what data + // it has demuxed. + Ranges buffered_ranges_; + // The last timestamp of buffer enqueued. This is used in two places: + // 1. Used with the timestamp of the current frame to calculate the + // buffer range. + // 2. Used by the demuxer to deteminate what type of frame to get next. + base::TimeDelta last_buffer_timestamp_ = kNoTimestamp; + bool stopped_ = false; + + BufferQueue buffer_queue_; + ReadQueue read_queue_; + + scoped_refptr task_runner_; + + size_t total_buffer_size_ = 0; +}; + +// Wraps a DataSourceReader in an even simpler API, where each read increments +// the read location. This better matches the C data source API. +class PositionalDataSource { + public: + explicit PositionalDataSource(scoped_refptr reader); + + // Disallow copy and assign. + PositionalDataSource(const PositionalDataSource&) = delete; + PositionalDataSource& operator=(const PositionalDataSource&) = delete; + + ~PositionalDataSource(); + + void Stop(); + + // Reads up to |bytes_requested|, writing the data into |data|. + int BlockingRead(uint8_t* data, int bytes_requested); + + // Seeks to |position|. + void SeekTo(int position); + + // Returns the current read position. + int64_t GetPosition() const; + + // Returns the size of the file. + // + // TODO(b/231744342): investigate whether we need to fix + // DataSourceReader::FileSize(). In testing, it sometimes returned inaccurate + // results before a file was fully downloaded. That behavior affects what this + // function returns. + int64_t GetSize(); + + private: + scoped_refptr reader_; + int64_t position_ = 0; +}; + +// Wraps the demuxer Cobalt Extension in the internal media::Demuxer API. +// Instances should be created via the Create method. +class DemuxerExtensionWrapper : public Demuxer { + public: + // Constructs a new DemuxerExtensionWrapper, returning null on failure. If + // |data_source| or |task_runner| is null, or if a demuxer cannot be created, + // this will return null. If |demuxer_api| is null, we will attempt to use the + // corresponding Cobalt extension. + static std::unique_ptr Create( + DataSource* data_source, + scoped_refptr task_runner, + const CobaltExtensionDemuxerApi* demuxer_api = nullptr); + + // Disallow copy and assign. + DemuxerExtensionWrapper(const DemuxerExtensionWrapper&) = delete; + DemuxerExtensionWrapper& operator=(const DemuxerExtensionWrapper&) = delete; + + ~DemuxerExtensionWrapper() override; + + // Demuxer implementation: + std::vector GetAllStreams() override; + std::string GetDisplayName() const override; + DemuxerType GetDemuxerType() const override { + // kFFmpegDemuxer is used in Chromium media for progressive demuxing. + return DemuxerType::kFFmpegDemuxer; + } + void Initialize(DemuxerHost* host, PipelineStatusCallback status_cb) override; + void AbortPendingReads() override; + void StartWaitingForSeek(base::TimeDelta seek_time) override; + void CancelPendingSeek(base::TimeDelta seek_time) override; + void Seek(base::TimeDelta time, PipelineStatusCallback status_cb) override; + bool IsSeekable() const override { return true; } + void Stop() override; + base::TimeDelta GetStartTime() const override; + base::Time GetTimelineOffset() const override; + int64_t GetMemoryUsage() const override; + void OnEnabledAudioTracksChanged(const std::vector& track_ids, + base::TimeDelta curr_time, + TrackChangeCB change_completed_cb) override; + void OnSelectedVideoTrackChanged(const std::vector& track_ids, + base::TimeDelta curr_time, + TrackChangeCB change_completed_cb) override; + void SetPlaybackRate(double rate) override { NOTREACHED(); } + + absl::optional GetContainerForMetrics() + const override { + NOTREACHED(); + return absl::nullopt; + } + + private: + // Only a forward declaration here, since the specifics of this class are an + // implementation detail. + class H264AnnexBConverter; + + // Arguments must not be null. + explicit DemuxerExtensionWrapper( + const CobaltExtensionDemuxerApi* demuxer_api, + CobaltExtensionDemuxer* demuxer, + std::unique_ptr data_source, + std::unique_ptr c_data_source, + scoped_refptr task_runner); + + void OnInitializeDone(PipelineStatusCallback status_cb, + CobaltExtensionDemuxerStatus status); + void Request(DemuxerStream::Type type); + bool HasStopped(); + void IssueNextRequest(); + void SeekTask(base::TimeDelta time, PipelineStatusCallback status_cb); + + // Returns the range of buffered data. If both audio and video streams are + // present, this is the intersection of their buffered ranges; otherwise, it + // is whatever range of data is buffered. + Ranges GetBufferedRanges(); + + const CobaltExtensionDemuxerApi* demuxer_api_ = nullptr; // Not owned. + // Owned by this class. Construction/destruction is done via demuxer_api_. + CobaltExtensionDemuxer* impl_ = nullptr; + std::unique_ptr data_source_; + std::unique_ptr c_data_source_; + DemuxerHost* host_ = nullptr; + mutable base::Lock lock_for_stopped_; + // Indicates whether Stop has been called. + bool stopped_ = false; + bool video_reached_eos_ = false; + bool audio_reached_eos_ = false; + bool flushing_ = false; + + absl::optional video_stream_; + absl::optional audio_stream_; + + std::unique_ptr h264_converter_; + + // Thread for blocking I/O operations. + base::Thread blocking_thread_; + + scoped_refptr task_runner_; + + SEQUENCE_CHECKER(sequence_checker_); + base::WeakPtrFactory weak_factory_{this}; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_DEMUXER_EXTENSION_WRAPPER_H_ diff --git a/media/starboard/progressive/endian_util.h b/media/starboard/progressive/endian_util.h new file mode 100644 index 000000000000..1059db3b891a --- /dev/null +++ b/media/starboard/progressive/endian_util.h @@ -0,0 +1,141 @@ +// Copyright 2016 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_ENDIAN_UTIL_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_ENDIAN_UTIL_H_ + +#include "base/sys_byteorder.h" + +// TODO: Consider Starboardize functions in this file. + +namespace media { +namespace endian_util { + +// The following functions must be able to support storing to/loading from +// non-aligned memory. Thus, casts like "*reinterpret_cast(p)" +// should be avoided as these can cause crashes due to alignment on some +// platforms. + +// Load 2 little-endian bytes at |p| and return as a host-endian uint16_t. +inline uint16_t load_uint16_little_endian(const uint8_t* p) { + uint16_t aligned_p; + memcpy(&aligned_p, p, sizeof(aligned_p)); + return base::ByteSwapToLE16(aligned_p); +} + +// Load 4 little-endian bytes at |p| and return as a host-endian uint32_t. +inline uint32_t load_uint32_little_endian(const uint8_t* p) { + uint32_t aligned_p; + memcpy(&aligned_p, p, sizeof(aligned_p)); + return base::ByteSwapToLE32(aligned_p); +} + +// Load 8 little-endian bytes at |p| and return as a host-endian uint64_t. +inline uint64_t load_uint64_little_endian(const uint8_t* p) { + uint64_t aligned_p; + memcpy(&aligned_p, p, sizeof(aligned_p)); + return base::ByteSwapToLE64(aligned_p); +} + +// Load 2 big-endian bytes at |p| and return as a host-endian uint16_t. +inline uint16_t load_uint16_big_endian(const uint8_t* p) { + uint16_t aligned_p; + memcpy(&aligned_p, p, sizeof(aligned_p)); + return base::NetToHost16(aligned_p); +} + +// Load 4 big-endian bytes at |p| and return as a host-endian uint32_t. +inline uint32_t load_uint32_big_endian(const uint8_t* p) { + uint32_t aligned_p; + memcpy(&aligned_p, p, sizeof(aligned_p)); + return base::NetToHost32(aligned_p); +} + +// Load 8 big-endian bytes at |p| and return as a host-endian uint64_t. +inline uint64_t load_uint64_big_endian(const uint8_t* p) { + uint64_t aligned_p; + memcpy(&aligned_p, p, sizeof(aligned_p)); + return base::NetToHost64(aligned_p); +} + +// Load 2 big-endian bytes at |p| and return as an host-endian int16_t. +inline int16_t load_int16_big_endian(const uint8_t* p) { + return static_cast(load_uint16_big_endian(p)); +} + +// Load 4 big-endian bytes at |p| and return as an host-endian int32_t. +inline int32_t load_int32_big_endian(const uint8_t* p) { + return static_cast(load_uint32_big_endian(p)); +} + +// Load 8 big-endian bytes at |p| and return as an host-endian int64_t. +inline int64_t load_int64_big_endian(const uint8_t* p) { + return static_cast(load_uint64_big_endian(p)); +} + +// Load 2 little-endian bytes at |p| and return as a host-endian int16_t. +inline int16_t load_int16_little_endian(const uint8_t* p) { + return static_cast(load_uint16_little_endian(p)); +} + +// Load 4 little-endian bytes at |p| and return as a host-endian int32_t. +inline int32_t load_int32_little_endian(const uint8_t* p) { + return static_cast(load_uint32_little_endian(p)); +} + +// Load 8 little-endian bytes at |p| and return as a host-endian int64_t. +inline int64_t load_int64_little_endian(const uint8_t* p) { + return static_cast(load_uint64_little_endian(p)); +} + +// Store 2 host-endian bytes as big-endian at |p|. +inline void store_uint16_big_endian(uint16_t d, uint8_t* p) { + uint16_t big_d = base::HostToNet16(d); + memcpy(p, &big_d, sizeof(big_d)); +} + +// Store 4 host-endian bytes as big-endian at |p|. +inline void store_uint32_big_endian(uint32_t d, uint8_t* p) { + uint32_t big_d = base::HostToNet32(d); + memcpy(p, &big_d, sizeof(big_d)); +} + +// Store 8 host-endian bytes as big-endian at |p|. +inline void store_uint64_big_endian(uint64_t d, uint8_t* p) { + uint64_t big_d = base::HostToNet64(d); + memcpy(p, &big_d, sizeof(big_d)); +} + +// Store 2 host-endian bytes as little-endian at |p|. +inline void store_uint16_little_endian(uint16_t d, uint8_t* p) { + uint16_t little_d = base::ByteSwapToLE16(d); + memcpy(p, &little_d, sizeof(little_d)); +} + +// Store 4 host-endian bytes as little-endian at |p|. +inline void store_uint32_little_endian(uint32_t d, uint8_t* p) { + uint32_t little_d = base::ByteSwapToLE32(d); + memcpy(p, &little_d, sizeof(little_d)); +} + +// Store 8 host-endian bytes as little-endian at |p|. +inline void store_uint64_little_endian(uint64_t d, uint8_t* p) { + uint64_t little_d = base::ByteSwapToLE64(d); + memcpy(p, &little_d, sizeof(little_d)); +} + +} // namespace endian_util +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_ENDIAN_UTIL_H_ diff --git a/media/starboard/progressive/mock_data_source_reader.h b/media/starboard/progressive/mock_data_source_reader.h new file mode 100644 index 000000000000..bc1fb70ddadd --- /dev/null +++ b/media/starboard/progressive/mock_data_source_reader.h @@ -0,0 +1,36 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_MOCK_DATA_SOURCE_READER_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_MOCK_DATA_SOURCE_READER_H_ + +#include "media/starboard/progressive/data_source_reader.h" +#include "testing/gmock/include/gmock/gmock.h" + +namespace media { + +class MockDataSourceReader : public DataSourceReader { + public: + MockDataSourceReader() {} + + // DataSourceReader implementation + MOCK_METHOD1(SetDataSource, void(DataSource*)); + MOCK_METHOD3(BlockingRead, int(int64, int, uint8*)); + MOCK_METHOD0(FileSize, int64()); + MOCK_METHOD0(Stop, void()); +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_MOCK_DATA_SOURCE_READER_H_ diff --git a/media/starboard/progressive/mp4_map.cc b/media/starboard/progressive/mp4_map.cc new file mode 100644 index 000000000000..8460da0694ae --- /dev/null +++ b/media/starboard/progressive/mp4_map.cc @@ -0,0 +1,1167 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/mp4_map.h" + +#include + +#include "base/strings/stringprintf.h" +#include "media/starboard/progressive/endian_util.h" +#include "media/starboard/progressive/mp4_parser.h" + +namespace media { + +// ==== TableCache ============================================================= + +MP4Map::TableCache::TableCache(uint64_t table_offset, + uint32_t entry_count, + uint32_t entry_size, + uint32_t cache_size_entries, + scoped_refptr reader) + : entry_size_(entry_size), + entry_count_(entry_count), + cache_size_entries_(cache_size_entries), + table_offset_(table_offset), + reader_(reader), + cache_first_entry_number_(-1), + cache_entry_count_(0) {} + +uint8_t* MP4Map::TableCache::GetBytesAtEntry(uint32_t entry_number) { + // don't fetch the unfetchable + if (entry_number >= entry_count_) { + return NULL; + } + // this query within valid range for the current cache table? + if (entry_number < cache_first_entry_number_ || + entry_number >= cache_first_entry_number_ + cache_entry_count_) { + // Calculate first entry in table keeping cache size alignment in table. + // Always cache one more entry as in stss we need to use the first entry + // of the next cache slot as an upper bound. + cache_entry_count_ = cache_size_entries_ + 1; + cache_first_entry_number_ = + (entry_number / cache_size_entries_) * cache_size_entries_; + // see if we have exceeded our table bounds + if (cache_first_entry_number_ + cache_entry_count_ > entry_count_) { + cache_entry_count_ = entry_count_ - cache_first_entry_number_; + } + // drop old data + cache_.clear(); + DCHECK_GE(cache_entry_count_, 0); + int bytes_to_read = cache_entry_count_ * entry_size_; + cache_.resize(bytes_to_read); + uint64_t file_offset = + table_offset_ + (cache_first_entry_number_ * entry_size_); + int bytes_read = + reader_->BlockingRead(file_offset, bytes_to_read, &cache_[0]); + if (bytes_read < bytes_to_read) { + cache_entry_count_ = 0; + return NULL; + } + } + // cache is assumed to be valid and to contain the entry from here on + DCHECK_GE(entry_number, cache_first_entry_number_); + DCHECK_LT(entry_number, cache_first_entry_number_ + cache_entry_count_); + + uint32_t cache_offset = entry_number - cache_first_entry_number_; + return &cache_[0] + (cache_offset * entry_size_); +} + +bool MP4Map::TableCache::ReadU32Entry(uint32_t entry_number, uint32_t* entry) { + if (uint8_t* data = GetBytesAtEntry(entry_number)) { + *entry = endian_util::load_uint32_big_endian(data); + return true; + } + + return false; +} + +bool MP4Map::TableCache::ReadU32PairEntry(uint32_t entry_number, + uint32_t* first, + uint32_t* second) { + if (uint8_t* data = GetBytesAtEntry(entry_number)) { + if (first) { + *first = endian_util::load_uint32_big_endian(data); + } + if (second) { + *second = endian_util::load_uint32_big_endian(data + 4); + } + return true; + } + + return false; +} + +bool MP4Map::TableCache::ReadU32EntryIntoU64(uint32_t entry_number, + uint64_t* entry) { + if (uint8_t* data = GetBytesAtEntry(entry_number)) { + *entry = endian_util::load_uint32_big_endian(data); + return true; + } + + return false; +} + +bool MP4Map::TableCache::ReadU64Entry(uint32_t entry_number, uint64_t* entry) { + if (uint8_t* data = GetBytesAtEntry(entry_number)) { + *entry = endian_util::load_uint64_big_endian(data); + return true; + } + + return false; +} + +// ==== MP4Map ============================================================ + +// atom | name | size | description, (*) means optional table +// -----+-----------------------+------+---------------------------------------- +// co64 | chunk offset (64-bit) | 8 | per-chunk list of chunk file offsets +// ctts | composition offset | 8 | (*) run-length sample number to cts +// stco | chunk offset (32-bit) | 4 | per-chunk list of chunk file offsets +// stsc | sample-to-chunk | 12 | chunk number to samples per chunk +// stss | sync sample | 4 | (*) list of keyframe sample numbers +// stts | time-to-sample | 8 | run-length sample number to duration +// stsz | sample size | 4 | per-sample list of sample sizes + +MP4Map::MP4Map(scoped_refptr reader) + : reader_(reader), + current_chunk_sample_(0), + next_chunk_sample_(0), + current_chunk_offset_(0), + highest_valid_sample_number_(UINT32_MAX), + ctts_first_sample_(0), + ctts_sample_offset_(0), + ctts_next_first_sample_(0), + ctts_table_index_(0), + stsc_first_chunk_(0), + stsc_first_chunk_sample_(0), + stsc_samples_per_chunk_(0), + stsc_next_first_chunk_(0), + stsc_next_first_chunk_sample_(0), + stsc_table_index_(0), + stss_last_keyframe_(0), + stss_next_keyframe_(0), + stss_table_index_(0), + stts_first_sample_(0), + stts_first_sample_time_(0), + stts_sample_duration_(0), + stts_next_first_sample_(0), + stts_next_first_sample_time_(0), + stts_table_index_(0), + stsz_default_size_(0) {} + +bool MP4Map::IsComplete() { + // all required table pointers must be valid for map to function + return (co64_ || stco_) && stsc_ && stts_ && (stsz_ || stsz_default_size_); +} + +// The sample size is a lookup in the stsz table, which is indexed per sample +// number. +bool MP4Map::GetSize(uint32_t sample_number, uint32_t* size_out) { + DCHECK(size_out); + DCHECK(stsz_ || stsz_default_size_); + + if (sample_number > highest_valid_sample_number_) { + return false; + } + + if (stsz_default_size_) { + *size_out = stsz_default_size_; + return true; + } + + return stsz_->ReadU32Entry(sample_number, size_out); +} + +// We first must integrate the stsc table to find the chunk number that the +// sample resides in, and the first sample number in that chunk. We look up that +// chunk offset from the stco or co64, which are indexed by chunk number. We +// then use the stsz to sum samples to the byte offset with that chunk. The sum +// of the chunk offset and the byte offset within the chunk is the offset of +// the sample. +bool MP4Map::GetOffset(uint32_t sample_number, uint64_t* offset_out) { + DCHECK(offset_out); + DCHECK(stsc_); + DCHECK(stco_ || co64_); + DCHECK(stsz_ || stsz_default_size_); + + if (sample_number > highest_valid_sample_number_) { + return false; + } + + // check for sequential access of sample numbers within the same chunk + if (sample_number < current_chunk_sample_ || + sample_number >= next_chunk_sample_) { + // integrate through stsc until we find the chunk range containing sample + if (!stsc_AdvanceToSample(sample_number)) { + return false; + } + // make sure stsc advance did its job correctly + DCHECK_GE(sample_number, stsc_first_chunk_sample_); + + // calculate chunk number based on chunk sample size for this range + uint32_t sample_offset = sample_number - stsc_first_chunk_sample_; + uint32_t chunk_range_offset = sample_offset / stsc_samples_per_chunk_; + uint32_t chunk_number = stsc_first_chunk_ + chunk_range_offset; + // should be within the range of chunks with this sample size + DCHECK_LT(chunk_number, stsc_next_first_chunk_); + // update first sample number contained within this chunk + current_chunk_sample_ = stsc_first_chunk_sample_ + + (chunk_range_offset * stsc_samples_per_chunk_); + // update first sample number of next chunk + next_chunk_sample_ = current_chunk_sample_ + stsc_samples_per_chunk_; + // find offset of this chunk within the file from co64/stco + if (co64_) { + if (!co64_->ReadU64Entry(chunk_number, ¤t_chunk_offset_)) { + return false; + } + } else if (!stco_->ReadU32EntryIntoU64(chunk_number, + ¤t_chunk_offset_)) { + return false; + } + } + + // at this point we should have sample_number within the range of our chunk + // offset summation saved state + DCHECK_LE(current_chunk_sample_, sample_number); + DCHECK_LT(sample_number, next_chunk_sample_); + + if (stsz_default_size_ > 0) { + current_chunk_offset_ += + (sample_number - current_chunk_sample_) * stsz_default_size_; + current_chunk_sample_ = sample_number; + } else { + // sum sample sizes within chunk to get to byte offset of sample + while (current_chunk_sample_ < sample_number) { + uint32_t sample_size = 0; + if (!GetSize(current_chunk_sample_, &sample_size)) { + return false; + } + current_chunk_offset_ += sample_size; + current_chunk_sample_++; + } + } + + *offset_out = current_chunk_offset_; + return true; +} + +// Given a current sample number we integrate through the stts to find the +// duration of the current sample, and at the same time integrate through the +// durations to find the dts of that sample number. We then integrate sample +// numbers through the ctts to find the composition time offset, which we add to +// the dts to return the pts. +bool MP4Map::GetTimestamp(uint32_t sample_number, uint64_t* timestamp_out) { + DCHECK(timestamp_out); + if (sample_number > highest_valid_sample_number_) { + return false; + } + + if (!stts_AdvanceToSample(sample_number)) { + return false; + } + DCHECK_LT(sample_number, stts_next_first_sample_); + DCHECK_GE(sample_number, stts_first_sample_); + uint64_t dts = stts_first_sample_time_ + + (sample_number - stts_first_sample_) * stts_sample_duration_; + if (ctts_) { + if (!ctts_AdvanceToSample(sample_number)) { + return false; + } + DCHECK_LT(sample_number, ctts_next_first_sample_); + DCHECK_GE(sample_number, ctts_first_sample_); + } + *timestamp_out = dts + ctts_sample_offset_; + return true; +} + +// Sum through the stts to find the duration of the given sample_number. +bool MP4Map::GetDuration(uint32_t sample_number, uint32_t* duration_out) { + DCHECK(duration_out); + if (sample_number > highest_valid_sample_number_) { + return false; + } + + if (!stts_AdvanceToSample(sample_number)) { + return false; + } + DCHECK_LT(sample_number, stts_next_first_sample_); + DCHECK_GE(sample_number, stts_first_sample_); + *duration_out = stts_sample_duration_; + return true; +} + +bool MP4Map::GetIsKeyframe(uint32_t sample_number, bool* is_keyframe_out) { + DCHECK(is_keyframe_out); + if (sample_number > highest_valid_sample_number_) { + return false; + } + + // no stts means every frame is a keyframe + if (!stss_) { + *is_keyframe_out = true; + return true; + } + + // check for keyframe match on either range value + if (sample_number == stss_next_keyframe_) { + *is_keyframe_out = true; + return stss_AdvanceStep(); + } else if (sample_number == stss_last_keyframe_) { + *is_keyframe_out = true; + return true; + } + + // this could be for a much earlier sample number, check if we are within + // current range of sample numbers + if (sample_number < stss_last_keyframe_ || + sample_number > stss_next_keyframe_) { + // search for containing entry + if (!stss_FindNearestKeyframe(sample_number)) { + return false; + } + } + // sample number must be in range of keyframe states + DCHECK_GE(sample_number, stss_last_keyframe_); + DCHECK_LT(sample_number, stss_next_keyframe_); + // stss_FindNearestKeyframe returns exact matches to + // sample_number in the stss_last_keyframe_ variable, so + // we check that for equality + *is_keyframe_out = (sample_number == stss_last_keyframe_); + + return true; +} + +bool MP4Map::IsEOS(uint32_t sample_number) { + return (sample_number > highest_valid_sample_number_); +} + +// First look up the sample number for the provided timestamp by integrating +// timestamps through the stts. Then do a binary search on the stss to find the +// keyframe nearest that sample number. +bool MP4Map::GetKeyframe(uint64_t timestamp, uint32_t* sample_out) { + DCHECK(sample_out); + // Advance stts to the provided timestamp range + if (!stts_AdvanceToTime(timestamp)) { + return false; + } + // ensure we got the correct sample duration range + DCHECK_LT(timestamp, stts_next_first_sample_time_); + DCHECK_GE(timestamp, stts_first_sample_time_); + // calculate sample number containing this timestamp + uint64_t time_offset_within_range = timestamp - stts_first_sample_time_; + uint32_t sample_number = + stts_first_sample_ + (time_offset_within_range / stts_sample_duration_); + + // TODO: ctts? + + // binary search on stts to find nearest keyframe beneath this sample number + if (stss_) { + if (!stss_FindNearestKeyframe(sample_number)) { + return false; + } + *sample_out = stss_last_keyframe_; + } else { + // an absent stts means every frame is a key frame, we can provide sample + // directly. + *sample_out = sample_number; + } + return true; +} + +// Set up map state and load first part of table, or entire table if it is small +// enough, for each of the supported atoms. +bool MP4Map::SetAtom(uint32_t four_cc, + uint64_t offset, + uint64_t size, + uint32_t cache_size_entries, + const uint8_t* atom) { + // All map atoms are variable-length tables starting with 4 bytes of + // version/flag info followed by a uint32_t indicating the number of items in + // table. The stsz atom bucks tradition by putting an optional default value + // at index 4. + uint32_t count = 0; + uint64_t table_offset = offset + 8; + if (four_cc == kAtomType_stsz) { + if (size < 12) { + return false; + } + stsz_default_size_ = endian_util::load_uint32_big_endian(atom + 4); + count = endian_util::load_uint32_big_endian(atom + 8); + highest_valid_sample_number_ = + std::min(count - 1, highest_valid_sample_number_); + // if a non-zero default size is provided don't bother loading the table + if (stsz_default_size_) { + stsz_ = nullptr; + return true; + } + + table_offset += 4; + } else { + if (size < 8) { + return false; + } + count = endian_util::load_uint32_big_endian(atom + 4); + } + + // if cache_size_entries is 0 we are to cache the entire table + if (cache_size_entries == 0) { + cache_size_entries = count; + } + + bool atom_init = false; + // initialize the appropriate table cache dependent on table type + switch (four_cc) { + case kAtomType_co64: + co64_ = new TableCache(table_offset, count, kEntrySize_co64, + cache_size_entries, reader_); + if (co64_) { + atom_init = co64_Init(); + } + break; + + case kAtomType_ctts: + ctts_ = new TableCache(table_offset, count, kEntrySize_ctts, + cache_size_entries, reader_); + if (ctts_) { + atom_init = ctts_Init(); + } + break; + + case kAtomType_stco: + stco_ = new TableCache(table_offset, count, kEntrySize_stco, + cache_size_entries, reader_); + if (stco_) { + atom_init = stco_Init(); + } + break; + + case kAtomType_stsc: + stsc_ = new TableCache(table_offset, count, kEntrySize_stsc, + cache_size_entries, reader_); + if (stsc_) { + atom_init = stsc_Init(); + } + break; + + case kAtomType_stss: + stss_ = new TableCache(table_offset, count, kEntrySize_stss, + cache_size_entries, reader_); + if (stss_) { + atom_init = stss_Init(); + } + break; + + case kAtomType_stts: + stts_ = new TableCache(table_offset, count, kEntrySize_stts, + cache_size_entries, reader_); + if (stts_) { + atom_init = stts_Init(); + } + break; + + case kAtomType_stsz: + stsz_ = new TableCache(table_offset, count, kEntrySize_stsz, + cache_size_entries, reader_); + if (stsz_) { + atom_init = stsz_Init(); + } + break; + + default: + NOTREACHED() << "unknown atom type provided to mp4 map"; + break; + } + + return atom_init; +} + +bool MP4Map::co64_Init() { + DCHECK(co64_); + // load offset of first chunk into current_chunk_offset_ + if (co64_->GetEntryCount() > 0) { + // can drop any stco table already allocated + stco_ = nullptr; + // load initial value of current_chunk_offset_ for 0th chunk + return co64_->ReadU64Entry(0, ¤t_chunk_offset_); + } + + co64_ = nullptr; + + return true; +} + +// The ctts table has the following per-entry layout: +// uint32_t sample count +// uint32_t composition offset in ticks +// +bool MP4Map::ctts_Init() { + DCHECK(ctts_); + // get cache segment vector to reserve table entries in advance + int cache_segments = + (ctts_->GetEntryCount() / ctts_->GetCacheSizeEntries()) + 1; + ctts_samples_.reserve(cache_segments); + if (ctts_->GetEntryCount() > 0) { + // save the start of the first table integration at 0 + ctts_samples_.push_back(0); + ctts_table_index_ = 0; + ctts_first_sample_ = 0; + // load first entry in table, to start integration + return ctts_->ReadU32PairEntry(0, &ctts_next_first_sample_, + &ctts_sample_offset_); + } + // drop empty ctts_ table + ctts_ = nullptr; + + return true; +} + +// To find the composition offset of a given sample number we must integrate +// through the ctts to find the range of samples containing sample_number. Note +// that the ctts is an optional table. +bool MP4Map::ctts_AdvanceToSample(uint32_t sample_number) { + // ctts table is optional, so treat not having one as non-fatal + if (!ctts_) { + return true; + } + // sample number could be before our saved first sample, meaning we've + // gone backward in sample numbers and will need to restart integration at + // the nearest saved sample count starting at a cache entry + if (sample_number < ctts_first_sample_) { + if (!ctts_SlipCacheToSample(sample_number, 0)) { + return false; + } + } + + // sample_number could also be ahead of our current range, for example when + // seeking forward. See if we've calculated these values ahead of us before, + // and if we can slip forward to them + int next_cache_index = (ctts_table_index_ / ctts_->GetCacheSizeEntries()) + 1; + if ((next_cache_index < ctts_samples_.size()) && + (sample_number >= ctts_samples_[next_cache_index])) { + if (!ctts_SlipCacheToSample(sample_number, next_cache_index)) { + return false; + } + } + + // perform integration until sample number is within correct ctts range + while (ctts_next_first_sample_ <= sample_number) { + // next first sample is now our the first sample + ctts_first_sample_ = ctts_next_first_sample_; + // advance to next entry in table + ctts_table_index_++; + // If this would be a new cache entry, keep a record of integration up + // to this point so we don't have to start from 0 on seeking back + if (!(ctts_table_index_ % ctts_->GetCacheSizeEntries())) { + int cache_index = ctts_table_index_ / ctts_->GetCacheSizeEntries(); + // check that this is our first time with these data + if (cache_index == ctts_samples_.size()) { + ctts_samples_.push_back(ctts_first_sample_); + } + // our integration at this point should always match any stored record + DCHECK_EQ(ctts_first_sample_, ctts_samples_[cache_index]); + } + + if (ctts_table_index_ < ctts_->GetEntryCount()) { + // load the sample count to determine next first sample + uint32_t sample_count; + if (!ctts_->ReadU32PairEntry(ctts_table_index_, &sample_count, + &ctts_sample_offset_)) { + return false; + } + ctts_next_first_sample_ = ctts_first_sample_ + sample_count; + } else { + // This means that the last entry in the table specified a sample range + // that this sample number has exceeded, and so the ctts of this sample + // number is undefined. While not a fatal error it's kind of a weird + // state, we set the offset back to zero and extend the next_first_sample + // to infinity + LOG(WARNING) << base::StringPrintf( + "out of range sample number %d in ctts, last valid sample number: %d", + sample_number, ctts_next_first_sample_); + ctts_sample_offset_ = 0; + ctts_next_first_sample_ = UINT32_MAX; + break; + } + } + return true; +} + +bool MP4Map::ctts_SlipCacheToSample(uint32_t sample_number, + int starting_cache_index) { + DCHECK_LT(starting_cache_index, ctts_samples_.size()); + int cache_index = starting_cache_index; + for (; cache_index + 1 < ctts_samples_.size(); cache_index++) { + if (sample_number < ctts_samples_[cache_index + 1]) { + break; + } + } + ctts_first_sample_ = ctts_samples_[cache_index]; + ctts_table_index_ = cache_index * ctts_->GetCacheSizeEntries(); + // read sample count and duration to set next values + uint32_t sample_count; + if (!ctts_->ReadU32PairEntry(ctts_table_index_, &sample_count, + &ctts_sample_offset_)) { + return false; + } + ctts_next_first_sample_ = ctts_first_sample_ + sample_count; + return true; +} + +bool MP4Map::stco_Init() { + DCHECK(stco_); + // load offset of first chunk into current_chunk_offset_ + if (stco_->GetEntryCount() > 0) { + co64_ = nullptr; + return stco_->ReadU32EntryIntoU64(0, ¤t_chunk_offset_); + } + + stco_ = nullptr; + + return true; +} + +// The stsc table has the following per-entry layout: +// uint32_t first chunk number with this sample count +// uint32_t samples-per-chunk +// uint32_t sample description id (unused) +bool MP4Map::stsc_Init() { + DCHECK(stsc_); + // set up vector to correct final size + int cache_segments = + (stsc_->GetEntryCount() / stsc_->GetCacheSizeEntries()) + 1; + stsc_sample_sums_.reserve(cache_segments); + // there must always be at least 1 entry in a valid stsc table + if (stsc_->GetEntryCount() > 0) { + stsc_first_chunk_ = 0; + stsc_first_chunk_sample_ = 0; + // first cached entry is always 0 + stsc_sample_sums_.push_back(0); + if (!stsc_->ReadU32PairEntry(0, NULL, &stsc_samples_per_chunk_)) { + stsc_ = nullptr; + return false; + } + // look up next first chunk at next index in table + if (stsc_->GetEntryCount() > 1) { + if (!stsc_->ReadU32PairEntry(1, &stsc_next_first_chunk_, NULL)) { + stsc_ = nullptr; + return false; + } + --stsc_next_first_chunk_; + stsc_next_first_chunk_sample_ = + stsc_next_first_chunk_ * stsc_samples_per_chunk_; + } else { + // every chunk in the file has the sample sample count, set next first + // chunk to highest valid chunk number. + stsc_next_first_chunk_ = UINT32_MAX; + stsc_next_first_chunk_sample_ = UINT32_MAX; + } + stsc_table_index_ = 0; + + // since we known the size of the first chunk we can set next_chunk_sample_ + next_chunk_sample_ = stsc_samples_per_chunk_; + + } else { + stsc_ = nullptr; + } + + return true; +} + +// To find the chunk number of an arbitrary sample we have to sum the +// samples-per-chunk value multiplied by the number of chunks with that sample +// count until the sum exceeds the sample number, then calculate the chunk +// number from that range of per-sample chunk sizes. Since this map is meant +// to be consumed incrementally and with minimal memory consumption we calculate +// this integration step only when needed, and save results for each cached +// piece of the table, to avoid having to recalculate needed data. +bool MP4Map::stsc_AdvanceToSample(uint32_t sample_number) { + DCHECK(stsc_); + // sample_number could be before first chunk, meaning that we are seeking + // backwards and have left the current chunk. Find the closest part of the + // cached table and integrate forward from there. + if (sample_number < stsc_first_chunk_sample_) { + if (!stsc_SlipCacheToSample(sample_number, 0)) { + return false; + } + } + + // sample_number could also be well head of our current piece of the + // cache, so see if we can reuse any previously calculated summations to + // skip to the nearest cache entry + int next_cache_index = (stsc_table_index_ / stsc_->GetCacheSizeEntries()) + 1; + if ((next_cache_index < stsc_sample_sums_.size()) && + (sample_number >= stsc_sample_sums_[next_cache_index])) { + if (!stsc_SlipCacheToSample(sample_number, next_cache_index)) { + return false; + } + } + + // Integrate through each table entry until we find sample_number in range + while (stsc_next_first_chunk_sample_ <= sample_number) { + // advance to next chunk sample range + stsc_first_chunk_sample_ = stsc_next_first_chunk_sample_; + // our next_first_chunk is now our first chunk + stsc_first_chunk_ = stsc_next_first_chunk_; + // advance to next entry in table + stsc_table_index_++; + // if we've advanced to a new segment of the cache, update the saved + // integration values + if (!(stsc_table_index_ % stsc_->GetCacheSizeEntries())) { + int cache_index = stsc_table_index_ / stsc_->GetCacheSizeEntries(); + // check that this is our first time with these data + if (cache_index == stsc_sample_sums_.size()) { + stsc_sample_sums_.push_back(stsc_first_chunk_sample_); + } + // our integration at this point should always match any stored record + DCHECK_EQ(stsc_first_chunk_sample_, stsc_sample_sums_[cache_index]); + } + if (stsc_table_index_ < stsc_->GetEntryCount()) { + // look up our new sample rate + if (!stsc_->ReadU32PairEntry(stsc_table_index_, NULL, + &stsc_samples_per_chunk_)) { + return false; + } + // we need to look up next table entry to determine next first chunk + if (stsc_table_index_ + 1 < stsc_->GetEntryCount()) { + // look up next first chunk + if (!stsc_->ReadU32PairEntry(stsc_table_index_ + 1, + &stsc_next_first_chunk_, NULL)) { + return false; + } + --stsc_next_first_chunk_; + // carry sum of first_samples forward to next chunk range + stsc_next_first_chunk_sample_ += + (stsc_next_first_chunk_ - stsc_first_chunk_) * + stsc_samples_per_chunk_; + } else { + // this is the normal place to encounter the end of the chunk table. + // set the next chunk to the highest valid chunk number + stsc_next_first_chunk_ = UINT32_MAX; + stsc_next_first_chunk_sample_ = UINT32_MAX; + } + } else { + // We should normally encounter the end of the chunk table on lookup + // of the next_first_chunk_ within the if clause associated with this + // else. Something has gone wrong. + NOTREACHED(); + return false; + } + } + return true; +} + +bool MP4Map::stsc_SlipCacheToSample(uint32_t sample_number, + int starting_cache_index) { + DCHECK_LT(starting_cache_index, stsc_sample_sums_.size()); + // look through old sample sums for the first entry that exceeds sample + // sample_number, we want the entry right before that + int cache_index = starting_cache_index; + for (; cache_index + 1 < stsc_sample_sums_.size(); cache_index++) { + if (sample_number < stsc_sample_sums_[cache_index + 1]) { + break; + } + } + // jump to new spot in table + stsc_first_chunk_sample_ = stsc_sample_sums_[cache_index]; + stsc_table_index_ = cache_index * stsc_->GetCacheSizeEntries(); + if (!stsc_->ReadU32PairEntry(stsc_table_index_, &stsc_first_chunk_, + &stsc_samples_per_chunk_)) { + return false; + } + // load current and next values + --stsc_first_chunk_; + if (stsc_table_index_ + 1 < stsc_->GetEntryCount()) { + if (!stsc_->ReadU32PairEntry(stsc_table_index_ + 1, &stsc_next_first_chunk_, + NULL)) { + return false; + } + --stsc_next_first_chunk_; + stsc_next_first_chunk_sample_ = + stsc_first_chunk_sample_ + + ((stsc_next_first_chunk_ - stsc_first_chunk_) * + stsc_samples_per_chunk_); + } else { + // We seem to have cached an entry 1 entry before end of the table, and + // are seeking to a region contained in that last entry in the table. + stsc_next_first_chunk_ = UINT32_MAX; + stsc_next_first_chunk_sample_ = UINT32_MAX; + } + return true; +} + +// stss is a list of sample numbers that are keyframes. +bool MP4Map::stss_Init() { + int cache_segments = + (stss_->GetEntryCount() / stss_->GetCacheSizeEntries()) + 1; + stss_keyframes_.reserve(cache_segments); + // empty stss means every frame is a keyframe, same as not + // providing one + if (stss_->GetEntryCount() > 0) { + // identify first keyframe from first entry in stss + if (!stss_->ReadU32Entry(0, &stss_last_keyframe_)) { + stss_ = nullptr; + return false; + } + --stss_last_keyframe_; + stss_keyframes_.push_back(stss_last_keyframe_); + stss_next_keyframe_ = stss_last_keyframe_; + stss_table_index_ = 0; + } else { + stss_ = nullptr; + } + + return true; +} + +// advance by one table entry through stss, updating cache if necessary +bool MP4Map::stss_AdvanceStep() { + DCHECK(stss_); + stss_last_keyframe_ = stss_next_keyframe_; + stss_table_index_++; + if (stss_table_index_ < stss_->GetEntryCount()) { + if (!stss_->ReadU32Entry(stss_table_index_, &stss_next_keyframe_)) { + return false; + } + --stss_next_keyframe_; + if (!(stss_table_index_ % stss_->GetCacheSizeEntries())) { + int cache_index = stss_table_index_ / stss_->GetCacheSizeEntries(); + // only add if this is the first time we've encountered this number + if (cache_index == stss_keyframes_.size()) { + stss_keyframes_.push_back(stss_next_keyframe_); + } + DCHECK_EQ(stss_next_keyframe_, stss_keyframes_[cache_index]); + } + } else { + stss_next_keyframe_ = UINT32_MAX; + } + return true; +} + +bool MP4Map::stss_FindNearestKeyframe(uint32_t sample_number) { + DCHECK(stss_); + // it is assumed that there's at least one cache entry created by + // stss_Init(); + DCHECK_GT(stss_keyframes_.size(), 0); + int cache_entry_number = stss_keyframes_.size() - 1; + int total_cache_entries = + (stss_->GetEntryCount() + stss_->GetCacheSizeEntries() - 1) / + stss_->GetCacheSizeEntries(); + // if there's more than one cache entry we can search the cached + // entries for the entry containing our keyframe, otherwise we skip + // directly to the binary search of the single cached entry + if (total_cache_entries > 1) { + // if the sample number resides within the range of cached entries + // we search those to find right table cache entry to load + if (sample_number < stss_keyframes_[cache_entry_number]) { + int lower_bound = 0; + int upper_bound = stss_keyframes_.size(); + // binary search to find range + while (lower_bound <= upper_bound) { + cache_entry_number = lower_bound + ((upper_bound - lower_bound) / 2); + if (sample_number < stss_keyframes_[cache_entry_number]) { + upper_bound = cache_entry_number - 1; + } else { // sample_number >= stss_keyframes_[cache_entry_number] + // if we are at end of list or next cache entry is higher than sample + // number we consider it a match + if (cache_entry_number == stss_keyframes_.size() - 1 || + sample_number < stss_keyframes_[cache_entry_number + 1]) { + break; + } + lower_bound = cache_entry_number + 1; + } + } + } + // We've gotten as close as we can using the cached values and must handle + // two cases. (a) is that we know that sample_number is contained in the + // cache_entry_number, because we know that: + // stts_keyframes_[cache_entry_number] <= sample_number < + // stts_keyframes_[cache_entry_number + 1] + // (b) is that we only know: + // stts_keyframes_[stts_keyframes_.size() - 1] <= sample_number + // because we have not cached an upper bound to sample_number. + // First step is to make (b) in to (a) by advancing through cache entries + // until last table entry in cache > sample_number or until we arrive + // at the cache entry in the table. + while ((cache_entry_number == stss_keyframes_.size() - 1) && + cache_entry_number < total_cache_entries - 1) { + // Use the first key frame in next cache as upper bound. + int next_cached_entry_number = + (cache_entry_number + 1) * stss_->GetCacheSizeEntries(); + uint32_t next_cached_keyframe; + if (!stss_->ReadU32Entry(next_cached_entry_number, + &next_cached_keyframe)) { + return false; + } + --next_cached_keyframe; + // if this keyframe is higher than our sample number we're in the right + // table, stop + if (sample_number < next_cached_keyframe) { + break; + } + // ok, we need to look in to the next cache entry, advance + cache_entry_number++; + int first_table_entry_number = + cache_entry_number * stss_->GetCacheSizeEntries(); + uint32_t first_keyframe_in_cache_entry; + if (!stss_->ReadU32Entry(first_table_entry_number, + &first_keyframe_in_cache_entry)) { + return false; + } + --first_keyframe_in_cache_entry; + // save first entry in keyframe cache + stss_keyframes_.push_back(first_keyframe_in_cache_entry); + } + // make sure we have an upper bound + if (cache_entry_number != total_cache_entries - 1 && + cache_entry_number == stss_keyframes_.size() - 1) { + int next_cached_entry_number = + ((cache_entry_number + 1) * stss_->GetCacheSizeEntries()); + uint32_t next_cached_keyframe; + if (!stss_->ReadU32Entry(next_cached_entry_number, + &next_cached_keyframe)) { + return false; + } + --next_cached_keyframe; + stss_keyframes_.push_back(next_cached_keyframe); + } + // ok, now we assume we are in state (a), and that we're either + // at the end of the table or within the cache entry bounds for our + // sample number + DCHECK(stss_keyframes_[cache_entry_number] <= sample_number && + (cache_entry_number == total_cache_entries - 1 || + sample_number < stss_keyframes_[cache_entry_number + 1])); + } + // binary search within stss cache entry for keyframes bounding sample_number + int lower_bound = cache_entry_number * stss_->GetCacheSizeEntries(); + int upper_bound = std::min(lower_bound + stss_->GetCacheSizeEntries(), + stss_->GetEntryCount()); + + while (lower_bound <= upper_bound) { + stss_table_index_ = lower_bound + ((upper_bound - lower_bound) / 2); + if (!stss_->ReadU32Entry(stss_table_index_, &stss_last_keyframe_)) { + return false; + } + --stss_last_keyframe_; + if (sample_number < stss_last_keyframe_) { + upper_bound = stss_table_index_ - 1; + } else { // sample_number >= last_keyframe + lower_bound = stss_table_index_ + 1; + // if this is the last entry in the table, we can stop here. + if (lower_bound == stss_->GetEntryCount()) { + stss_next_keyframe_ = UINT32_MAX; + break; + } + // load next entry in table, see if we actually found the upper bound + if (!stss_->ReadU32Entry(lower_bound, &stss_next_keyframe_)) { + return false; + } + --stss_next_keyframe_; + if (sample_number < stss_next_keyframe_) { + stss_table_index_ = lower_bound; + break; + } + } + } + return sample_number >= stss_last_keyframe_ && + sample_number < stss_next_keyframe_; +} + +// The stts table has the following per-entry layout: +// uint32_t sample count - number of sequential samples with this duration +// uint32_t sample duration - duration in ticks of this sample range +bool MP4Map::stts_Init() { + int cache_segments = + (stts_->GetEntryCount() / stts_->GetCacheSizeEntries()) + 1; + stts_samples_.reserve(cache_segments); + stts_timestamps_.reserve(cache_segments); + // need at least one entry in valid stts + if (stts_->GetEntryCount() > 0) { + // integration starts at 0 for both cache entries + stts_samples_.push_back(0); + stts_timestamps_.push_back(0); + if (!stts_->ReadU32PairEntry(0, &stts_next_first_sample_, + &stts_sample_duration_)) { + stts_ = nullptr; + return false; + } + stts_first_sample_ = 0; + stts_first_sample_time_ = 0; + stts_next_first_sample_time_ = + stts_next_first_sample_ * stts_sample_duration_; + stts_table_index_ = 0; + } else { + stts_ = nullptr; + } + + return true; +} + +bool MP4Map::stts_AdvanceToSample(uint32_t sample_number) { + DCHECK(stts_); + // sample_number could be before our current sample range, in which case + // we skip to the nearest table entry before sample_number and integrate + // forward to the sample_number again. + if (sample_number < stts_first_sample_) { + if (!stts_SlipCacheToSample(sample_number, 0)) { + return false; + } + } + + // sample number could also be well ahead of this cache segment, if we've + // previously calculated summations ahead let's skip to the correct one + int next_cache_index = (stts_table_index_ / stts_->GetCacheSizeEntries()) + 1; + if ((next_cache_index < stts_samples_.size()) && + (sample_number >= stts_samples_[next_cache_index])) { + if (!stts_SlipCacheToSample(sample_number, next_cache_index)) { + return false; + } + } + + // integrate through the stts until sample_number is within current range + while (stts_next_first_sample_ <= sample_number) { + if (!stts_IntegrateStep()) { + return false; + } + } + return true; +} + +// Move our integration steps to a previously saved entry in the cache tables. +// Searches linearly through the vector of old cached values, so can accept a +// starting index to do the search from. +bool MP4Map::stts_SlipCacheToSample(uint32_t sample_number, + int starting_cache_index) { + DCHECK_LT(starting_cache_index, stts_samples_.size()); + int cache_index = starting_cache_index; + for (; cache_index + 1 < stts_samples_.size(); cache_index++) { + if (sample_number < stts_samples_[cache_index + 1]) { + break; + } + } + stts_first_sample_ = stts_samples_[cache_index]; + stts_first_sample_time_ = stts_timestamps_[cache_index]; + stts_table_index_ = cache_index * stts_->GetCacheSizeEntries(); + uint32_t sample_count; + // read sample count and duration to set next values + if (!stts_->ReadU32PairEntry(stts_table_index_, &sample_count, + &stts_sample_duration_)) { + return false; + } + stts_next_first_sample_ = stts_first_sample_ + sample_count; + stts_next_first_sample_time_ = + stts_first_sample_time_ + (sample_count * stts_sample_duration_); + return true; +} + +bool MP4Map::stts_AdvanceToTime(uint64_t timestamp) { + DCHECK(stts_); + + if (timestamp < stts_first_sample_time_) { + if (!stts_SlipCacheToTime(timestamp, 0)) { + return false; + } + } + + // sample number could also be well ahead of this cache segment, if we've + // previously calculated summations ahead let's skip to the correct one + int next_cache_index = (stts_table_index_ / stts_->GetCacheSizeEntries()) + 1; + if ((next_cache_index < stts_timestamps_.size()) && + (timestamp >= stts_timestamps_[next_cache_index])) { + if (!stts_SlipCacheToTime(timestamp, next_cache_index)) { + return false; + } + } + + // integrate through the stts until sample_number is within current range + while (stts_next_first_sample_time_ <= timestamp) { + if (!stts_IntegrateStep()) { + return false; + } + } + return true; +} + +bool MP4Map::stts_IntegrateStep() { + // advance time to next sample range + uint32_t range_size = stts_next_first_sample_ - stts_first_sample_; + stts_first_sample_time_ += (range_size * stts_sample_duration_); + // advance sample counter to next range + stts_first_sample_ = stts_next_first_sample_; + // bump table counter to next entry + stts_table_index_++; + // see if we just crossed a cache boundary and should cache results + if (!(stts_table_index_ % stts_->GetCacheSizeEntries())) { + int cache_index = stts_table_index_ / stts_->GetCacheSizeEntries(); + // check that this is our first time with these data + if (cache_index == stts_samples_.size()) { + // both tables should always grow together + DCHECK_EQ(stts_samples_.size(), stts_timestamps_.size()); + stts_samples_.push_back(stts_first_sample_); + stts_timestamps_.push_back(stts_first_sample_time_); + } + // our integration at this point should always match any stored record + DCHECK_EQ(stts_first_sample_, stts_samples_[cache_index]); + DCHECK_EQ(stts_first_sample_time_, stts_timestamps_[cache_index]); + } + if (stts_table_index_ < stts_->GetEntryCount()) { + // load next entry data + uint32_t sample_count; + if (!stts_->ReadU32PairEntry(stts_table_index_, &sample_count, + &stts_sample_duration_)) { + return false; + } + // calculate next sample number from range size + stts_next_first_sample_ = stts_first_sample_ + sample_count; + // and load duration of this sample range + stts_next_first_sample_time_ = + stts_first_sample_time_ + (sample_count * stts_sample_duration_); + } else { + // We've gone beyond the range defined by the last entry in the stts. + // this is an error. + highest_valid_sample_number_ = + std::min(highest_valid_sample_number_, stts_first_sample_ - 1); + return false; + } + return true; +} + +bool MP4Map::stts_SlipCacheToTime(uint64_t timestamp, + int starting_cache_index) { + DCHECK_LT(starting_cache_index, stts_timestamps_.size()); + int cache_index = starting_cache_index; + for (; cache_index + 1 < stts_timestamps_.size(); cache_index++) { + if (timestamp < stts_timestamps_[cache_index + 1]) { + break; + } + } + stts_first_sample_ = stts_samples_[cache_index]; + stts_first_sample_time_ = stts_timestamps_[cache_index]; + stts_table_index_ = cache_index * stts_->GetCacheSizeEntries(); + // read sample count and duration to set next values + uint32_t sample_count; + if (!stts_->ReadU32PairEntry(stts_table_index_, &sample_count, + &stts_sample_duration_)) { + return false; + } + stts_next_first_sample_ = stts_first_sample_ + sample_count; + stts_next_first_sample_time_ = + stts_first_sample_time_ + (sample_count * stts_sample_duration_); + return true; +} + +bool MP4Map::stsz_Init() { + return stsz_->GetBytesAtEntry(0) != NULL; +} + +} // namespace media diff --git a/media/starboard/progressive/mp4_map.h b/media/starboard/progressive/mp4_map.h new file mode 100644 index 000000000000..626ff8458034 --- /dev/null +++ b/media/starboard/progressive/mp4_map.h @@ -0,0 +1,218 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_MP4_MAP_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_MP4_MAP_H_ + +#include + +#include "base/functional/callback.h" +#include "base/memory/ref_counted.h" +#include "media/starboard/progressive/data_source_reader.h" + +namespace media { + +// per-atom sizes of individual table entries +static const int kEntrySize_co64 = 8; +static const int kEntrySize_ctts = 8; +static const int kEntrySize_stco = 4; +static const int kEntrySize_stts = 8; +static const int kEntrySize_stsc = 12; +static const int kEntrySize_stss = 4; +static const int kEntrySize_stsz = 4; + +// Utility class to parse the various subatoms of the stbl mp4 atom and use +// them to provide byte offsets, sizes, and timestamps of a mp4 atom. The +// caching design benefits from, but does not require, sequential access +// in sample numbers. +class MP4Map : public base::RefCountedThreadSafe { + public: + explicit MP4Map(scoped_refptr reader); + + bool IsComplete(); + + // All Get() methods return true on success and save their values by + // reference in the latter argument. + bool GetSize(uint32_t sample_number, uint32_t* size_out); + bool GetOffset(uint32_t sample_number, uint64_t* offset_out); + // all time values in *ticks* as defined by the mp4 container + bool GetTimestamp(uint32_t sample_number, uint64_t* timestamp_out); + bool GetDuration(uint32_t sample_number, uint32_t* duration_out); + bool GetIsKeyframe(uint32_t sample_number, bool* is_keyframe_out); + // Used to determine if the failure reported by any of the above methods + // is due to EOS or other (fatal) error. The length of a mp4 file in samples + // may not be known until iterating through almost the entire map, in the + // case of a default sample size (rare in compressed media) + bool IsEOS(uint32_t sample_number); + + // Returns the keyframe sample number nearest the provided timestamp + bool GetKeyframe(uint64_t timestamp, uint32_t* sample_out); + + // pass 0 as cache_size_entries to force caching of the entire map. + bool SetAtom( + uint32_t four_cc, // fourCC code ascii code as big-endian uint32_t + uint64_t offset, // offset of atom body in file + uint64_t size, // total size of atom in bytes + uint32_t cache_size_entries, // num of entries to cache in memory + const uint8_t* atom); // pointer to atom body start + + private: + bool co64_Init(); + + bool ctts_Init(); + // advance the ctts cache and integration state to contain sample number. + bool ctts_AdvanceToSample(uint32_t sample_number); + bool ctts_SlipCacheToSample(uint32_t sample_number, int starting_cache_index); + + bool stco_Init(); + + bool stsc_Init(); + // advance the stsc cache and integration state to contain sample number. + bool stsc_AdvanceToSample(uint32_t sample_number); + // reuse previously calculated sums to jump through the table to get to the + // nearest cache entry that contains given sample number. Starts the search + // from the starting_cache_index. + bool stsc_SlipCacheToSample(uint32_t sample_number, int starting_cache_index); + + bool stss_Init(); + // step through table by one table entry, return false on error + bool stss_AdvanceStep(); + // search for nearest keyframe, update state to contain it + bool stss_FindNearestKeyframe(uint32_t sample_number); + + bool stts_Init(); + bool stts_AdvanceToSample(uint32_t sample_number); + bool stts_SlipCacheToSample(uint32_t sample_number, int starting_cache_index); + bool stts_AdvanceToTime(uint64_t timestamp); + bool stts_SlipCacheToTime(uint64_t timestamp, int starting_cache_index); + // step through the stts table by one table entry, return false on error + bool stts_IntegrateStep(); + + bool stsz_Init(); + + // TableCache manages the caching of each atom table in a separate instance. + // As each atom has a different per-entry byte size, and may want different + // caching behavior based on consumption rate of entries and the overall size + // of the table, it allows each atom to use its own policy for caching. + // To keep things relatively simple it always keep a table of size of the + // minimum of cache_size_entries or entry_count in memory, and that cached + // table is always aligned with the full table in cache_size_entries, so + // that the position in the cache is trivially calculated from + // entry_number % cache_size_entries, and the cache index is similarly + // calculated from entry_number / cache_size_entries. + class TableCache : public base::RefCountedThreadSafe { + public: + TableCache( + uint64_t table_offset, // byte offset of start of table in file + uint32_t entry_count, // number of entries in table + uint32_t entry_size, // size in bytes of each entry in table + uint32_t cache_size_entries, // number of entries to cache in mem + scoped_refptr reader); // reader to use + + // The following Read* functions all read values in big endian. + bool ReadU32Entry(uint32_t entry_number, uint32_t* entry); + bool ReadU32PairEntry(uint32_t entry_number, + uint32_t* first, + uint32_t* second); + bool ReadU32EntryIntoU64(uint32_t entry_number, uint64_t* entry); + bool ReadU64Entry(uint32_t entry_number, uint64_t* entry); + + uint8_t* GetBytesAtEntry(uint32_t entry_number); + + // how many entries total in the table? + inline uint32_t GetEntryCount() const { return entry_count_; } + // how many entries are we caching in memory at once? + inline uint32_t GetCacheSizeEntries() const { return cache_size_entries_; } + + private: + friend class base::RefCountedThreadSafe; + uint32_t entry_size_; // size of entry in bytes + uint32_t entry_count_; // size of table in entries + uint32_t cache_size_entries_; // max number of entries to fit in memory + uint64_t table_offset_; // offset of table in stream + scoped_refptr reader_; // means to read more table + + // current cache state + std::vector cache_; // the cached part of the table + uint32_t cache_first_entry_number_; // first table entry number in cache + uint32_t cache_entry_count_; // number of valid entries in cache + }; + + scoped_refptr reader_; + + // current integration state for GetOffset(), we save the sum of sample sizes + // within the current chunk. + uint32_t current_chunk_sample_; // sample number last included in summation + uint32_t next_chunk_sample_; // first sample number of next chunk + uint64_t current_chunk_offset_; // file byte offset of current_chunk_sample_ + + // Can be set by a stsz entry count but an stsz may provide a default size, + // in which case this number may not be known until iteration through + // the ctts, or stts has completed. In the event that one of those tables + // ends at a lower number than the others this number will be amended + // to return the lower number. + uint32_t highest_valid_sample_number_; + + // ==== c064 - per-chunk list of file offsets (64-bit) + scoped_refptr co64_; + + // ==== ctts - run-length sample number to composition time offset + scoped_refptr ctts_; + uint32_t ctts_first_sample_; + uint32_t ctts_sample_offset_; + uint32_t ctts_next_first_sample_; + uint32_t ctts_table_index_; + std::vector ctts_samples_; + + // ==== stco - per-chunk list of chunk file offsets (32-bit) + scoped_refptr stco_; + + // ==== stsc - chunk-number to samples-per-chunk + scoped_refptr stsc_; + uint32_t stsc_first_chunk_; // first chunk of the current sample size range + uint32_t stsc_first_chunk_sample_; // sum of samples of all prev chunk ranges + uint32_t stsc_samples_per_chunk_; // current samples-per-chunk in this range + uint32_t + stsc_next_first_chunk_; // the chunk number the next region begins in + uint32_t + stsc_next_first_chunk_sample_; // sample number next region begins in + uint32_t stsc_table_index_; // the index in the table of the current range + std::vector stsc_sample_sums_; // saved sums of cache segments + + // ==== stss - list of keyframe sample numbers + scoped_refptr stss_; + uint32_t stss_last_keyframe_; + uint32_t stss_next_keyframe_; + uint32_t stss_table_index_; // index of stss_next_keyframe_ in table + std::vector stss_keyframes_; + + // ==== stts - run-length sample number to time duration + scoped_refptr stts_; + uint32_t stts_first_sample_; // first sample of the current duration range + uint64_t stts_first_sample_time_; // sum of all durations of previous ranges + uint32_t stts_sample_duration_; // current duration of samples in this range + uint32_t stts_next_first_sample_; // first sample number of next range + uint64_t stts_next_first_sample_time_; // first timestamp of next range + uint32_t stts_table_index_; // index in the table of the next entry + std::vector stts_samples_; + std::vector stts_timestamps_; + + // ==== stsz - per-sample list of sample sizes + scoped_refptr stsz_; + uint32_t stsz_default_size_; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_MP4_MAP_H_ diff --git a/media/starboard/progressive/mp4_map_unittest.cc b/media/starboard/progressive/mp4_map_unittest.cc new file mode 100644 index 000000000000..6edda94a3817 --- /dev/null +++ b/media/starboard/progressive/mp4_map_unittest.cc @@ -0,0 +1,1121 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/mp4_map.h" + +#include // for rand and srand + +#include // for std::min +#include +#include +#include +#include + +#include "media/starboard/progressive/endian_util.h" +#include "media/starboard/progressive/mock_data_source_reader.h" +#include "media/starboard/progressive/mp4_parser.h" +#include "starboard/memory.h" +#include "starboard/types.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using load_uint32_big_endian; +using store_uint32_big_endian; +using store_uint64_big_endian; + +using ::testing::_; +using ::testing::AllOf; +using ::testing::AnyNumber; +using ::testing::DoAll; +using ::testing::Ge; +using ::testing::Invoke; +using ::testing::Lt; +using ::testing::Return; +using ::testing::SetArrayArgument; + +namespace { + +int RandomRange(int min, int max) { + return min + rand() % (max - min + 1); +} + +// Data structure represent a sample inside stbl. It has redundant data for +// easy access. +struct Sample { + bool is_key_frame; + int size; + int offset; + int chunk_index; + int dts_duration; + int dts; + int cts; +}; + +typedef std::vector SampleVector; + +class SampleTable { + public: + // All ranges are inclusive at both ends. + // Set range of composition timestamp to [0, 0] to disable ctts. + SampleTable(unsigned int seed, + int num_of_samples, + int min_sample_size, + int max_sample_size, + int min_samples_per_chunk, + int max_samples_per_chunk, + int min_key_frame_gap, + int max_key_frame_gap, + int min_sample_decode_timestamp_offset, + int max_sample_decode_timestamp_offset, + int min_sample_composition_timestamp_offset, + int max_sample_composition_timestamp_offset) + : read_count_(0), read_bytes_(0) { + srand(seed); + CHECK_GT(num_of_samples, 0); + CHECK_GT(min_sample_size, 0); + CHECK_LE(min_sample_size, max_sample_size); + CHECK_GT(min_samples_per_chunk, 0); + CHECK_LE(min_samples_per_chunk, max_samples_per_chunk); + CHECK_GT(min_key_frame_gap, 0); + CHECK_LE(min_key_frame_gap, max_key_frame_gap); + CHECK_GT(min_sample_decode_timestamp_offset, 0); + CHECK_LE(min_sample_decode_timestamp_offset, + max_sample_decode_timestamp_offset); + CHECK_LE(min_sample_composition_timestamp_offset, + max_sample_composition_timestamp_offset); + + samples_.resize(num_of_samples); + + int remaining_sample_in_chunk = 0; + int current_chunk_index = -1; + int next_key_frame = 0; + + for (int i = 0; i < num_of_samples; ++i) { + samples_[i].size = RandomRange(min_sample_size, max_sample_size); + samples_[i].offset = + i == 0 ? rand() : samples_[i - 1].offset + samples_[i - 1].size; + samples_[i].dts = + i == 0 ? 0 : samples_[i - 1].dts + samples_[i - 1].dts_duration; + samples_[i].dts_duration = + RandomRange(min_sample_decode_timestamp_offset, + max_sample_decode_timestamp_offset); + + samples_[i].cts = samples_[i].dts + + RandomRange(min_sample_composition_timestamp_offset, + max_sample_composition_timestamp_offset); + if (!remaining_sample_in_chunk) { + ++current_chunk_index; + remaining_sample_in_chunk = + RandomRange(min_samples_per_chunk, max_samples_per_chunk); + } + + if (i >= next_key_frame) { + samples_[i].is_key_frame = true; + next_key_frame += RandomRange(min_key_frame_gap, max_key_frame_gap); + } else { + samples_[i].is_key_frame = false; + } + + samples_[i].chunk_index = current_chunk_index; + --remaining_sample_in_chunk; + } + + PopulateBoxes(); + } + + int64 GetBoxOffset(uint32 atom_type) const { + switch (atom_type) { + case kAtomType_stsz: + return stsz_offset_; + case kAtomType_stco: + return stco_offset_; + case kAtomType_co64: + return co64_offset_; + case kAtomType_stsc: + return stsc_offset_; + case kAtomType_ctts: + return ctts_offset_; + case kAtomType_stts: + return stts_offset_; + case kAtomType_stss: + return stss_offset_; + default: + NOTREACHED(); + return 0; + } + } + + int64 GetBoxSize(uint32 atom_type) const { + switch (atom_type) { + case kAtomType_stsz: + return stsz_.size(); + case kAtomType_stco: + return stco_.size(); + case kAtomType_co64: + return co64_.size(); + case kAtomType_stsc: + return stsc_.size(); + case kAtomType_ctts: + return ctts_.size(); + case kAtomType_stts: + return stts_.size(); + case kAtomType_stss: + return stss_.size(); + default: + NOTREACHED(); + return 0; + } + } + + const uint8_t* GetBoxData(uint32 atom_type) const { + return &combined_[0] + GetBoxOffset(atom_type) - file_offset_; + } + + size_t sample_count() const { return samples_.size(); } + const Sample& sample(int i) const { return samples_.at(i); } + + size_t keyframe_count() const { return (stss_.size() - 8) / kEntrySize_stss; } + + int BlockingRead(int64 position, int size, uint8* data) { + CHECK_GE(position, file_offset_); + CHECK_LE(position + size, file_offset_ + combined_.size()); + uint32 offset = position - file_offset_; + memcpy(data, &combined_[0] + offset, size); + ++read_count_; + read_bytes_ += size; + return size; + } + + void ClearReadStatistics() { + read_count_ = 0; + read_bytes_ = 0; + } + + int read_count() const { return read_count_; } + int read_bytes() const { return read_bytes_; } + + void Dump() const { + std::stringstream ss; + uint32 boxes[] = {kAtomType_stsz, kAtomType_stco, kAtomType_co64, + kAtomType_stsc, kAtomType_ctts, kAtomType_stts, + kAtomType_stss}; + const char* names[] = {"stsz", "stco", "co64", "stsc", + "ctts", "stts", "stss"}; + for (uint32 i = 0; i < sizeof(boxes) / sizeof(*boxes); ++i) { + ss << "\n======================== " << names[i] + << " ========================\n"; + int64 size = GetBoxSize(boxes[i]); + const uint8_t* data = GetBoxData(boxes[i]); + for (int64 j = 0; j < size; ++j) { + ss << static_cast(data[j]) << ' '; + if (j != 0 && j % 32 == 0) { + ss << '\n'; + } + } + } + LOG(INFO) << ss.str(); + } + + private: + SampleVector samples_; + + int64 file_offset_; + int64 stsz_offset_; + int64 stco_offset_; + int64 co64_offset_; + int64 stsc_offset_; + int64 ctts_offset_; + int64 stts_offset_; + int64 stss_offset_; + + std::vector combined_; + std::vector stsz_; + std::vector stco_; + std::vector co64_; + std::vector stsc_; + std::vector ctts_; + std::vector stts_; + std::vector stss_; + + int read_count_; + int read_bytes_; + + static void inc_uint32_big_endian(uint8_t* data) { + uint32 value = load_uint32_big_endian(data); + store_uint32_big_endian(value + 1, data); + } + + void PopulateBoxes() { + CHECK(!samples_.empty()); + bool all_key_frames = true; + bool all_ctts_offset_is_zero = true; + bool all_sample_has_same_size = true; + + for (SampleVector::const_iterator iter = samples_.begin(); + iter != samples_.end(); ++iter) { + if (!iter->is_key_frame) { + all_key_frames = false; + } + if (iter->dts != iter->cts) { + all_ctts_offset_is_zero = false; + } + if (iter->size != samples_[0].size) { + all_sample_has_same_size = false; + } + } + + // populate the stsz box: 4 bytes flags + 4 bytes default size + // + 4 bytes count + size table if default size is 0 + if (all_sample_has_same_size) { + stsz_.resize(12); + store_uint32_big_endian(samples_[0].size, &stsz_[4]); + } else { + stsz_.resize(samples_.size() * kEntrySize_stsz + 12); + store_uint32_big_endian(0, &stsz_[4]); + + uint8_t* table_offset = &stsz_[12]; + + for (SampleVector::const_iterator iter = samples_.begin(); + iter != samples_.end(); ++iter) { + store_uint32_big_endian(iter->size, table_offset); + table_offset += kEntrySize_stsz; + } + } + store_uint32_big_endian(samples_.size(), &stsz_[8]); + + // populate stco, co64 and stsc + // stco = 4 bytes count + (4 bytes offset)* + // co64 = 4 bytes count + (8 bytes offset)* + // stsc = 4 bytes count + (4 bytes chunk index + 4 bytes sample per chunk + // + 4 bytes sample description index)* + stco_.resize(8); + co64_.resize(8); + stsc_.resize(8); + uint32 chunk_offset = samples_[0].offset; + int current_chunk_index = -1; + for (SampleVector::const_iterator iter = samples_.begin(); + iter != samples_.end(); ++iter) { + if (current_chunk_index != iter->chunk_index) { + current_chunk_index = iter->chunk_index; + stco_.resize(stco_.size() + kEntrySize_stco); + store_uint32_big_endian(chunk_offset, + &stco_[stco_.size() - kEntrySize_stco]); + co64_.resize(co64_.size() + kEntrySize_co64); + store_uint64_big_endian(chunk_offset, + &co64_[co64_.size() - kEntrySize_co64]); + stsc_.resize(stsc_.size() + kEntrySize_stsc); + store_uint32_big_endian(current_chunk_index + 1, // start from 1 + &stsc_[stsc_.size() - kEntrySize_stsc]); + } + inc_uint32_big_endian(&stsc_[stsc_.size() - kEntrySize_stsc + 4]); + chunk_offset += iter->size; + } + store_uint32_big_endian((stco_.size() - 8) / kEntrySize_stco, &stco_[4]); + store_uint32_big_endian((co64_.size() - 8) / kEntrySize_co64, &co64_[4]); + store_uint32_big_endian((stsc_.size() - 8) / kEntrySize_stsc, &stsc_[4]); + + // populate stts and ctts. + // stts = 4 bytes count + (4 bytes sample count + 4 bytes sample delta)* + // ctts = 4 bytes count + (4 bytes sample count + 4 bytes sample offset)* + // the offset is to stts. + stts_.resize(8); + ctts_.resize(all_ctts_offset_is_zero ? 0 : 8); + int32 last_stts_duration = -1; + int32 last_ctts_offset = samples_[0].cts - samples_[0].dts - 1; + + for (size_t i = 0; i < samples_.size(); ++i) { + int32 ctts_offset = samples_[i].cts - samples_[i].dts; + if (last_stts_duration != samples_[i].dts_duration) { + stts_.resize(stts_.size() + kEntrySize_stts); + last_stts_duration = samples_[i].dts_duration; + store_uint32_big_endian(last_stts_duration, &stts_[stts_.size() - 4]); + } + inc_uint32_big_endian(&stts_[stts_.size() - 8]); + if (!all_ctts_offset_is_zero) { + if (last_ctts_offset != ctts_offset) { + ctts_.resize(ctts_.size() + kEntrySize_ctts); + store_uint32_big_endian(ctts_offset, &ctts_[ctts_.size() - 4]); + last_ctts_offset = ctts_offset; + } + inc_uint32_big_endian(&ctts_[ctts_.size() - 8]); + } + } + store_uint32_big_endian((stts_.size() - 8) / kEntrySize_stts, &stts_[4]); + if (!all_ctts_offset_is_zero) { + store_uint32_big_endian((ctts_.size() - 8) / kEntrySize_ctts, &ctts_[4]); + } + + // populate stss box + // stss = 4 bytes count + (4 bytes sample index)* + if (!all_key_frames) { + stss_.resize(8); + for (size_t i = 0; i < samples_.size(); ++i) { + if (samples_[i].is_key_frame) { + stss_.resize(stss_.size() + kEntrySize_stss); + store_uint32_big_endian(i + 1, + &stss_[stss_.size() - kEntrySize_stss]); + } + } + store_uint32_big_endian((stss_.size() - 8) / kEntrySize_stss, &stss_[4]); + } + + const int kGarbageSize = 1024; + std::vector garbage; + garbage.reserve(kGarbageSize); + for (int i = 0; i < kGarbageSize; ++i) { + garbage.push_back(RandomRange(0xef, 0xfe)); + } + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + combined_.insert(combined_.end(), stsz_.begin(), stsz_.end()); + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + combined_.insert(combined_.end(), stco_.begin(), stco_.end()); + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + combined_.insert(combined_.end(), co64_.begin(), co64_.end()); + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + combined_.insert(combined_.end(), stsc_.begin(), stsc_.end()); + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + combined_.insert(combined_.end(), ctts_.begin(), ctts_.end()); + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + combined_.insert(combined_.end(), stts_.begin(), stts_.end()); + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + combined_.insert(combined_.end(), stss_.begin(), stss_.end()); + combined_.insert(combined_.end(), garbage.begin(), garbage.end()); + + file_offset_ = abs(rand()); + stsz_offset_ = file_offset_ + kGarbageSize; + stco_offset_ = stsz_offset_ + stsz_.size() + kGarbageSize; + co64_offset_ = stco_offset_ + stco_.size() + kGarbageSize; + stsc_offset_ = co64_offset_ + co64_.size() + kGarbageSize; + ctts_offset_ = stsc_offset_ + stsc_.size() + kGarbageSize; + stts_offset_ = ctts_offset_ + ctts_.size() + kGarbageSize; + stss_offset_ = stts_offset_ + stts_.size() + kGarbageSize; + } +}; + +class MP4MapTest : public testing::Test { + protected: + MP4MapTest() { + // make a new mock reader + reader_ = new ::testing::NiceMock(); + // make a new map with a mock reader. + map_ = new MP4Map(reader_); + } + + virtual ~MP4MapTest() { + DCHECK(map_->HasOneRef()); + map_ = NULL; + + reader_->Stop(); + DCHECK(reader_->HasOneRef()); + reader_ = NULL; + } + + void ResetMap() { map_ = new MP4Map(reader_); } + + void CreateTestSampleTable(unsigned int seed, + int num_of_samples, + int min_sample_size, + int max_sample_size, + int min_samples_per_chunk, + int max_samples_per_chunk, + int min_key_frame_gap, + int max_key_frame_gap, + int min_sample_decode_timestamp_offset, + int max_sample_decode_timestamp_offset, + int min_sample_composition_timestamp_offset, + int max_sample_composition_timestamp_offset) { + sample_table_.reset(new SampleTable( + seed, num_of_samples, min_sample_size, max_sample_size, + min_samples_per_chunk, max_samples_per_chunk, min_key_frame_gap, + max_key_frame_gap, min_sample_decode_timestamp_offset, + max_sample_decode_timestamp_offset, + min_sample_composition_timestamp_offset, + max_sample_composition_timestamp_offset)); + ON_CALL(*reader_, BlockingRead(_, _, _)) + .WillByDefault(Invoke(sample_table_.get(), &SampleTable::BlockingRead)); + } + + void SetTestTable(uint32 four_cc, uint32 cache_size_entries) { + map_->SetAtom(four_cc, sample_table_->GetBoxOffset(four_cc), + sample_table_->GetBoxSize(four_cc), cache_size_entries, + sample_table_->GetBoxData(four_cc)); + } + + const Sample& GetTestSample(uint32 sample_number) const { + return sample_table_->sample(sample_number); + } + + // ==== Test Fixture Members + scoped_refptr map_; + scoped_refptr reader_; + std::unique_ptr sample_table_; +}; + +// ==== SetAtom() Tests ======================================================== +/* +TEST_F(MP4MapTest, SetAtomWithZeroDefaultSize) { + // SetAtom() should fail with a zero default size on an stsc. + NOTIMPLEMENTED(); +} +*/ +// ==== GetSize() Tests ======================================================== + +TEST_F(MP4MapTest, GetSizeWithDefaultSize) { + CreateTestSampleTable(100, 1000, 0xb0df00d, 0xb0df00d, 5, 10, 5, 10, 10, 20, + 10, 20); + sample_table_->ClearReadStatistics(); + + for (int i = 13; i < 21; ++i) { + ResetMap(); + SetTestTable(kAtomType_stsz, i); + + uint32 returned_size; + ASSERT_TRUE(map_->GetSize(0, &returned_size)); + ASSERT_EQ(returned_size, 0xb0df00d); + ASSERT_FALSE(map_->GetSize(2000, &returned_size)); + ASSERT_TRUE(map_->GetSize(2, &returned_size)); + ASSERT_EQ(returned_size, 0xb0df00d); + ASSERT_TRUE(map_->GetSize(120, &returned_size)); + ASSERT_EQ(returned_size, 0xb0df00d); + } + + ASSERT_EQ(sample_table_->read_count(), 0); +} + +TEST_F(MP4MapTest, GetSizeIterationWithHugeCache) { + for (int max_sample_size = 10; max_sample_size < 20; ++max_sample_size) { + CreateTestSampleTable(200 + max_sample_size, 1000, 10, max_sample_size, 5, + 10, 5, 10, 10, 20, 10, 20); + for (int i = 1500; i < 10000; i = i * 2 + 1) { + ResetMap(); + sample_table_->ClearReadStatistics(); + SetTestTable(kAtomType_stsz, i); + + for (uint32 j = 0; j < sample_table_->sample_count(); j++) { + uint32 map_reported_size = 0; + ASSERT_TRUE(map_->GetSize(j, &map_reported_size)); + uint32 table_size = GetTestSample(j).size; + // reported size should match table size + ASSERT_EQ(map_reported_size, table_size); + } + + // call to a sample past the size of the table should fail + uint32 failed_size = 0; + ASSERT_FALSE(map_->GetSize(sample_table_->sample_count(), &failed_size)); + ASSERT_LE(sample_table_->read_count(), 1); + ASSERT_LE(sample_table_->read_bytes(), + sample_table_->GetBoxSize(kAtomType_stsz)); + } + } +} + +TEST_F(MP4MapTest, GetSizeIterationTinyCache) { + for (int max_sample_size = 10; max_sample_size < 20; ++max_sample_size) { + CreateTestSampleTable(300 + max_sample_size, 1000, 10, max_sample_size, 5, + 10, 5, 10, 10, 20, 10, 20); + for (int i = 5; i < 12; ++i) { + ResetMap(); + SetTestTable(kAtomType_stsz, i); + sample_table_->ClearReadStatistics(); + for (uint32 j = 0; j < sample_table_->sample_count(); j++) { + uint32 map_reported_size = 0; + ASSERT_TRUE(map_->GetSize(j, &map_reported_size)); + uint32 table_size = GetTestSample(j).size; + ASSERT_EQ(map_reported_size, table_size); + } + ASSERT_LE(sample_table_->read_count(), + sample_table_->sample_count() / i + 1); + if (sample_table_->read_count()) { + ASSERT_LE(sample_table_->read_bytes() / sample_table_->read_count(), + (i + 1) * kEntrySize_stsz); + } + // call to sample past the table size should still fail + uint32 failed_size = 0; + ASSERT_FALSE(map_->GetSize(sample_table_->sample_count(), &failed_size)); + } + } +} + +TEST_F(MP4MapTest, GetSizeRandomAccess) { + CreateTestSampleTable(101, 2000, 20, 24, 5, 10, 5, 10, 10, 20, 10, 20); + for (int i = 24; i < 27; ++i) { + ResetMap(); + SetTestTable(kAtomType_stsz, i); + sample_table_->ClearReadStatistics(); + // test first sample query somewhere later in the table, sample 105 + uint32 map_reported_size = 0; + ASSERT_TRUE(map_->GetSize(i * 4 + 5, &map_reported_size)); + uint32 table_size = GetTestSample(i * 4 + 5).size; + ASSERT_EQ(map_reported_size, table_size); + ASSERT_EQ(sample_table_->read_count(), 1); + ASSERT_LE(sample_table_->read_bytes(), (i + 1) * kEntrySize_stsz); + + // now jump back to sample 0 + sample_table_->ClearReadStatistics(); + ASSERT_TRUE(map_->GetSize(0, &map_reported_size)); + table_size = GetTestSample(0).size; + ASSERT_EQ(map_reported_size, table_size); + ASSERT_EQ(sample_table_->read_count(), 1); + ASSERT_LE(sample_table_->read_bytes(), (i + 1) * kEntrySize_stsz); + + // now seek well past the end, this query should fail but not break + // subsequent queries or issue a recache + ASSERT_FALSE( + map_->GetSize(sample_table_->sample_count(), &map_reported_size)); + + // a query back within the first table should not cause recache + ASSERT_TRUE(map_->GetSize(10, &map_reported_size)); + table_size = GetTestSample(10).size; + ASSERT_EQ(map_reported_size, table_size); + + // check sample queries right on cache boundaries out-of-order + sample_table_->ClearReadStatistics(); + ASSERT_TRUE(map_->GetSize(2 * i, &map_reported_size)); + table_size = GetTestSample(2 * i).size; + ASSERT_EQ(map_reported_size, table_size); + ASSERT_EQ(sample_table_->read_count(), 1); + ASSERT_TRUE(map_->GetSize(2 * i - 1, &map_reported_size)); + table_size = GetTestSample(2 * i - 1).size; + ASSERT_EQ(map_reported_size, table_size); + ASSERT_TRUE(map_->GetSize(2 * i - 2, &map_reported_size)); + table_size = GetTestSample(2 * i - 2).size; + ASSERT_EQ(map_reported_size, table_size); + ASSERT_EQ(sample_table_->read_count(), 2); + } +} + +// ==== GetOffset() Tests ====================================================== + +TEST_F(MP4MapTest, GetOffsetIterationHugeCache) { + for (int coindex = 0; coindex < 2; ++coindex) { + CreateTestSampleTable(102 + coindex, 1000, 20, 25, 5, 10, 5, 10, 10, 20, 10, + 20); + ResetMap(); + SetTestTable(kAtomType_stsz, 1000); + SetTestTable(kAtomType_stsc, 1000); + SetTestTable(coindex ? kAtomType_stco : kAtomType_co64, 1000); + + // no expectations on reader_, all tables should now be in memory + for (uint32 i = 0; i < sample_table_->sample_count(); ++i) { + uint64 map_reported_offset = 0; + ASSERT_TRUE(map_->GetOffset(i, &map_reported_offset)); + uint64 table_offset = GetTestSample(i).offset; + ASSERT_EQ(map_reported_offset, table_offset); + } + + // calls to sample numbers outside file range should fail non-fatally + uint64 failed_offset; + ASSERT_FALSE( + map_->GetOffset(sample_table_->sample_count(), &failed_offset)); + } +} + +TEST_F(MP4MapTest, GetOffsetIterationTinyCache) { + for (int coindex = 0; coindex < 2; ++coindex) { + CreateTestSampleTable(103, 30, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + for (int i = 1; i < 12; ++i) { + ResetMap(); + SetTestTable(kAtomType_stsz, i); + SetTestTable(kAtomType_stsc, i); + SetTestTable(coindex ? kAtomType_stco : kAtomType_co64, i); + + // iterate through all samples in range + for (uint32 j = 0; j < sample_table_->sample_count(); j += 2) { + uint64 map_reported_offset = 0; + ASSERT_TRUE(map_->GetOffset(j, &map_reported_offset)); + uint64 table_offset = GetTestSample(j).offset; + ASSERT_EQ(map_reported_offset, table_offset); + } + + // calls to sample numbers outside file range should fail non-fatally + uint64 failed_offset; + ASSERT_FALSE( + map_->GetOffset(sample_table_->sample_count(), &failed_offset)); + } + } +} + +// Random access within cache should just result in correct re-integration +// through the stsc. +TEST_F(MP4MapTest, GetOffsetRandomAccessHugeCache) { + for (int coindex = 0; coindex < 2; ++coindex) { + CreateTestSampleTable(104, 300, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stsz, 300); + SetTestTable(kAtomType_stsc, 300); + SetTestTable(coindex ? kAtomType_stco : kAtomType_co64, 300); + + for (int i = 0; i < 1000; ++i) { + uint32 sample_number = rand() % sample_table_->sample_count(); + uint64 map_reported_offset = 0; + ASSERT_TRUE(map_->GetOffset(sample_number, &map_reported_offset)); + uint64 table_offset = GetTestSample(sample_number).offset; + ASSERT_EQ(map_reported_offset, table_offset); + } + } +} + +// Random access across cache boundaries should not break computation of +// offsets. +TEST_F(MP4MapTest, GetOffsetRandomAccessTinyCache) { + for (int coindex = 0; coindex < 2; ++coindex) { + CreateTestSampleTable(105, 300, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stsz, 7); + SetTestTable(kAtomType_stsc, 7); + SetTestTable(coindex ? kAtomType_stco : kAtomType_co64, 7); + + // calls to sample numbers outside file range should fail non-fatally + uint64 failed_offset; + ASSERT_FALSE( + map_->GetOffset(sample_table_->sample_count(), &failed_offset)); + + // second sample in the file + uint32 sample_number = 1; + uint64 map_reported_offset = 0; + ASSERT_TRUE(map_->GetOffset(sample_number, &map_reported_offset)); + uint64 table_offset = GetTestSample(sample_number).offset; + ASSERT_EQ(map_reported_offset, table_offset); + + for (int i = 1; i < 15; ++i) { + ResetMap(); + SetTestTable(kAtomType_stsz, 7); + SetTestTable(kAtomType_stsc, 7); + SetTestTable(kAtomType_stco, 7); + + sample_number = sample_table_->sample_count() - i; + ASSERT_TRUE(map_->GetOffset(sample_number, &map_reported_offset)); + table_offset = GetTestSample(sample_number).offset; + ASSERT_EQ(map_reported_offset, table_offset); + + sample_number--; + ASSERT_TRUE(map_->GetOffset(sample_number, &map_reported_offset)); + table_offset = GetTestSample(sample_number).offset; + ASSERT_EQ(map_reported_offset, table_offset); + + // now iterate through a few samples in the middle + sample_number /= 2; + for (int j = 0; j < 40; j++) { + ASSERT_TRUE(map_->GetOffset(sample_number + j, &map_reported_offset)); + table_offset = GetTestSample(sample_number + j).offset; + ASSERT_EQ(map_reported_offset, table_offset); + } + + // now iterate backwards from the same starting point + for (int j = 0; j < 40; j++) { + ASSERT_TRUE(map_->GetOffset(sample_number - j, &map_reported_offset)); + table_offset = GetTestSample(sample_number - j).offset; + ASSERT_EQ(map_reported_offset, table_offset); + } + } + } +} + +TEST_F(MP4MapTest, GetOffsetRandomAccessWithDefaultSize) { + for (int coindex = 0; coindex < 2; ++coindex) { + CreateTestSampleTable(106, 300, 20, 20, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stsz, 7); + SetTestTable(kAtomType_stsc, 7); + SetTestTable(coindex ? kAtomType_stco : kAtomType_co64, 7); + + // Calculating offset of an out-of-range sample should still return an + // error. + uint64 map_reported_offset = 0; + ASSERT_FALSE(map_->GetOffset(sample_table_->sample_count() + 2, + &map_reported_offset)); + + // First sample in file should still work, though. + ASSERT_TRUE(map_->GetOffset(0, &map_reported_offset)); + uint64 table_offset = GetTestSample(0).offset; + ASSERT_EQ(map_reported_offset, table_offset); + + // Last sample should also work. + ASSERT_TRUE(map_->GetOffset(sample_table_->sample_count() - 1, + &map_reported_offset)); + table_offset = GetTestSample(sample_table_->sample_count() - 1).offset; + ASSERT_EQ(map_reported_offset, table_offset); + + // Skip by 3 through the file a few times + for (int i = 0; i < sample_table_->sample_count(); ++i) { + int sample_index = (i * 3) % sample_table_->sample_count(); + ASSERT_TRUE(map_->GetOffset(sample_index, &map_reported_offset)); + table_offset = GetTestSample(sample_index).offset; + ASSERT_EQ(map_reported_offset, table_offset); + } + } +} + +// ==== GetDuration() Tests ==================================================== + +TEST_F(MP4MapTest, GetDurationIteration) { + CreateTestSampleTable(107, 60, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stts, 2); + + for (uint32 i = 0; i < sample_table_->sample_count(); ++i) { + uint32 map_reported_duration = 0; + ASSERT_TRUE(map_->GetDuration(i, &map_reported_duration)); + uint32 table_duration = GetTestSample(i).dts_duration; + ASSERT_EQ(map_reported_duration, table_duration); + } + + // entries past end of table should fail + uint32 failed_duration = 0; + ASSERT_FALSE( + map_->GetDuration(sample_table_->sample_count(), &failed_duration)); +} + +TEST_F(MP4MapTest, GetDurationRandomAccess) { + CreateTestSampleTable(108, 60, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stts, 3); + + // first sample in table + uint32 map_reported_duration = 0; + ASSERT_TRUE(map_->GetDuration(0, &map_reported_duration)); + uint32 table_duration = GetTestSample(0).dts_duration; + ASSERT_EQ(map_reported_duration, table_duration); + + // last sample in table + ASSERT_TRUE(map_->GetDuration(sample_table_->sample_count() - 1, + &map_reported_duration)); + table_duration = + GetTestSample(sample_table_->sample_count() - 1).dts_duration; + ASSERT_EQ(map_reported_duration, table_duration); + + // sample just past end should fail + ASSERT_FALSE( + map_->GetDuration(sample_table_->sample_count(), &map_reported_duration)); + + // but shouldn't break other sample lookups + ASSERT_TRUE(map_->GetDuration(2, &map_reported_duration)); + table_duration = GetTestSample(2).dts_duration; + ASSERT_EQ(map_reported_duration, table_duration); + + // now iterate backwards through entire table back to first sample + for (int i = sample_table_->sample_count() - 1; i >= 1; i--) { + ASSERT_TRUE(map_->GetDuration(i, &map_reported_duration)); + table_duration = GetTestSample(i).dts_duration; + ASSERT_EQ(map_reported_duration, table_duration); + } +} + +// ==== GetTimestamp() Tests =================================================== + +TEST_F(MP4MapTest, GetTimestampIterationNoCompositionTime) { + CreateTestSampleTable(109, 60, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stts, 7); + + for (uint32 i = 0; i < sample_table_->sample_count(); ++i) { + uint64 map_reported_timestamp = 0; + ASSERT_TRUE(map_->GetTimestamp(i, &map_reported_timestamp)); + uint64 table_timestamp = GetTestSample(i).dts; + ASSERT_EQ(map_reported_timestamp, table_timestamp); + } + + // entries past end of table should fail + uint64 failed_timestamp = 0; + ASSERT_FALSE( + map_->GetTimestamp(sample_table_->sample_count(), &failed_timestamp)); +} + +TEST_F(MP4MapTest, GetTimestampRandomAccessNoCompositionTime) { + CreateTestSampleTable(110, 60, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stts, 10); + + // skip by sevens through the file, seven times + for (int i = 0; i < sample_table_->sample_count(); ++i) { + uint32 sample_number = (i * 7) % sample_table_->sample_count(); + uint64 map_reported_timestamp = 0; + ASSERT_TRUE(map_->GetTimestamp(sample_number, &map_reported_timestamp)); + uint64 table_timestamp = GetTestSample(sample_number).dts; + ASSERT_EQ(map_reported_timestamp, table_timestamp); + } + + // check a failed entry + uint64 failed_timestamp = 0; + ASSERT_FALSE( + map_->GetTimestamp(sample_table_->sample_count() * 2, &failed_timestamp)); + + // should still be able to recover with valid input, this time skip by 21s + // backward through the file 21 times + for (int i = sample_table_->sample_count() - 1; i >= 0; i--) { + uint32 sample_number = (i * 21) % sample_table_->sample_count(); + uint64 map_reported_timestamp = 0; + ASSERT_TRUE(map_->GetTimestamp(sample_number, &map_reported_timestamp)); + uint64 table_timestamp = GetTestSample(sample_number).dts; + ASSERT_EQ(map_reported_timestamp, table_timestamp); + } +} + +TEST_F(MP4MapTest, GetTimestampIteration) { + CreateTestSampleTable(111, 300, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + for (int i = 1; i < 20; ++i) { + ResetMap(); + SetTestTable(kAtomType_ctts, i); + SetTestTable(kAtomType_stts, i); + + for (int j = 0; j < sample_table_->sample_count(); ++j) { + uint64 map_reported_timestamp = 0; + ASSERT_TRUE(map_->GetTimestamp(j, &map_reported_timestamp)); + uint64 table_timestamp = GetTestSample(j).cts; + ASSERT_EQ(map_reported_timestamp, table_timestamp); + + uint32 map_reported_duration = 0; + ASSERT_TRUE(map_->GetDuration(j, &map_reported_duration)); + uint32 table_duration = GetTestSample(j).dts_duration; + ASSERT_EQ(map_reported_duration, table_duration); + } + } +} + +TEST_F(MP4MapTest, GetTimestampRandomAccess) { + CreateTestSampleTable(112, 300, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + for (int i = 1; i < 20; ++i) { + ResetMap(); + SetTestTable(kAtomType_ctts, i); + SetTestTable(kAtomType_stts, i); + + for (int j = 0; j < 100; ++j) { + uint32 sample_number = rand() % sample_table_->sample_count(); + uint64 map_reported_timestamp = 0; + ASSERT_TRUE(map_->GetTimestamp(sample_number, &map_reported_timestamp)); + uint64 table_timestamp = GetTestSample(sample_number).cts; + ASSERT_EQ(map_reported_timestamp, table_timestamp); + + uint32 map_reported_duration = 0; + ASSERT_TRUE(map_->GetDuration(sample_number, &map_reported_duration)); + uint32 table_duration = GetTestSample(sample_number).dts_duration; + ASSERT_EQ(map_reported_duration, table_duration); + } + } +} + +// ==== GetIsKeyframe() Tests ================================================== + +// the map should consider every valid sample number a keyframe without an stss +TEST_F(MP4MapTest, GetIsKeyframeNoKeyframeTable) { + ResetMap(); + bool is_keyframe_out = false; + ASSERT_TRUE(map_->GetIsKeyframe(100, &is_keyframe_out)); + ASSERT_TRUE(is_keyframe_out); + + is_keyframe_out = false; + ASSERT_TRUE(map_->GetIsKeyframe(5, &is_keyframe_out)); + ASSERT_TRUE(is_keyframe_out); + + for (int i = 17; i < 174; i += 3) { + is_keyframe_out = false; + ASSERT_TRUE(map_->GetIsKeyframe(i, &is_keyframe_out)); + ASSERT_TRUE(is_keyframe_out); + } +} + +TEST_F(MP4MapTest, GetIsKeyframeIteration) { + CreateTestSampleTable(113, 1000, 0xb0df00d, 0xb0df00d, 5, 10, 5, 10, 10, 20, + 10, 20); + ResetMap(); + sample_table_->ClearReadStatistics(); + SetTestTable(kAtomType_stss, sample_table_->keyframe_count() / 2 + 5); + + for (uint32 i = 0; i < sample_table_->sample_count(); ++i) { + bool map_is_keyframe_out = false; + ASSERT_TRUE(map_->GetIsKeyframe(i, &map_is_keyframe_out)); + bool table_is_keyframe = GetTestSample(i).is_key_frame; + ASSERT_EQ(map_is_keyframe_out, table_is_keyframe); + } +} + +TEST_F(MP4MapTest, GetIsKeyframeRandomAccess) { + CreateTestSampleTable(114, 1000, 0xb0df00d, 0xb0df00d, 5, 10, 5, 10, 10, 20, + 10, 20); + ResetMap(); + sample_table_->ClearReadStatistics(); + SetTestTable(kAtomType_stss, sample_table_->keyframe_count() / 2 + 5); + + // pick a keyframe about halfway + uint32 sample_number = sample_table_->sample_count() / 2; + while (!GetTestSample(sample_number).is_key_frame) { + ++sample_number; + } + // sample one past it should not be a keyframe + bool map_is_keyframe_out = false; + ASSERT_TRUE(map_->GetIsKeyframe(sample_number + 1, &map_is_keyframe_out)); + ASSERT_FALSE(map_is_keyframe_out); + // sample one before keyframe should not be a keyframe either + ASSERT_TRUE(map_->GetIsKeyframe(sample_number - 1, &map_is_keyframe_out)); + ASSERT_FALSE(map_is_keyframe_out); + // however it should be a keyframe + ASSERT_TRUE(map_->GetIsKeyframe(sample_number, &map_is_keyframe_out)); + ASSERT_TRUE(map_is_keyframe_out); + + // first keyframe + sample_number = 0; + while (!GetTestSample(sample_number).is_key_frame) { + ++sample_number; + } + // next sample should not be a keyframe + ASSERT_TRUE(map_->GetIsKeyframe(sample_number + 1, &map_is_keyframe_out)); + ASSERT_FALSE(map_is_keyframe_out); + // but it should be + ASSERT_TRUE(map_->GetIsKeyframe(sample_number, &map_is_keyframe_out)); + ASSERT_TRUE(map_is_keyframe_out); + + // iterate backwards from end of file to beginning + for (int i = sample_table_->sample_count() - 1; i >= 0; --i) { + ASSERT_TRUE(map_->GetIsKeyframe(i, &map_is_keyframe_out)); + ASSERT_EQ(map_is_keyframe_out, GetTestSample(i).is_key_frame); + } + + // iterate backwards through keyframes only + for (int i = sample_table_->sample_count() - 1; i >= 0; --i) { + if (GetTestSample(i).is_key_frame) { + ASSERT_TRUE(map_->GetIsKeyframe(i, &map_is_keyframe_out)); + ASSERT_TRUE(map_is_keyframe_out); + } + } + + // iterate forwards but skip all keyframes + for (int i = sample_table_->sample_count() - 1; i >= 0; --i) { + if (!GetTestSample(i).is_key_frame) { + ASSERT_TRUE(map_->GetIsKeyframe(i, &map_is_keyframe_out)); + ASSERT_FALSE(map_is_keyframe_out); + } + } + + ResetMap(); + sample_table_->ClearReadStatistics(); + SetTestTable(kAtomType_stss, 7); + + // random access + for (int i = 0; i < 1000; ++i) { + sample_number = rand() % sample_table_->sample_count(); + ASSERT_TRUE(map_->GetIsKeyframe(sample_number, &map_is_keyframe_out)); + ASSERT_EQ(map_is_keyframe_out, GetTestSample(sample_number).is_key_frame); + } +} + +// ==== GetKeyframe() Tests ==================================================== + +// every frame should be returned as a keyframe. This tests if our computation +// of timestamps => sample numbers is equivalent to sample numbers => timestamps +TEST_F(MP4MapTest, GetKeyframeNoKeyframeTableIteration) { + CreateTestSampleTable(115, 30, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stts, 7); + + for (int i = 0; i < sample_table_->sample_count(); ++i) { + // get actual timestamp and duration of this sample + uint64 sample_timestamp = GetTestSample(i).dts; + uint32 sample_duration = GetTestSample(i).dts_duration; + // add a bit of time to sample timestamp, but keep time within this frame + sample_timestamp += i % sample_duration; + uint32 map_keyframe = 0; + ASSERT_TRUE(map_->GetKeyframe(sample_timestamp, &map_keyframe)); + ASSERT_EQ(map_keyframe, i); + } +} + +TEST_F(MP4MapTest, GetKeyframeNoKeyframeTableRandomAccess) { + CreateTestSampleTable(116, 30, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stts, 5); + + // backwards through the middle third of samples + for (int i = (sample_table_->sample_count() * 2) / 3; + i >= sample_table_->sample_count() / 3; --i) { + uint64 sample_timestamp = GetTestSample(i).dts; + uint32 sample_duration = GetTestSample(i).dts_duration; + sample_timestamp += sample_duration - 1 - (i % sample_duration); + uint32 map_keyframe = 0; + ASSERT_TRUE(map_->GetKeyframe(sample_timestamp, &map_keyframe)); + ASSERT_EQ(map_keyframe, i); + } + + // highest valid timestamp in file + uint64 highest_timestamp = + GetTestSample(sample_table_->sample_count() - 1).dts; + highest_timestamp += + GetTestSample(sample_table_->sample_count() - 1).dts_duration - 1; + uint32 map_keyframe = 0; + ASSERT_TRUE(map_->GetKeyframe(highest_timestamp, &map_keyframe)); + ASSERT_EQ(map_keyframe, sample_table_->sample_count() - 1); + + // lowest valid timestamp in file + ASSERT_TRUE(map_->GetKeyframe(0, &map_keyframe)); + ASSERT_EQ(map_keyframe, 0); + + // should fail on higher timestamps + ASSERT_FALSE(map_->GetKeyframe(highest_timestamp + 1, &map_keyframe)); +} + +// GetKeyframe is not normally called iteratively, so we test random access +TEST_F(MP4MapTest, GetKeyframe) { + CreateTestSampleTable(117, 60, 20, 25, 5, 10, 5, 10, 10, 20, 10, 20); + ResetMap(); + SetTestTable(kAtomType_stss, 3); + SetTestTable(kAtomType_stts, 7); + + // find first keyframe in file, should be first frame + uint32 map_keyframe = 0; + ASSERT_TRUE(map_->GetKeyframe(0, &map_keyframe)); + ASSERT_EQ(map_keyframe, 0); + + // find a first quarter keyframe in file + uint32 qtr_keyframe = sample_table_->sample_count() / 4; + while (!GetTestSample(qtr_keyframe).is_key_frame) { + ++qtr_keyframe; + } + uint32 next_keyframe = qtr_keyframe + 1; + while (!GetTestSample(next_keyframe).is_key_frame) { + ++next_keyframe; + } + uint32 prev_keyframe = qtr_keyframe - 1; + while (!GetTestSample(prev_keyframe).is_key_frame) { + --prev_keyframe; + } + uint32 last_keyframe = sample_table_->sample_count() - 1; + while (!GetTestSample(last_keyframe).is_key_frame) { + --last_keyframe; + } + // midway between this keyframe and the next one + uint32 test_frame = qtr_keyframe + ((next_keyframe - qtr_keyframe) / 2); + // get time for this frame + uint64 test_frame_timestamp = GetTestSample(test_frame).dts; + // get duration for this frame + uint32 test_frame_duration = GetTestSample(test_frame).dts_duration; + // midway through this frame + test_frame_timestamp += test_frame_duration / 2; + // find lower bound keyframe, should be qtr_keyframe + ASSERT_TRUE(map_->GetKeyframe(test_frame_timestamp, &map_keyframe)); + ASSERT_EQ(map_keyframe, qtr_keyframe); + + // timestamp one tick before qtr_keyframe should find previous keyframe + test_frame_timestamp = GetTestSample(qtr_keyframe).dts - 1; + ASSERT_TRUE(map_->GetKeyframe(test_frame_timestamp, &map_keyframe)); + ASSERT_EQ(map_keyframe, prev_keyframe); + + // very highest timestamp in file should return last keyframe + uint64 highest_timestamp = + GetTestSample(sample_table_->sample_count() - 1).dts; + highest_timestamp += + GetTestSample(sample_table_->sample_count() - 1).dts_duration - 1; + ASSERT_TRUE(map_->GetKeyframe(highest_timestamp, &map_keyframe)); + ASSERT_EQ(map_keyframe, last_keyframe); +} + +} // namespace diff --git a/media/starboard/progressive/mp4_parser.cc b/media/starboard/progressive/mp4_parser.cc new file mode 100644 index 000000000000..bb07505a9a66 --- /dev/null +++ b/media/starboard/progressive/mp4_parser.cc @@ -0,0 +1,665 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/mp4_parser.h" + +#include + +#include +#include + +#include "base/strings/stringprintf.h" +#include "media/formats/mp4/es_descriptor.h" +#include "media/starboard/progressive/endian_util.h" +#include "starboard/types.h" + +namespace media { + +// how many bytes to skip within an avc1 before the config atoms start? +static const int kSkipBytes_avc1 = 78; + +// what's the smallest meaningful mp4 atom size? +static const int kAtomMinSize = 8; + +// Full Box has a one byte version and three bytes flags after the header +static const int kFullBoxHeaderAndFlagSize = 4; + +// how much to download of an hdlr to get the trak type? +static const int kDesiredBytes_hdlr = 12; +static const uint32_t kAudioSubtype_hdlr_soun = 0x736f756e; +static const uint32_t kVideoSubtype_hdlr_vide = 0x76696465; + +// how much to download of an mp4a to determine version number? +static const int kDesiredBytes_mp4a = 2; +// how big is the mp4a atom before optional extension atoms? +static const int kTotalSize_mp4a_v0 = 28; +static const int kTotalSize_mp4a_v1 = 44; +static const int kTotalSize_mp4a_v2 = 64; + +// how many bytes should we download from an mdhd? +static const int kDesiredBytes_mdhd = 20; + +// how many bytes should we download from an mvhd? +static const int kDesiredBytes_mvhd = 20; + +// how many bytes to skip within an stsd before the config atoms start? +static const int kSkipBytes_stsd = 8; + +// use average values of mp4 metadata tables plus 2 standard deviations to +// hold most metadata atoms entirely in memory. +static const int kMapTableAtomCacheEntries_stsz = 260591 / kEntrySize_stsz; +static const int kMapTableAtomCacheEntries_stco = 22859 / kEntrySize_stco; +static const int kMapTableAtomCacheEntries_stss = 3786 / kEntrySize_stss; +static const int kMapTableAtomCacheEntries_stts = 164915 / kEntrySize_stts; +static const int kMapTableAtomCacheEntries_stsc = 32199 / kEntrySize_stsc; +static const int kMapTableAtomCacheEntries_co64 = 740212 / kEntrySize_co64; +static const int kMapTableAtomCacheEntries_ctts = 51543 / kEntrySize_ctts; + +// static +PipelineStatus MP4Parser::Construct(scoped_refptr reader, + const uint8_t* construction_header, + scoped_refptr* parser, + MediaLog* media_log) { + DCHECK(parser); + DCHECK(media_log); + *parser = nullptr; + + // detect mp4 stream by looking for ftyp atom at top of file + uint32_t ftyp = endian_util::load_uint32_big_endian(construction_header + 4); + if (ftyp != kAtomType_ftyp) { + // not an mp4 + return DEMUXER_ERROR_COULD_NOT_PARSE; + } + + // first 4 bytes will be the size of the ftyp atom + uint32_t ftyp_atom_size = + endian_util::load_uint32_big_endian(construction_header); + if (ftyp_atom_size < kAtomMinSize) { + return DEMUXER_ERROR_COULD_NOT_PARSE; + } + + // construct new mp4 parser + *parser = new MP4Parser(reader, ftyp_atom_size, media_log); + return PIPELINE_OK; +} + +MP4Parser::MP4Parser(scoped_refptr reader, + uint32_t ftyp_atom_size, + MediaLog* media_log) + : AVCParser(reader, media_log), + atom_offset_(ftyp_atom_size), // start at next atom, skipping over ftyp + current_trak_is_video_(false), + current_trak_is_audio_(false), + current_trak_time_scale_(0), + video_time_scale_hz_(0), + audio_time_scale_hz_(0), + audio_map_(new MP4Map(reader)), + video_map_(new MP4Map(reader)), + audio_sample_(0), + video_sample_(0), + first_audio_hole_ticks_(0), + first_audio_hole_(base::Seconds(0)) {} + +MP4Parser::~MP4Parser() {} + +// For MP4 we traverse the file's atom structure attempting to find the audio +// and video configuration information and the locations in the file of the +// various stbl subatoms which detail the position of the audio and video +// NALUs in the file. As some of the stbl subatoms can be quite large we cache +// a fixed maximum quantity of each stbl subatom and update the cache only on +// miss. +bool MP4Parser::ParseConfig() { + while (!IsConfigComplete() || !audio_map_->IsComplete() || + !video_map_->IsComplete()) { + if (!ParseNextAtom()) { + return false; + } + } + return true; +} + +scoped_refptr MP4Parser::GetNextAU(DemuxerStream::Type type) { + uint32_t size = 0; + uint32_t duration_ticks = 0; + uint64_t timestamp_ticks = 0; + uint64_t offset = 0; + bool is_keyframe = false; + base::TimeDelta timestamp; + base::TimeDelta duration; + if (type == DemuxerStream::AUDIO) { + if (audio_time_scale_hz_ == 0) { + LOG(ERROR) << "|audio_time_scale_hz_| cannot be 0."; + return nullptr; + } + if (!audio_map_->GetSize(audio_sample_, &size) || + !audio_map_->GetOffset(audio_sample_, &offset) || + !audio_map_->GetDuration(audio_sample_, &duration_ticks) || + !audio_map_->GetTimestamp(audio_sample_, ×tamp_ticks)) { + // determine if EOS or error + if (audio_map_->IsEOS(audio_sample_)) { + return AvcAccessUnit::CreateEndOfStreamAU(DemuxerStream::AUDIO, + audio_track_duration_); + } else { + LOG(ERROR) << "parsed bad audio AU"; + return nullptr; + } + } + // all aac frames are random-access, so all are keyframes + is_keyframe = true; + audio_sample_++; + timestamp = TicksToTime(timestamp_ticks, audio_time_scale_hz_); + duration = TicksToTime(duration_ticks, audio_time_scale_hz_); + + // It would be very unusual to encounter non-contiguous audio + // in an mp4, but you never know. Make sure this timestamp is + // contiguous in ticks from the last one + if (first_audio_hole_ticks_ == timestamp_ticks) { + // Much of the audio stack assumes that audio timestamps are + // contiguous. While the timestamps coming out of the map are + // normally continuous, they are on a different time scale. Due + // to roundoff error in conversion the timestamps produced may + // be discontinuous. To correct this we correct the timestamp + // to the one the system is expecting for continuity, then modify + // the duration by the negative of that same (small) value, + // so as to not accumulate roundoff error over time. + base::TimeDelta time_difference = timestamp - first_audio_hole_; + timestamp = first_audio_hole_; + duration += time_difference; + first_audio_hole_ = timestamp + duration; + first_audio_hole_ticks_ += duration_ticks; + } else { + LOG(WARNING) << "parsed non-contiguous mp4 audio timestamp"; + // reset hole tracking past gap + first_audio_hole_ticks_ = timestamp_ticks + duration_ticks; + first_audio_hole_ = timestamp + duration; + } + } else if (type == DemuxerStream::VIDEO) { + if (video_time_scale_hz_ == 0) { + LOG(ERROR) << "|video_time_scale_hz_| cannot be 0."; + return nullptr; + } + if (!video_map_->GetSize(video_sample_, &size) || + !video_map_->GetOffset(video_sample_, &offset) || + !video_map_->GetDuration(video_sample_, &duration_ticks) || + !video_map_->GetTimestamp(video_sample_, ×tamp_ticks) || + !video_map_->GetIsKeyframe(video_sample_, &is_keyframe)) { + if (video_map_->IsEOS(video_sample_)) { + return AvcAccessUnit::CreateEndOfStreamAU(DemuxerStream::VIDEO, + video_track_duration_); + } else { + LOG(ERROR) << "parsed bad video AU"; + return nullptr; + } + } + video_sample_++; + timestamp = TicksToTime(timestamp_ticks, video_time_scale_hz_); + duration = TicksToTime(duration_ticks, video_time_scale_hz_); + // due to b-frames it's much more likely we'll encounter discontinuous + // video buffers. As a result we add a small duration to each video + // buffer, equal in value to one full tick at the video timescale. The + // showing of video frames is actually keyed on the audio clock, so this + // shouldn't create too much jitter in the output. + duration += one_video_tick_; + } else { + NOTREACHED() << "unsupported stream type"; + return nullptr; + } + + size_t prepend_size = CalculatePrependSize(type, is_keyframe); + + if (type == DemuxerStream::AUDIO) { + return AvcAccessUnit::CreateAudioAU(offset, size, prepend_size, is_keyframe, + timestamp, duration, this); + } + return AvcAccessUnit::CreateVideoAU(offset, size, prepend_size, + nal_header_size_, is_keyframe, timestamp, + duration, this); +} + +bool MP4Parser::SeekTo(base::TimeDelta timestamp) { + if (audio_time_scale_hz_ == 0 || video_time_scale_hz_ == 0) { + LOG_IF(ERROR, audio_time_scale_hz_ == 0) + << "|audio_time_scale_hz_| cannot be 0."; + LOG_IF(ERROR, video_time_scale_hz_ == 0) + << "|video_time_scale_hz_| cannot be 0."; + return false; + } + + // get video timestamp in video time units + uint64_t video_ticks = TimeToTicks(timestamp, video_time_scale_hz_); + // find nearest keyframe from map, make it our next video sample + if (!video_map_->GetKeyframe(video_ticks, &video_sample_)) { + return false; + } + // get the timestamp for this video keyframe + uint64_t video_keyframe_time_ticks = 0; + if (!video_map_->GetTimestamp(video_sample_, &video_keyframe_time_ticks)) { + return false; + } + base::TimeDelta video_keyframe_time = + TicksToTime(video_keyframe_time_ticks, video_time_scale_hz_); + // find the closest audio frame that bounds that timestamp + uint64_t audio_ticks = TimeToTicks(video_keyframe_time, audio_time_scale_hz_); + if (!audio_map_->GetKeyframe(audio_ticks, &audio_sample_)) { + return false; + } + LOG(INFO) << base::StringPrintf( + "seeking to timestamp: %" PRId64 ", video sample: %d, audio sample: %d", + timestamp.InMilliseconds(), video_sample_, audio_sample_); + // cheat our buffer continuity system + if (!audio_map_->GetTimestamp(audio_sample_, &first_audio_hole_ticks_)) { + return false; + } + first_audio_hole_ = + TicksToTime(first_audio_hole_ticks_, audio_time_scale_hz_); + return true; +} + +// parse the atom starting at atom_offset_, update appropriate internal state, +// return false on fatal error. General structure of an MP4 atom is: +// field | type | comment +// ------------------+--------+--------- +// atom size | uint32_t | if 0 means "rest of file", if 1 means extended +// fourCC code | ASCII | four-byte ASCII code we treat as uint32_t +// extended size | uint64_t | optional size field, only here if atom size is +// 1 +// <--- rest of atom body starts here +bool MP4Parser::ParseNextAtom() { + uint8_t atom[kAtomDownload]; + int bytes_read = reader_->BlockingRead(atom_offset_, kAtomDownload, atom); + if (bytes_read < kAtomDownload) { + return false; + } + // first 4 bytes are size of atom uint32_t + uint64_t atom_size = + static_cast(endian_util::load_uint32_big_endian(atom)); + // normally atom body starts just past fourCC code + uint32_t atom_body = kAtomMinSize; + // if 1 we need to load the extended size which will be appended just past + // the fourCC code + if (atom_size == 1) { + atom_size = endian_util::load_uint64_big_endian(atom + 8); + // advance atom_body past the 8 bytes of size we just parsed + atom_body += 8; + } else if (atom_size == 0) { + // calculate size of this atom from remainder of file + DCHECK_LE(atom_offset_, + static_cast(std::numeric_limits::max())); + if (reader_->FileSize() > static_cast(atom_offset_)) { + atom_size = reader_->FileSize() - atom_offset_; + } + } + // atom sizes also include the size of the start of the atom, so sanity-check + // the size we just parsed against the number of bytes we needed to parse it + if (atom_size < atom_body) { + LOG(WARNING) << base::StringPrintf("atom size: %" PRId64 + " less than min body size %d", + atom_size, atom_body); + return false; + } + + // extract fourCC code as big-endian uint32_t + uint32_t four_cc = endian_util::load_uint32_big_endian(atom + 4); + LOG(INFO) << base::StringPrintf("four_cc: %c%c%c%c", atom[4], atom[5], + atom[6], atom[7]); + + // advance read pointer to atom body + atom_offset_ += atom_body; + // adjust size of body of atom from size of header + uint64_t atom_data_size = atom_size - atom_body; + + bool atom_parse_success = true; + + // We use 95% certainty intervals for video metadata atom sizes. The map + // is written to handle larger atom sizes but having to recache metadata + // increases latencies on things like seeks. + int map_table_atom_cache_entries = 0; + + // now take appropriate action based on atom type + switch (four_cc) { + // avc1 atoms are contained within stsd atoms and carry their own + // configuration baggage load, which we skip over and parse the atoms + // within, normally an avcC atom. + case kAtomType_avc1: + atom_offset_ += kSkipBytes_avc1; + break; + + // avcC atoms contain the AVCConfigRecord, our video configuration info + case kAtomType_avcC: + atom_parse_success = + DownloadAndParseAVCConfigRecord(atom_offset_, atom_data_size); + if (atom_parse_success) { + atom_offset_ += atom_data_size; + } + break; + + // esds atoms contain actually usable audio configuration info for AAC. + case kAtomType_esds: + return ParseMP4_esds(atom_data_size); + + // can tell us if mdia and mdhd atoms relate to audio or video metadata + case kAtomType_hdlr: + return ParseMP4_hdlr(atom_data_size, atom + atom_body); + + // provides a duration and a timescale unique to a given track + case kAtomType_mdhd: + return ParseMP4_mdhd(atom_data_size, atom + atom_body); + + // mp4a atoms contain audio configuration info, but we only want to know + // which version it is so we can skip to the esds, which we must be present + // when using AAC + case kAtomType_mp4a: + return ParseMP4_mp4a(atom_data_size, atom + atom_body); + + // movie header atom contains track duration and time unit scale, we trust + // these data as the authoritative duration data for the mp4 + case kAtomType_mvhd: + return ParseMP4_mvhd(atom_data_size, atom + atom_body); + + // stsd atoms may contain avc1 atoms, which themselves may contain avcC + // atoms, which contain actually usable configuration information. skip to + // subatom. + case kAtomType_stsd: + atom_offset_ += kSkipBytes_stsd; + break; + + // We're very much interested in the contents of the trak container atom, + // blow away state that we may have been keeping about any prior trak + // atoms we've parsed. + case kAtomType_trak: + current_trak_is_video_ = false; + current_trak_is_audio_ = false; + break; + + // if one of the stbl subatoms add it to the appropriate audio or video map + // and then advance past it. + case kAtomType_co64: + map_table_atom_cache_entries = kMapTableAtomCacheEntries_co64; + break; + + case kAtomType_ctts: + map_table_atom_cache_entries = kMapTableAtomCacheEntries_ctts; + break; + + case kAtomType_stco: + map_table_atom_cache_entries = kMapTableAtomCacheEntries_stco; + break; + + case kAtomType_stts: + map_table_atom_cache_entries = kMapTableAtomCacheEntries_stts; + break; + + case kAtomType_stsc: + map_table_atom_cache_entries = kMapTableAtomCacheEntries_stsc; + break; + + case kAtomType_stss: + map_table_atom_cache_entries = kMapTableAtomCacheEntries_stss; + break; + + case kAtomType_stsz: + map_table_atom_cache_entries = kMapTableAtomCacheEntries_stsz; + break; + + // these are container atoms, so we dont want to advance past the header + // as we are interested in their contents. Parsing them is trivial + // as all they are is a size header and a fourCC type tag, which we've + // already parsed and advanced past. + case kAtomType_mdia: + case kAtomType_minf: + case kAtomType_moov: + case kAtomType_stbl: + // no-op + break; + + // known atom types that we wish to just skip past the body without warning + case kAtomType_dinf: + case kAtomType_dref: + case kAtomType_smhd: + case kAtomType_tkhd: + case kAtomType_vmhd: + atom_offset_ += atom_data_size; + break; + + // parse functions are assumed to advance read_position_ themselves, + // as we are flattening a tree of atoms so that the atom_size we parsed + // this time, if it's a container, may not have been entirely consumed + // in this single call. However for unsupported atoms we just skip them + // entirely, meaning we will skip their contents too. + default: + atom_offset_ += atom_data_size; + LOG(INFO) << base::StringPrintf("skipping unsupported MP4 atom: %c%c%c%c", + atom[4], atom[5], atom[6], atom[7]); + break; + } + + if (map_table_atom_cache_entries > 0) { + if (current_trak_is_video_) { + atom_parse_success = + video_map_->SetAtom(four_cc, atom_offset_, atom_data_size, + map_table_atom_cache_entries, atom + atom_body); + } else if (current_trak_is_audio_) { + atom_parse_success = + audio_map_->SetAtom(four_cc, atom_offset_, atom_data_size, + map_table_atom_cache_entries, atom + atom_body); + } + atom_offset_ += atom_data_size; + } + + if (!atom_parse_success) { + LOG(ERROR) << base::StringPrintf("Unable to parse MP4 atom: %c%c%c%c", + atom[4], atom[5], atom[6], atom[7]); + } + + return atom_parse_success; +} + +bool MP4Parser::ParseMP4_esds(uint64_t atom_data_size) { + if (atom_data_size < kFullBoxHeaderAndFlagSize) { + LOG(WARNING) << base::StringPrintf( + "esds box should at least be %d bytes but now it is %" PRId64 " bytes", + kFullBoxHeaderAndFlagSize, atom_data_size); + return false; + } + + uint64_t esds_offset = atom_offset_ + kFullBoxHeaderAndFlagSize; + uint64_t esds_size = atom_data_size - kFullBoxHeaderAndFlagSize; + + if (esds_size == 0) { + return false; + } + // we'll need to download entire esds, allocate buffer for it + std::vector esds_storage(esds_size); + uint8_t* esds = &esds_storage[0]; + // download esds + int bytes_read = reader_->BlockingRead(esds_offset, esds_size, esds); + if (bytes_read < esds_size) { + LOG(WARNING) << "failed to download esds"; + return false; + } + mp4::ESDescriptor es_descriptor; + std::vector data(esds, esds + esds_size); + if (es_descriptor.Parse(data)) { + const std::vector& dsi = es_descriptor.decoder_specific_info(); + if (dsi.size() >= 2) { + ParseAudioSpecificConfig(dsi[0], dsi[1]); + atom_offset_ += atom_data_size; + return true; + } + LOG(WARNING) << "esds audio specific config shorter than 2 bytes"; + } else { + LOG(WARNING) << "error in parse esds box"; + } + + return false; +} + +bool MP4Parser::ParseMP4_hdlr(uint64_t atom_data_size, uint8_t* hdlr) { + // ensure we're downloading enough of the hdlr to parse + DCHECK_LE(kDesiredBytes_hdlr + 16, kAtomDownload); + // sanity-check for minimum size + if (atom_data_size < kDesiredBytes_hdlr) { + LOG(WARNING) << base::StringPrintf("bad size %" PRId64 " on hdlr", + atom_data_size); + return false; + } + // last 4 bytes of the 12 we need are an ascii code for the trak type, we + // want 'vide' for video or 'soun' for audio. ignore the rest. + uint32_t hdlr_subtype = endian_util::load_uint32_big_endian(hdlr + 8); + // update state flags + current_trak_is_video_ = (hdlr_subtype == kVideoSubtype_hdlr_vide); + current_trak_is_audio_ = (hdlr_subtype == kAudioSubtype_hdlr_soun); + // save a time scale if pending + if (current_trak_time_scale_ > 0 && current_trak_is_video_) { + video_time_scale_hz_ = current_trak_time_scale_; + current_trak_time_scale_ = 0; + video_track_duration_ = current_trak_duration_; + one_video_tick_ = base::Microseconds(1000000 / video_time_scale_hz_); + } + if (current_trak_time_scale_ > 0 && current_trak_is_audio_) { + audio_time_scale_hz_ = current_trak_time_scale_; + current_trak_time_scale_ = 0; + audio_track_duration_ = current_trak_duration_; + } + // skip rest of atom + atom_offset_ += atom_data_size; + return true; +} + +bool MP4Parser::ParseMP4_mdhd(uint64_t atom_data_size, uint8_t* mdhd) { + DCHECK_LE(kDesiredBytes_mdhd + 16, kAtomDownload); + if (atom_data_size < kDesiredBytes_mdhd) { + LOG(WARNING) << base::StringPrintf("bad size %" PRId64 " on mdhd", + atom_data_size); + return false; + } + uint32_t time_scale = endian_util::load_uint32_big_endian(mdhd + 12); + if (time_scale == 0) { + LOG(WARNING) << "got 0 time scale for mvhd"; + return false; + } + // double-check track duration, it may be different from the movie duration + uint32_t track_duration_ticks = + endian_util::load_uint32_big_endian(mdhd + 16); + base::TimeDelta track_duration = + TicksToTime(track_duration_ticks, time_scale); + if (track_duration > duration_) { + LOG(WARNING) << base::StringPrintf("mdhd has longer duration: %" PRId64 + " ms than old value: %" PRId64 " ms.", + track_duration.InMicroseconds(), + duration_.InMicroseconds()); + duration_ = track_duration; + } + if (current_trak_is_video_) { + video_time_scale_hz_ = time_scale; + current_trak_time_scale_ = 0; + video_track_duration_ = track_duration; + one_video_tick_ = base::Microseconds(1000000 / video_time_scale_hz_); + } else if (current_trak_is_audio_) { + audio_time_scale_hz_ = time_scale; + current_trak_time_scale_ = 0; + audio_track_duration_ = track_duration; + } else { + // it's possible we will encounter the mdhd before we encounter the hdlr, + // in that event we save the time scale value until we know. + current_trak_time_scale_ = time_scale; + current_trak_duration_ = track_duration; + } + atom_offset_ += atom_data_size; + return true; +} + +bool MP4Parser::ParseMP4_mp4a(uint64_t atom_data_size, uint8_t* mp4a) { + DCHECK_LE(kDesiredBytes_mp4a + 16, kAtomDownload); + // we only need the first two bytes of the header, which details the version + // number of this atom, which tells us the size of the rest of the header, + // telling us how much we should skip to get to the extension contents. + if (atom_data_size < kDesiredBytes_mp4a) { + LOG(WARNING) << base::StringPrintf("bad size %" PRId64 " on mp4a", + atom_data_size); + return false; + } + uint16_t mp4a_version = endian_util::load_uint16_big_endian(mp4a); + switch (mp4a_version) { + case 0: + atom_offset_ += kTotalSize_mp4a_v0; + return true; + + case 1: + atom_offset_ += kTotalSize_mp4a_v1; + return true; + + case 2: + atom_offset_ += kTotalSize_mp4a_v2; + return true; + + default: + // unknown mp4a atom version, parse failure + LOG(ERROR) << base::StringPrintf("parsed bad mp4a version %d", + mp4a_version); + return false; + } +} + +// partial layout of mvhd header is: +// offset | name | size in bytes +// -------+-------------------+--------------- +// 0 | version | 1 (ignored) +// 1 | flags | 3 (ignored) +// 4 | creation time | 4 (ignored) +// 8 | modification time | 4 (ignored) +// 12 | time scale | 4 +// 16 | duration: | 4 +// +bool MP4Parser::ParseMP4_mvhd(uint64_t atom_data_size, uint8_t* mvhd) { + DCHECK_LE(kDesiredBytes_mvhd + 16, kAtomDownload); + // it should be at least long enough for us to extract the parts we want + if (atom_data_size < kDesiredBytes_mvhd) { + LOG(WARNING) << base::StringPrintf("bad size %" PRId64 " on mvhd", + atom_data_size); + return false; + } + uint32_t time_scale_hz = endian_util::load_uint32_big_endian(mvhd + 12); + if (time_scale_hz == 0) { + LOG(WARNING) << "got 0 time scale for mvhd"; + return false; + } + // duration is in units of the time scale we just extracted + uint64_t duration_ticks = endian_util::load_uint32_big_endian(mvhd + 16); + // calculate actual duration from that and the time scale + duration_ = TicksToTime(duration_ticks, time_scale_hz); + // advance read position + atom_offset_ += atom_data_size; + return true; +} + +base::TimeDelta MP4Parser::TicksToTime(uint64_t ticks, uint32_t time_scale_hz) { + DCHECK_NE(time_scale_hz, 0); + + if (time_scale_hz == 0) { + return base::Seconds(0); + } + return base::Microseconds((ticks * 1000000ULL) / time_scale_hz); +} + +uint64_t MP4Parser::TimeToTicks(base::TimeDelta time, uint32_t time_scale_hz) { + DCHECK_NE(time_scale_hz, 0); + + if (time_scale_hz == 0) { + return 0; + } + return (time.InMicroseconds() * time_scale_hz) / 1000000ULL; +} + +} // namespace media diff --git a/media/starboard/progressive/mp4_parser.h b/media/starboard/progressive/mp4_parser.h new file mode 100644 index 000000000000..5ecf26adae4a --- /dev/null +++ b/media/starboard/progressive/mp4_parser.h @@ -0,0 +1,113 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_MP4_PARSER_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_MP4_PARSER_H_ + +#include "media/base/media_log.h" +#include "media/starboard/progressive/avc_parser.h" +#include "media/starboard/progressive/mp4_map.h" + +namespace media { + +// How many bytes to download from the start of the atom? Should be large +// enough that we can extract all the data we need from the atom without +// second download (typically), but no larger. This is currently set at 16 +// bytes for the 8 byte header + optional 8 byte size extension plus 20 bytes +// for the needed values within an mvhd header. We leave this is the header so +// that MP4Map can reuse, +static const int kAtomDownload = 36; + +// mp4 atom fourCC codes as big-endian unsigned ints +static const uint32_t kAtomType_avc1 = 0x61766331; // skip in to subatom +static const uint32_t kAtomType_avcC = 0x61766343; // download and parse +static const uint32_t kAtomType_co64 = 0x636f3634; // cache in table +static const uint32_t kAtomType_ctts = 0x63747473; // cache in table +static const uint32_t kAtomType_dinf = 0x64696e66; // skip whole atom +static const uint32_t kAtomType_dref = 0x64726566; // skip whole atom +static const uint32_t kAtomType_esds = 0x65736473; // download and parse +static const uint32_t kAtomType_ftyp = 0x66747970; // top of the file only +static const uint32_t kAtomType_hdlr = 0x68646c72; // parse first 12 bytes +static const uint32_t kAtomType_mdhd = 0x6d646864; // parse first 20 bytes +static const uint32_t kAtomType_mdia = 0x6d646961; // container atom, no-op +static const uint32_t kAtomType_minf = 0x6d696e66; // container atom, no-op +static const uint32_t kAtomType_moov = 0x6d6f6f76; // container atom, no-op +static const uint32_t kAtomType_mp4a = 0x6d703461; // parse first 10 bytes +static const uint32_t kAtomType_mvhd = 0x6d766864; // parse first 20 bytes +static const uint32_t kAtomType_smhd = 0x736d6862; // skip whole atom +static const uint32_t kAtomType_stbl = 0x7374626c; // container atom, no-op +static const uint32_t kAtomType_stco = 0x7374636f; // cache in table +static const uint32_t kAtomType_stts = 0x73747473; // cache in table +static const uint32_t kAtomType_stsc = 0x73747363; // cache in table +static const uint32_t kAtomType_stsd = 0x73747364; // skip in to subatom +static const uint32_t kAtomType_stss = 0x73747373; // cache in table +static const uint32_t kAtomType_stsz = 0x7374737a; // cache in table +static const uint32_t kAtomType_trak = 0x7472616b; // container atom, no-op +static const uint32_t kAtomType_tkhd = 0x746b6864; // skip whole atom +static const uint32_t kAtomType_vmhd = 0x766d6864; // skip whole atom +// TODO: mp4v!! + +class MP4Parser : public AVCParser { + public: + // Attempts to make sense of the provided bytes of the top of a file as an + // flv, and if it does make sense returns PIPELINE_OK and |*parser| contains a + // MP4Parser initialized with some basic state. If it doesn't make sense + // this returns an error status and |*parser| contains NULL. + static ::media::PipelineStatus Construct( + scoped_refptr reader, + const uint8_t* construction_header, + scoped_refptr* parser, + MediaLog* media_log); + MP4Parser(scoped_refptr reader, + uint32_t ftyp_atom_size, + MediaLog* media_log); + ~MP4Parser() override; + + // === ProgressiveParser implementation + bool ParseConfig() override; + scoped_refptr GetNextAU(DemuxerStream::Type type) override; + bool SeekTo(base::TimeDelta timestamp) override; + + private: + bool ParseNextAtom(); + bool ParseMP4_esds(uint64_t atom_data_size); + bool ParseMP4_hdlr(uint64_t atom_data_size, uint8_t* hdlr); + bool ParseMP4_mdhd(uint64_t atom_data_size, uint8_t* mdhd); + bool ParseMP4_mp4a(uint64_t atom_data_size, uint8_t* mp4a); + bool ParseMP4_mvhd(uint64_t atom_data_size, uint8_t* mvhd); + base::TimeDelta TicksToTime(uint64_t ticks, uint32_t time_scale_hz); + uint64_t TimeToTicks(base::TimeDelta time, uint32_t time_scale_hz); + + uint64_t atom_offset_; + bool current_trak_is_video_; + bool current_trak_is_audio_; + uint32_t current_trak_time_scale_; + base::TimeDelta current_trak_duration_; + uint32_t video_time_scale_hz_; + base::TimeDelta one_video_tick_; + uint32_t audio_time_scale_hz_; + base::TimeDelta audio_track_duration_; + base::TimeDelta video_track_duration_; + scoped_refptr audio_map_; + scoped_refptr video_map_; + uint32_t audio_sample_; + uint32_t video_sample_; + // for keeping buffers continuous across time scales + uint64_t first_audio_hole_ticks_; + base::TimeDelta first_audio_hole_; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_MP4_PARSER_H_ diff --git a/media/starboard/progressive/progressive_demuxer.cc b/media/starboard/progressive/progressive_demuxer.cc new file mode 100644 index 000000000000..ea42c7f5a334 --- /dev/null +++ b/media/starboard/progressive/progressive_demuxer.cc @@ -0,0 +1,566 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/progressive_demuxer.h" + +#include + +#include + +#include "base/functional/bind.h" +#include "base/functional/callback.h" +#include "base/functional/callback_helpers.h" +#include "base/strings/stringprintf.h" +#include "base/task/bind_post_task.h" +#include "base/task/sequenced_task_runner.h" +#include "base/time/time.h" +#include "base/trace_event/trace_event.h" +#include "media/base/data_source.h" +#include "media/base/timestamp_constants.h" +#include "media/starboard/starboard_utils.h" +#include "starboard/types.h" + +namespace media { + +ProgressiveDemuxerStream::ProgressiveDemuxerStream(ProgressiveDemuxer* demuxer, + Type type) + : demuxer_(demuxer), type_(type) { + DCHECK(demuxer_); +} + +void ProgressiveDemuxerStream::Read(uint32_t count, ReadCB read_cb) { + LOG(ERROR) << "Cobalt " << __func__; + DCHECK(!read_cb.is_null()); + + base::AutoLock auto_lock(lock_); + + // Don't accept any additional reads if we've been told to stop. + // The demuxer_ may have been destroyed in the pipeline thread. + if (stopped_) { + std::move(read_cb).Run(DemuxerStream::kOk, + {DecoderBuffer::CreateEOSBuffer()}); + return; + } + + // Buffers are only queued when there are no pending reads. + DCHECK(buffer_queue_.empty() || read_queue_.empty()); + + if (!buffer_queue_.empty()) { + // Send the oldest buffer back. + scoped_refptr buffer = buffer_queue_.front(); + if (buffer->end_of_stream()) { + LOG(INFO) << "media_stack ProgressiveDemuxerStream::Read() EOS sent."; + } else { + // Do not pop EOS buffers, so that subsequent read requests also get EOS + total_buffer_size_ -= buffer->data_size(); + --total_buffer_count_; + buffer_queue_.pop_front(); + } + std::move(read_cb).Run(DemuxerStream::kOk, {buffer}); + } else { + read_queue_.push_back(std::move(read_cb)); + } +} + +AudioDecoderConfig ProgressiveDemuxerStream::audio_decoder_config() { + return demuxer_->AudioConfig(); +} + +VideoDecoderConfig ProgressiveDemuxerStream::video_decoder_config() { + return demuxer_->VideoConfig(); +} + +Ranges ProgressiveDemuxerStream::GetBufferedRanges() { + base::AutoLock auto_lock(lock_); + return buffered_ranges_; +} + +DemuxerStream::Type ProgressiveDemuxerStream::type() const { + return type_; +} + +void ProgressiveDemuxerStream::EnableBitstreamConverter() { + NOTIMPLEMENTED(); +} + +void ProgressiveDemuxerStream::EnqueueBuffer( + scoped_refptr buffer) { + base::AutoLock auto_lock(lock_); + if (stopped_) { + // it's possible due to pipelining both downstream and within the + // demuxer that several pipelined reads will be enqueuing packets + // on a stopped stream. Drop them after complaining. + LOG(WARNING) << "attempted to enqueue packet on stopped stream"; + return; + } + + if (buffer->end_of_stream()) { + LOG(INFO) << "media_stack ProgressiveDemuxerStream::EnqueueBuffer() EOS " + "received."; + } else if (buffer->timestamp() != kNoTimestamp) { + if (last_buffer_timestamp_ != kNoTimestamp && + last_buffer_timestamp_ < buffer->timestamp()) { + buffered_ranges_.Add(last_buffer_timestamp_, buffer->timestamp()); + } + last_buffer_timestamp_ = buffer->timestamp(); + } else { + LOG(WARNING) << "bad timestamp info on enqueued buffer."; + } + + // Check for any already waiting reads, service oldest read if there + if (read_queue_.size()) { + // assumption here is that buffer queue is empty + DCHECK_EQ(buffer_queue_.size(), 0); + ReadCB read_cb = std::move(read_queue_.front()); + read_queue_.pop_front(); + std::move(read_cb).Run(DemuxerStream::kOk, {buffer}); + } else { + // save the buffer for next read request + buffer_queue_.push_back(buffer); + if (!buffer->end_of_stream()) { + total_buffer_size_ += buffer->data_size(); + ++total_buffer_count_; + } + } +} + +base::TimeDelta ProgressiveDemuxerStream::GetLastBufferTimestamp() const { + base::AutoLock auto_lock(lock_); + return last_buffer_timestamp_; +} + +size_t ProgressiveDemuxerStream::GetTotalBufferSize() const { + base::AutoLock auto_lock(lock_); + return total_buffer_size_; +} + +size_t ProgressiveDemuxerStream::GetTotalBufferCount() const { + base::AutoLock auto_lock(lock_); + return total_buffer_count_; +} + +void ProgressiveDemuxerStream::FlushBuffers() { + base::AutoLock auto_lock(lock_); + // TODO: Investigate if the following warning is valid. + LOG_IF(WARNING, !read_queue_.empty()) << "Read requests should be empty"; + buffer_queue_.clear(); + total_buffer_size_ = 0; + total_buffer_count_ = 0; + last_buffer_timestamp_ = kNoTimestamp; +} + +void ProgressiveDemuxerStream::Stop() { + DCHECK(demuxer_->RunsTasksInCurrentSequence()); + base::AutoLock auto_lock(lock_); + buffer_queue_.clear(); + total_buffer_size_ = 0; + total_buffer_count_ = 0; + last_buffer_timestamp_ = kNoTimestamp; + // fulfill any pending callbacks with EOS buffers set to end timestamp + for (ReadQueue::iterator it = read_queue_.begin(); it != read_queue_.end(); + ++it) { + std::move(*it).Run(DemuxerStream::kOk, {DecoderBuffer::CreateEOSBuffer()}); + } + read_queue_.clear(); + stopped_ = true; +} + +// +// ProgressiveDemuxer +// +ProgressiveDemuxer::ProgressiveDemuxer( + const scoped_refptr& task_runner, + DataSource* data_source, + MediaLog* const media_log) + : task_runner_(task_runner), + host_(NULL), + blocking_thread_("ProgDemuxerBlk"), + data_source_(data_source), + media_log_(media_log), + stopped_(false), + flushing_(false), + audio_reached_eos_(false), + video_reached_eos_(false) { + DCHECK(task_runner_); + DCHECK(data_source_); + DCHECK(media_log_); + reader_ = new DataSourceReader(); + reader_->SetDataSource(data_source_); +} + +ProgressiveDemuxer::~ProgressiveDemuxer() { + // Explicitly stop |blocking_thread_| to ensure that it stops before the + // destructing of any other members. + blocking_thread_.Stop(); +} + +void ProgressiveDemuxer::Initialize(DemuxerHost* host, + PipelineStatusCallback status_cb) { + DCHECK(RunsTasksInCurrentSequence()); + DCHECK(reader_); + DCHECK(!parser_); + + LOG(INFO) << "this is a PROGRESSIVE playback."; + + host_ = host; + + // create audio and video demuxer stream objects + audio_demuxer_stream_.reset( + new ProgressiveDemuxerStream(this, DemuxerStream::AUDIO)); + video_demuxer_stream_.reset( + new ProgressiveDemuxerStream(this, DemuxerStream::VIDEO)); + + // start the blocking thread and have it download and parse the media config + if (!blocking_thread_.Start()) { + std::move(status_cb).Run(DEMUXER_ERROR_COULD_NOT_PARSE); + return; + } + + blocking_thread_.task_runner()->PostTask( + FROM_HERE, base::BindOnce(&ProgressiveDemuxer::ParseConfigBlocking, + base::Unretained(this), std::move(status_cb))); +} + +void ProgressiveDemuxer::ParseConfigBlocking(PipelineStatusCallback status_cb) { + DCHECK(blocking_thread_.task_runner()->RunsTasksInCurrentSequence()); + DCHECK(!parser_); + + // construct stream parser with error callback + PipelineStatus status = + ProgressiveParser::Construct(reader_, &parser_, media_log_); + // if we can't construct a parser for this stream it's a fatal error, return + // false so ParseConfigDone will notify the caller to Initialize() via + // status_cb. + if (!parser_ || status != PIPELINE_OK) { + DCHECK(!parser_); + DCHECK_NE(status, PIPELINE_OK); + if (status == PIPELINE_OK) { + status = DEMUXER_ERROR_COULD_NOT_PARSE; + } + ParseConfigDone(std::move(status_cb), status); + return; + } + + // instruct the parser to extract audio and video config from the file + if (!parser_->ParseConfig()) { + ParseConfigDone(std::move(status_cb), DEMUXER_ERROR_COULD_NOT_PARSE); + return; + } + + // make sure we got a valid and complete configuration + if (!parser_->IsConfigComplete()) { + ParseConfigDone(std::move(status_cb), DEMUXER_ERROR_COULD_NOT_PARSE); + return; + } + + // IsConfigComplete() should guarantee we know the duration + DCHECK(parser_->Duration() != kInfiniteDuration); + host_->SetDuration(parser_->Duration()); + + // successful parse of config data, inform the nonblocking demuxer thread + DCHECK_EQ(status, PIPELINE_OK); + ParseConfigDone(std::move(status_cb), PIPELINE_OK); +} + +void ProgressiveDemuxer::ParseConfigDone(PipelineStatusCallback status_cb, + PipelineStatus status) { + DCHECK(blocking_thread_.task_runner()->RunsTasksInCurrentSequence()); + + if (HasStopCalled()) { + return; + } + + // if the blocking parser thread cannot parse config we're done. + if (status != PIPELINE_OK) { + std::move(status_cb).Run(status); + return; + } + DCHECK(parser_); + // start downloading data + Request(DemuxerStream::AUDIO); + + std::move(status_cb).Run(PIPELINE_OK); +} + +void ProgressiveDemuxer::Request(DemuxerStream::Type type) { + if (!blocking_thread_.task_runner()->RunsTasksInCurrentSequence()) { + blocking_thread_.task_runner()->PostTask( + FROM_HERE, base::BindRepeating(&ProgressiveDemuxer::Request, + base::Unretained(this), type)); + return; + } + + DCHECK(!requested_au_) << "overlapping requests not supported!"; + flushing_ = false; + // Ask parser for next AU + scoped_refptr au = parser_->GetNextAU(type); + // fatal parsing error returns NULL or malformed AU + if (!au || !au->IsValid()) { + if (!HasStopCalled()) { + LOG(ERROR) << "got back bad AU from parser"; + host_->OnDemuxerError(DEMUXER_ERROR_COULD_NOT_PARSE); + } + return; + } + + // make sure we got back an AU of the correct type + DCHECK(au->GetType() == type); + + const char* event_type = type == DemuxerStream::AUDIO ? "audio" : "video"; + + // don't issue allocation requests for EOS AUs + if (au->IsEndOfStream()) { + // enqueue EOS buffer with correct stream + scoped_refptr eos_buffer = DecoderBuffer::CreateEOSBuffer(); + if (type == DemuxerStream::AUDIO) { + audio_reached_eos_ = true; + audio_demuxer_stream_->EnqueueBuffer(eos_buffer); + } else if (type == DemuxerStream::VIDEO) { + video_reached_eos_ = true; + video_demuxer_stream_->EnqueueBuffer(eos_buffer); + } + IssueNextRequest(); + return; + } + + // enqueue the request + requested_au_ = au; + + AllocateBuffer(); +} + +void ProgressiveDemuxer::AllocateBuffer() { + DCHECK(requested_au_); + + if (HasStopCalled()) { + return; + } + + if (requested_au_) { + size_t total_buffer_size = audio_demuxer_stream_->GetTotalBufferSize() + + video_demuxer_stream_->GetTotalBufferSize(); + size_t total_buffer_count = audio_demuxer_stream_->GetTotalBufferCount() + + video_demuxer_stream_->GetTotalBufferCount(); + // Only sdr video is supported in progressive mode. + const int kBitDepth = 8; + int progressive_budget = SbMediaGetProgressiveBufferBudget( + MediaVideoCodecToSbMediaVideoCodec(VideoConfig().codec()), + VideoConfig().visible_rect().size().width(), + VideoConfig().visible_rect().size().height(), kBitDepth); + base::TimeDelta progressive_duration_cap = base::Microseconds( + SbMediaGetBufferGarbageCollectionDurationThreshold()); + const int kEstimatedBufferCountPerSeconds = 70; + int progressive_buffer_count_cap = + progressive_duration_cap.InSeconds() * kEstimatedBufferCountPerSeconds; + if (total_buffer_size >= progressive_budget || + total_buffer_count > progressive_buffer_count_cap) { + // Retry after 100 milliseconds. + const base::TimeDelta kDelay = base::Milliseconds(100); + blocking_thread_.task_runner()->PostDelayedTask( + FROM_HERE, + base::BindRepeating(&ProgressiveDemuxer::AllocateBuffer, + base::Unretained(this)), + kDelay); + return; + } + + scoped_refptr decoder_buffer( + new DecoderBuffer(requested_au_->GetMaxSize())); + DCHECK(decoder_buffer); + decoder_buffer->set_is_key_frame(requested_au_->IsKeyframe()); + Download(decoder_buffer); + } +} + +void ProgressiveDemuxer::Download(scoped_refptr buffer) { + DCHECK(base::SequencedTaskRunner::GetCurrentDefault() + ->RunsTasksInCurrentSequence()); + // We need a requested_au_ or to have canceled this request and + // are buffering to a new location for this to make sense + DCHECK(requested_au_); + + const char* event_type = + requested_au_->GetType() == DemuxerStream::AUDIO ? "audio" : "video"; + // do nothing if stopped + if (HasStopCalled()) { + LOG(INFO) << "aborting download task, stopped"; + return; + } + + // Flushing is a signal to restart the request->download cycle with + // a new request. Drop current request and issue a new one. + // flushing_ will be reset by the next call to RequestTask() + if (flushing_) { + LOG(INFO) << "skipped AU download due to flush"; + requested_au_ = nullptr; + IssueNextRequest(); + return; + } + + if (!requested_au_->Read(reader_.get(), buffer.get())) { + LOG(ERROR) << "au read failed"; + host_->OnDemuxerError(PIPELINE_ERROR_READ); + return; + } + + // copy timestamp and duration values + buffer->set_timestamp(requested_au_->GetTimestamp()); + buffer->set_duration(requested_au_->GetDuration()); + + // enqueue buffer into appropriate stream + if (requested_au_->GetType() == DemuxerStream::AUDIO) { + audio_demuxer_stream_->EnqueueBuffer(buffer); + } else if (requested_au_->GetType() == DemuxerStream::VIDEO) { + video_demuxer_stream_->EnqueueBuffer(buffer); + } else { + NOTREACHED() << "invalid buffer type enqueued"; + } + + // finished with this au, deref + requested_au_ = nullptr; + + // Calculate total range of buffered data for both audio and video. + Ranges buffered( + audio_demuxer_stream_->GetBufferedRanges().IntersectionWith( + video_demuxer_stream_->GetBufferedRanges())); + // Notify host of each disjoint range. + host_->OnBufferedTimeRangesChanged(buffered); + + base::SequencedTaskRunner::GetCurrentDefault()->PostTask( + FROM_HERE, base::BindRepeating(&ProgressiveDemuxer::IssueNextRequest, + base::Unretained(this))); +} + +void ProgressiveDemuxer::IssueNextRequest() { + DCHECK(!requested_au_); + // if we're stopped don't download anymore + if (HasStopCalled()) { + LOG(INFO) << "stopped so request loop is stopping"; + return; + } + + DemuxerStream::Type type = DemuxerStream::UNKNOWN; + // if we have eos in one or both buffers the decision is easy + if (audio_reached_eos_ || video_reached_eos_) { + if (audio_reached_eos_) { + if (video_reached_eos_) { + // both are true, issue no more requests! + LOG(INFO) << "both streams at EOS, request loop stopping"; + return; + } else { + // audio is at eos, video isn't, get more video + type = DemuxerStream::VIDEO; + } + } else { + // audio is not at eos, video is, get more audio + type = DemuxerStream::AUDIO; + } + } else { + // priority order for figuring out what to download next + base::TimeDelta audio_stamp = + audio_demuxer_stream_->GetLastBufferTimestamp(); + base::TimeDelta video_stamp = + video_demuxer_stream_->GetLastBufferTimestamp(); + // if the audio demuxer stream is empty, always fill it first + if (audio_stamp == kNoTimestamp) { + type = DemuxerStream::AUDIO; + } else if (video_stamp == kNoTimestamp) { + // the video demuxer stream is empty, we need data for it + type = DemuxerStream::VIDEO; + } else if (video_stamp < audio_stamp) { + // video is earlier, fill it first + type = DemuxerStream::VIDEO; + } else { + type = DemuxerStream::AUDIO; + } + } + DCHECK_NE(type, DemuxerStream::UNKNOWN); + // We cannot call Request() directly even if this function is also run on + // |blocking_thread_| as otherwise it is possible that this function is + // running in a tight loop and seek or stop request has no chance to kick in. + base::SequencedTaskRunner::GetCurrentDefault()->PostTask( + FROM_HERE, base::BindRepeating(&ProgressiveDemuxer::Request, + base::Unretained(this), type)); +} + +void ProgressiveDemuxer::Stop() { + DCHECK(RunsTasksInCurrentSequence()); + // set our internal stop flag, to not treat read failures as + // errors anymore but as a natural part of stopping + { + base::AutoLock auto_lock(lock_for_stopped_); + stopped_ = true; + } + + // stop the reader, which will stop the datasource and call back + reader_->Stop(); +} + +bool ProgressiveDemuxer::HasStopCalled() { + base::AutoLock auto_lock(lock_for_stopped_); + return stopped_; +} + +void ProgressiveDemuxer::Seek(base::TimeDelta time, PipelineStatusCallback cb) { + blocking_thread_.task_runner()->PostTask( + FROM_HERE, + base::BindOnce(&ProgressiveDemuxer::SeekTask, base::Unretained(this), + time, BindPostTaskToCurrentDefault(std::move(cb)))); +} + +// runs on blocking thread +void ProgressiveDemuxer::SeekTask(base::TimeDelta time, + PipelineStatusCallback cb) { + LOG(INFO) << base::StringPrintf("seek to: %" PRId64 " ms", + time.InMilliseconds()); + // clear any enqueued buffers on demuxer streams + audio_demuxer_stream_->FlushBuffers(); + video_demuxer_stream_->FlushBuffers(); + // advance parser to new timestamp + if (!parser_->SeekTo(time)) { + LOG(ERROR) << "parser seek failed."; + std::move(cb).Run(PIPELINE_ERROR_READ); + return; + } + // if both streams had finished downloading, we need to restart the request + bool issue_new_request = audio_reached_eos_ && video_reached_eos_; + audio_reached_eos_ = false; + video_reached_eos_ = false; + flushing_ = true; + std::move(cb).Run(PIPELINE_OK); + if (issue_new_request) { + LOG(INFO) << "restarting stopped request loop"; + Request(DemuxerStream::AUDIO); + } +} + +std::vector ProgressiveDemuxer::GetAllStreams() { + return std::vector( + {audio_demuxer_stream_.get(), video_demuxer_stream_.get()}); +} + +base::TimeDelta ProgressiveDemuxer::GetStartTime() const { + // we always assume a start time of 0 + return base::TimeDelta(); +} + +const AudioDecoderConfig& ProgressiveDemuxer::AudioConfig() { + return parser_->AudioConfig(); +} + +const VideoDecoderConfig& ProgressiveDemuxer::VideoConfig() { + return parser_->VideoConfig(); +} + +} // namespace media diff --git a/media/starboard/progressive/progressive_demuxer.h b/media/starboard/progressive/progressive_demuxer.h new file mode 100644 index 000000000000..3518e8247dfc --- /dev/null +++ b/media/starboard/progressive/progressive_demuxer.h @@ -0,0 +1,204 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_PROGRESSIVE_DEMUXER_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_PROGRESSIVE_DEMUXER_H_ + +#include +#include +#include +#include + +#include "base/logging.h" +#include "base/task/sequenced_task_runner.h" +#include "base/threading/thread.h" +#include "media/base/decoder_buffer.h" +#include "media/base/demuxer.h" +#include "media/base/demuxer_stream.h" +#include "media/base/media_log.h" +#include "media/base/ranges.h" +#include "media/starboard/progressive/progressive_parser.h" + +namespace media { + +class DecoderBuffer; +class ProgressiveDemuxer; + +class ProgressiveDemuxerStream : public ::media::DemuxerStream { + public: + ProgressiveDemuxerStream(ProgressiveDemuxer* demuxer, Type type); + ProgressiveDemuxerStream(const ProgressiveDemuxerStream&) = delete; + ProgressiveDemuxerStream& operator=(const ProgressiveDemuxerStream&) = delete; + + // DemuxerStream implementation + void Read(uint32_t count, ReadCB read_cb) override; + AudioDecoderConfig audio_decoder_config() override; + VideoDecoderConfig video_decoder_config() override; + Type type() const override; + void EnableBitstreamConverter() override; + bool SupportsConfigChanges() override { return false; } + + // Functions used by ProgressiveDemuxer + ::media::Ranges GetBufferedRanges(); + void EnqueueBuffer(scoped_refptr buffer); + void FlushBuffers(); + void Stop(); + base::TimeDelta GetLastBufferTimestamp() const; + size_t GetTotalBufferSize() const; + size_t GetTotalBufferCount() const; + + private: + // The Ranges object doesn't offer a complement object so we rebuild + // enqueued ranges from the union of all of the buffers in the queue. + // Call me whenever _removing_ data from buffer_queue_. + void RebuildEnqueuedRanges_Locked(); + + // non-owning pointer to avoid circular reference + ProgressiveDemuxer* demuxer_; + Type type_; + + // Used to protect everything below. + mutable base::Lock lock_; + // Keeps track of all time ranges this object has seen since creation. + // The demuxer uses these ranges to update the pipeline about what data + // it has demuxed. + ::media::Ranges buffered_ranges_; + // The last timestamp of buffer enqueued. This is used in two places: + // 1. Used with the timestamp of the current frame to calculate the + // buffer range. + // 2. Used by the demuxer to deteminate what type of frame to get next. + base::TimeDelta last_buffer_timestamp_ = ::media::kNoTimestamp; + bool stopped_ = false; + + typedef std::deque> BufferQueue; + BufferQueue buffer_queue_; + + typedef std::deque ReadQueue; + ReadQueue read_queue_; + + size_t total_buffer_size_ = 0; + size_t total_buffer_count_ = 0; +}; + +class MEDIA_EXPORT ProgressiveDemuxer : public ::media::Demuxer { + public: + typedef ::media::AudioDecoderConfig AudioDecoderConfig; + typedef ::media::DecoderBuffer DecoderBuffer; + typedef ::media::DemuxerHost DemuxerHost; + typedef ::media::DemuxerStream DemuxerStream; + typedef ::media::MediaLog MediaLog; + typedef ::media::PipelineStatus PipelineStatus; + typedef ::media::PipelineStatusCallback PipelineStatusCallback; + typedef ::media::VideoDecoderConfig VideoDecoderConfig; + + ProgressiveDemuxer( + const scoped_refptr& task_runner, + DataSource* data_source, + MediaLog* const media_log); + ~ProgressiveDemuxer() override; + + // Demuxer implementation. + std::string GetDisplayName() const override { return "ProgressiveDemuxer"; } + ::media::DemuxerType GetDemuxerType() const override { + // kFFmpegDemuxer is used in Chromium media for progressive demuxing. + return ::media::DemuxerType::kFFmpegDemuxer; + } + void Initialize(DemuxerHost* host, PipelineStatusCallback status_cb) override; + void AbortPendingReads() override {} + void StartWaitingForSeek(base::TimeDelta seek_time) override {} + void CancelPendingSeek(base::TimeDelta seek_time) override {} + void Stop() override; + void Seek(base::TimeDelta time, PipelineStatusCallback cb) override; + bool IsSeekable() const override { return true; } + std::vector GetAllStreams() override; + base::TimeDelta GetStartTime() const override; + base::Time GetTimelineOffset() const override { return base::Time(); } + int64_t GetMemoryUsage() const override { + // TODO(b/322033277): Consider to pass memory usage for blink. + return 0; + } + absl::optional<::media::container_names::MediaContainerName> + GetContainerForMetrics() const override { + return absl::nullopt; + } + void OnEnabledAudioTracksChanged( + const std::vector<::media::MediaTrack::Id>& track_ids, + base::TimeDelta currTime, + TrackChangeCB change_completed_cb) override { + // NOTREACHED(); + } + void OnSelectedVideoTrackChanged( + const std::vector<::media::MediaTrack::Id>& track_ids, + base::TimeDelta currTime, + TrackChangeCB change_completed_cb) override { + // NOTREACHED(); + } + void SetPlaybackRate(double rate) override { + // NOTREACHED(); + } + + // TODO: Consider move the following functions to private section. + + // Issues a task to the demuxer to identify the next buffer of provided type + // in the stream, allocate memory to contain that buffer, download the bytes + // in to it, and enqueue the data in the appropriate demuxer stream. + void Request(DemuxerStream::Type type); + + // The DemuxerStream objects ask their parent ProgressiveDemuxer stream class + // for these configuration data rather than duplicating in the child classes + const AudioDecoderConfig& AudioConfig(); + const VideoDecoderConfig& VideoConfig(); + + // Provide access to ProgressiveDemuxerStream. + bool RunsTasksInCurrentSequence() const { + return task_runner_->RunsTasksInCurrentSequence(); + } + + private: + void ParseConfigDone(PipelineStatusCallback status_cb, PipelineStatus status); + bool HasStopCalled(); + + // methods that perform blocking I/O, and are therefore run on the + // blocking_thread_ download enough of the stream to parse the configuration. + void ParseConfigBlocking(PipelineStatusCallback status_cb); + void AllocateBuffer(); + void Download(scoped_refptr buffer); + void IssueNextRequest(); + void SeekTask(base::TimeDelta time, PipelineStatusCallback cb); + + scoped_refptr task_runner_; + DemuxerHost* host_; + + // Thread on which all blocking operations are executed. + base::Thread blocking_thread_; + DataSource* data_source_; + MediaLog* media_log_; + scoped_refptr reader_; + + base::Lock lock_for_stopped_; + bool stopped_; + bool flushing_; + + std::unique_ptr audio_demuxer_stream_; + std::unique_ptr video_demuxer_stream_; + scoped_refptr parser_; + + scoped_refptr requested_au_; + bool audio_reached_eos_; + bool video_reached_eos_; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_PROGRESSIVE_DEMUXER_H_ diff --git a/media/starboard/progressive/progressive_parser.cc b/media/starboard/progressive/progressive_parser.cc new file mode 100644 index 000000000000..b2c13aab7f13 --- /dev/null +++ b/media/starboard/progressive/progressive_parser.cc @@ -0,0 +1,60 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/progressive_parser.h" + +#include "base/logging.h" +#include "media/base/timestamp_constants.h" +#include "media/starboard/progressive/mp4_parser.h" + +namespace media { + +// ==== ProgressiveParser +// ============================================================ + +// how many bytes to download of the file to determine type? +const int ProgressiveParser::kInitialHeaderSize = 9; + +// static +PipelineStatus ProgressiveParser::Construct( + scoped_refptr reader, + scoped_refptr* parser, + MediaLog* media_log) { + DCHECK(parser); + DCHECK(media_log); + *parser = nullptr; + + // download first kInitialHeaderSize bytes of stream to determine file type + // and extract basic container-specific stream configuration information + uint8_t header[kInitialHeaderSize]; + int bytes_read = reader->BlockingRead(0, kInitialHeaderSize, header); + if (bytes_read != kInitialHeaderSize) { + return DEMUXER_ERROR_COULD_NOT_PARSE; + } + + // attempt to construct mp4 parser from this header + return MP4Parser::Construct(reader, header, parser, media_log); +} + +ProgressiveParser::ProgressiveParser(scoped_refptr reader) + : reader_(reader), duration_(kInfiniteDuration) {} + +ProgressiveParser::~ProgressiveParser() {} + +bool ProgressiveParser::IsConfigComplete() { + return video_config_.IsValidConfig() && audio_config_.IsValidConfig() && + duration_ != kInfiniteDuration; +} + +} // namespace media diff --git a/media/starboard/progressive/progressive_parser.h b/media/starboard/progressive/progressive_parser.h new file mode 100644 index 000000000000..1631263f12d1 --- /dev/null +++ b/media/starboard/progressive/progressive_parser.h @@ -0,0 +1,79 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_PROGRESSIVE_PARSER_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_PROGRESSIVE_PARSER_H_ + +#include "base/memory/ref_counted.h" +#include "media/base/audio_decoder_config.h" +#include "media/base/demuxer_stream.h" +#include "media/base/media_log.h" +#include "media/base/pipeline.h" +#include "media/base/video_decoder_config.h" +#include "media/starboard/progressive/avc_access_unit.h" +#include "media/starboard/progressive/data_source_reader.h" + +namespace media { + +// abstract base class to define a stream parser interface used by +// ProgressiveDemuxer. +class ProgressiveParser : public base::RefCountedThreadSafe { + public: + static const int kInitialHeaderSize; + // Determine stream type, construct appropriate parser object, and returns + // PIPELINE_OK on success or error code. + static PipelineStatus Construct(scoped_refptr reader, + scoped_refptr* parser, + MediaLog* media_log); + explicit ProgressiveParser(scoped_refptr reader); + + // Seek through the file looking for audio and video configuration info, + // saving as much config state as is possible. Should try to be fast but this + // may result in the downloading of MB of data. Returns false on fatal error. + virtual bool ParseConfig() = 0; + + // Returns a populated, valid AU indicating the needed information for + // downloading and decoding the next access unit in the stream, or NULL on + // fatal error. On success this advances the respective audio or video cursor + // to the next AU. + virtual scoped_refptr GetNextAU(DemuxerStream::Type type) = 0; + // Write the appropriate prepend header for the supplied au into the supplied + // buffer. Return false on error. + virtual bool Prepend(scoped_refptr au, + scoped_refptr buffer) = 0; + // Advance internal state to provided timestamp. Return false on error. + virtual bool SeekTo(base::TimeDelta timestamp) = 0; + + // ======= config state methods, values should be set by ParseConfig() + // Returns true if all of the required variables defined below are valid. + // BitsPerSecond() is optional. + virtual bool IsConfigComplete(); + // time-duration of file, may return kInfiniteDuration() if unknown + virtual base::TimeDelta Duration() { return duration_; } + virtual const AudioDecoderConfig& AudioConfig() { return audio_config_; } + virtual const VideoDecoderConfig& VideoConfig() { return video_config_; } + + protected: + // only allow RefCountedThreadSafe to delete us + friend class base::RefCountedThreadSafe; + virtual ~ProgressiveParser(); + scoped_refptr reader_; + AudioDecoderConfig audio_config_; + VideoDecoderConfig video_config_; + base::TimeDelta duration_; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_PROGRESSIVE_PARSER_H_ diff --git a/media/starboard/progressive/rbsp_stream.cc b/media/starboard/progressive/rbsp_stream.cc new file mode 100644 index 000000000000..0d167d49a8f3 --- /dev/null +++ b/media/starboard/progressive/rbsp_stream.cc @@ -0,0 +1,215 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/rbsp_stream.h" + +#include "base/check.h" +#include "base/logging.h" + +namespace media { + +RBSPStream::RBSPStream(const uint8_t* nalu_buffer, size_t nalu_buffer_size) + : nalu_buffer_(nalu_buffer), + nalu_buffer_size_(nalu_buffer_size), + nalu_buffer_byte_offset_(0), + current_nalu_byte_(0), + number_consecutive_zeros_(0), + rbsp_bit_offset_(0) {} + +// read unsigned Exp-Golomb coded integer, ISO 14496-10 Section 9.1 +bool RBSPStream::ReadUEV(uint32_t* uev_out) { + CHECK(uev_out); + int leading_zero_bits = -1; + for (uint8_t b = 0; b == 0; leading_zero_bits++) { + if (!ReadRBSPBit(&b)) { + return false; + } + } + // we can only fit 31 bits of Exp-Golomb coded data into a 32-bit number + if (leading_zero_bits >= 32) { + return false; + } + uint32_t result = (1 << leading_zero_bits) - 1; + uint32_t remainder = 0; + if (!ReadBits(leading_zero_bits, &remainder)) { + return false; + } + result += remainder; + *uev_out = result; + return true; +} + +// read signed Exp-Golomb coded integer, ISO 14496-10 Section 9.1 +bool RBSPStream::ReadSEV(int32_t* sev_out) { + CHECK(sev_out); + // we start off by reading an unsigned Exp-Golomb coded number + uint32_t uev = 0; + if (!ReadUEV(&uev)) { + return false; + } + // the LSb in this number is treated as the inverted sign bit + bool is_negative = !(uev & 1); + int32_t result = static_cast((uev + 1) >> 1); + if (is_negative) { + result *= -1; + } + *sev_out = result; + return true; +} + +// read and return up to 32 bits, filling from the right, meaning that +// ReadBits(17) on a stream of all 1s would return 0x01ffff +bool RBSPStream::ReadBits(size_t bits, uint32_t* bits_out) { + CHECK(bits_out); + if (bits > 32) { + return false; + } + if (bits == 0) { + return true; + } + uint32_t result = 0; + size_t bytes = bits >> 3; + // read bytes first + for (int i = 0; i < bytes; i++) { + uint8_t new_byte = 0; + if (!ReadRBSPByte(&new_byte)) { + return false; + } + result = result << 8; + result = result | static_cast(new_byte); + } + // scoot any leftover bits in + bits = bits % 8; + for (int i = 0; i < bits; i++) { + uint8_t new_bit = 0; + if (!ReadRBSPBit(&new_bit)) { + return false; + } + result = result << 1; + result = result | static_cast(new_bit); + } + *bits_out = result; + return true; +} + +// jump over bytes in the RBSP stream +bool RBSPStream::SkipBytes(size_t bytes) { + for (int i = 0; i < bytes; ++i) { + if (!ConsumeNALUByte()) { + return false; + } + } + return true; +} + +// jump over bits in the RBSP stream +bool RBSPStream::SkipBits(size_t bits) { + // skip bytes first + size_t bytes = bits >> 3; + if (bytes > 0) { + if (!SkipBytes(bytes)) { + return false; + } + } + // mask off byte skips + bits = bits & 7; + // if no bits left to skip just return + if (bits == 0) { + return true; + } + // obey the convention that if our bit offset is 0 we haven't loaded the + // current byte, extract it from NALU stream as we are going to advance + // the bit cursor in to it (or potentially past it) + if (rbsp_bit_offset_ == 0) { + if (!ConsumeNALUByte()) { + return false; + } + } + // add to our bit offset + rbsp_bit_offset_ += bits; + // if we jumped in to the next byte advance the NALU stream, respecting the + // convention that if we're at 8 bits stay on the current byte + if (rbsp_bit_offset_ >= 9) { + if (!ConsumeNALUByte()) { + return false; + } + } + rbsp_bit_offset_ = rbsp_bit_offset_ % 8; + return true; +} + +// advance by one byte through the NALU buffer, respecting the encoding of +// 00 00 03 => 00 00. Updates the state of current_nalu_byte_ to the new value. +bool RBSPStream::ConsumeNALUByte() { + if (nalu_buffer_byte_offset_ >= nalu_buffer_size_) { + return false; + } + current_nalu_byte_ = nalu_buffer_[nalu_buffer_byte_offset_]; + if (current_nalu_byte_ == 0x03 && number_consecutive_zeros_ >= 2) { + ++nalu_buffer_byte_offset_; + current_nalu_byte_ = nalu_buffer_[nalu_buffer_byte_offset_]; + number_consecutive_zeros_ = 0; + } + + if (current_nalu_byte_ == 0) { + ++number_consecutive_zeros_; + } else { + number_consecutive_zeros_ = 0; + } + ++nalu_buffer_byte_offset_; + return true; +} + +// return single bit in the LSb from the RBSP stream. Bits are read from MSb +// to LSb in the stream. +bool RBSPStream::ReadRBSPBit(uint8_t* bit_out) { + CHECK(bit_out); + // check to see if we need to consume a fresh byte + if (rbsp_bit_offset_ == 0) { + if (!ConsumeNALUByte()) { + return false; + } + } + // since we read from MSb to LSb in stream we shift right + uint8_t bit = (current_nalu_byte_ >> (7 - rbsp_bit_offset_)) & 1; + // increment bit offset + rbsp_bit_offset_ = (rbsp_bit_offset_ + 1) % 8; + *bit_out = bit; + return true; +} + +bool RBSPStream::ReadRBSPByte(uint8_t* byte_out) { + CHECK(byte_out); + // fast path for byte-aligned access + if (rbsp_bit_offset_ == 0) { + if (!ConsumeNALUByte()) { + return false; + } + *byte_out = current_nalu_byte_; + return true; + } + // at least some of the bits in the current byte will be included in this + // next byte, absorb them + uint8_t upper_part = current_nalu_byte_; + // read next byte from stream + if (!ConsumeNALUByte()) { + return false; + } + // form the byte from the two bytes + *byte_out = (upper_part << rbsp_bit_offset_) | + (current_nalu_byte_ >> (8 - rbsp_bit_offset_)); + return true; +} + +} // namespace media diff --git a/media/starboard/progressive/rbsp_stream.h b/media/starboard/progressive/rbsp_stream.h new file mode 100644 index 000000000000..cbdbc91072e7 --- /dev/null +++ b/media/starboard/progressive/rbsp_stream.h @@ -0,0 +1,71 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIA_STARBOARD_PROGRESSIVE_RBSP_STREAM_H_ +#define MEDIA_STARBOARD_PROGRESSIVE_RBSP_STREAM_H_ + +#include + +namespace media { + +// ISO 14496-10 describes a byte encoding format for NALUs (network abstraction +// layer units) and rules to convert it into a RBSP stream, which is the format +// that some other atoms are defined. This class takes a non-owning reference +// to a buffer and extract various types from the stream while silently +// consuming the extra encoding bytes and advancing a bit stream pointer. +class RBSPStream { + public: + // NON-OWNING pointer to buffer. It is assumed the client will dispose of + // this buffer. + RBSPStream(const uint8_t* nalu_buffer, size_t nalu_buffer_size); + // all Read/Skip methods return the value by reference and return true + // on success, false on read error/EOB. Once the object has returned + // false the consistency of the data is not guaranteed. + // read unsigned Exp-Golomb coded integer, ISO 14496-10 Section 9.1 + bool ReadUEV(uint32_t* uev_out); + // read signed Exp-Golomb coded integer, ISO 14496-10 Section 9.1 + bool ReadSEV(int32_t* sev_out); + // read and return up to 32 bits, filling from the right, meaning that + // ReadBits(17) on a stream of all 1s would return 0x01ffff + bool ReadBits(size_t bits, uint32_t* bits_out); + bool ReadByte(uint8_t* byte_out) { return ReadRBSPByte(byte_out); } + bool ReadBit(uint8_t* bit_out) { return ReadRBSPBit(bit_out); } + // jump over bytes in the RBSP stream + bool SkipBytes(size_t bytes); + // jump over bits in the RBSP stream + bool SkipBits(size_t bits); + + private: + // advance by one byte through the NALU buffer, respecting the encoding of + // 00 00 03 => 00 00. Updates the state of current_nalu_byte_ to the new + // value. + // returns false if we have moved past the end of the buffer. + bool ConsumeNALUByte(); + // return single bit in the LSb from the RBSP stream. Bits are read from MSb + // to LSb in the stream. + bool ReadRBSPBit(uint8_t* bit_out); + bool ReadRBSPByte(uint8_t* byte_out); + + const uint8_t* nalu_buffer_; + size_t nalu_buffer_size_; + size_t nalu_buffer_byte_offset_; + uint8_t current_nalu_byte_; + int number_consecutive_zeros_; + // location of rbsp bit cursor within current_nalu_byte_ + size_t rbsp_bit_offset_; +}; + +} // namespace media + +#endif // MEDIA_STARBOARD_PROGRESSIVE_RBSP_STREAM_H_ diff --git a/media/starboard/progressive/rbsp_stream_unittest.cc b/media/starboard/progressive/rbsp_stream_unittest.cc new file mode 100644 index 000000000000..bb2130b351ef --- /dev/null +++ b/media/starboard/progressive/rbsp_stream_unittest.cc @@ -0,0 +1,635 @@ +// Copyright 2012 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/starboard/progressive/rbsp_stream.h" + +#include +#include +#include + +#include "base/logging.h" +#include "base/strings/stringprintf.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +class RBSPStreamTest : public testing::Test { + protected: + RBSPStreamTest() {} + + virtual ~RBSPStreamTest() {} + + // Given num encode the value in signed exp-golomb syntax and push + // the value on the provided bitlist + void EncodeSEV(int32 num, std::list& bits) { + bool is_negative = (num < 0); + uint32 unum = 0; + if (is_negative) { + unum = static_cast(num * -1); + } else { + unum = static_cast(num); + } + // multiply unsigned value by 2 + unum = unum << 1; + // subtract one from the positive values + if (!is_negative) { + --unum; + } + // encode the resulting uev + EncodeUEV(unum, bits); + } + + // Given num encode the value in unsigned exp-golomb syntax and push + // the value on to the provided bitlist + void EncodeUEV(uint32 num, std::list& bits) { + // find largest (2^pow) - 1 smaller than num + uint32 pow = 31; + uint32 base = 0x7fffffff; + while (base > num) { + pow--; + base = base >> 1; + } + // encoding calls for pow leading zeros, followed by a 1, followed + // by pow digits of the input number - ((2^pow) - 1). + // we move from MSb to LSb, so start by pushing back the leading 0s + for (int i = 0; i < pow; i++) { + bits.push_back(false); + } + // now push the separating one + bits.push_back(true); + // and now pow bits of the remainder bitfield MSb to LSb + uint32 remainder = num - base; + for (int i = pow - 1; i >= 0; --i) { + bits.push_back((remainder >> i) & 0x01); + } + } + + // after building a bitlist in various fun ways call this method to + // create a buffer on the heap that can be passed to RBSPStream + // for deserialization. + std::unique_ptr SerializeToBuffer(const std::list& bitlist, + bool add_sequence_bytes, + size_t& buffer_size_out) { + // start by building a list of bytes, so we can add the + // 00 00 => 00 00 03 sequence bytes + std::list bytelist; + uint8 push_byte = 0; + uint32 bit_counter = 0; + for (std::list::const_iterator it = bitlist.begin(); + it != bitlist.end(); ++it) { + bit_counter++; + push_byte = push_byte << 1; + if (*it) { + push_byte |= 1; + } + if (!(bit_counter % 8)) { + bytelist.push_back(push_byte); + push_byte = 0; + } + } + // push any remaining bits on as the final byte + if (bit_counter % 8) { + bytelist.push_back(push_byte << (8 - (bit_counter % 8))); + } + // if we should add sequence bytes we iterate through the new + // byte list looking for 00 00 and inserting a 03 after each. + if (add_sequence_bytes) { + int num_zeros = 0; + for (std::list::iterator it = bytelist.begin(); + it != bytelist.end(); ++it) { + // if we just passed two sequential zeros insert a 03 + if (num_zeros == 2) { + bytelist.insert(it, 0x03); + // reset the counter + num_zeros = 0; + } + if (*it == 0) { + ++num_zeros; + } else { + num_zeros = 0; + } + } + } else { + // we will need to detect any naturally occurring 00 00 03s + // and protect them from removal of the 03, by inserting a + // second 03 + int num_zeros = 0; + for (std::list::iterator it = bytelist.begin(); + it != bytelist.end(); ++it) { + if ((num_zeros >= 2) && (*it == 0x03)) { + bytelist.insert(it, 0x03); + } + if (*it == 0) { + ++num_zeros; + } else { + num_zeros = 0; + } + } + } + // alright we can make the final output buffer + std::unique_ptr buf(new uint8[bytelist.size()]); + int index = 0; + for (std::list::iterator it = bytelist.begin(); it != bytelist.end(); + it++) { + buf[index] = *it; + index++; + } + buffer_size_out = bytelist.size(); + return std::move(buf); + } +}; + +TEST_F(RBSPStreamTest, ReadUEV) { + std::list fibbits; + // encode first 47 Fibonacci numbers + uint32 f_n_minus_2 = 0; + EncodeUEV(f_n_minus_2, fibbits); + uint32 f_n_minus_1 = 1; + EncodeUEV(f_n_minus_1, fibbits); + for (int i = 2; i < 47; i++) { + uint32 f_n = f_n_minus_1 + f_n_minus_2; + EncodeUEV(f_n, fibbits); + // update values + f_n_minus_2 = f_n_minus_1; + f_n_minus_1 = f_n; + } + // convert to buffer + size_t fib_buffer_size = 0; + std::unique_ptr fib_buffer = + SerializeToBuffer(fibbits, true, fib_buffer_size); + size_t fib_buffer_no_sequence_size; + std::unique_ptr fib_buffer_no_sequence = + SerializeToBuffer(fibbits, false, fib_buffer_no_sequence_size); + RBSPStream fib_stream(fib_buffer.get(), fib_buffer_size); + RBSPStream fib_stream_no_sequence(fib_buffer_no_sequence.get(), + fib_buffer_no_sequence_size); + // deserialize the same sequence from both buffers + uint32 uev = 0; + uint32 uev_n = 0; + f_n_minus_2 = 0; + ASSERT_TRUE(fib_stream.ReadUEV(&uev)); + ASSERT_EQ(uev, f_n_minus_2); + ASSERT_TRUE(fib_stream_no_sequence.ReadUEV(&uev_n)); + ASSERT_EQ(uev_n, f_n_minus_2); + + f_n_minus_1 = 1; + ASSERT_TRUE(fib_stream.ReadUEV(&uev)); + ASSERT_EQ(uev, f_n_minus_1); + ASSERT_TRUE(fib_stream_no_sequence.ReadUEV(&uev_n)); + ASSERT_EQ(uev_n, f_n_minus_1); + + for (int i = 2; i < 47; i++) { + uint32 f_n = f_n_minus_1 + f_n_minus_2; + ASSERT_TRUE(fib_stream.ReadUEV(&uev)); + ASSERT_EQ(uev, f_n); + ASSERT_TRUE(fib_stream_no_sequence.ReadUEV(&uev_n)); + ASSERT_EQ(uev_n, f_n); + f_n_minus_2 = f_n_minus_1; + f_n_minus_1 = f_n; + } + // subsequent call to ReadUEV should fail + ASSERT_FALSE(fib_stream.ReadUEV(&uev)); + ASSERT_FALSE(fib_stream_no_sequence.ReadUEV(&uev_n)); +} + +TEST_F(RBSPStreamTest, ReadSEV) { + std::list lucasbits; + // encode first 44 Lucas numbers with alternating sign + int32 l_n_minus_2 = 1; + EncodeSEV(l_n_minus_2, lucasbits); + int32 l_n_minus_1 = 2; + EncodeSEV(-l_n_minus_1, lucasbits); + for (int i = 2; i < 44; ++i) { + int32 l_n = l_n_minus_1 + l_n_minus_2; + if (i % 2) { + EncodeSEV(-l_n, lucasbits); + } else { + EncodeSEV(l_n, lucasbits); + } + l_n_minus_2 = l_n_minus_1; + l_n_minus_1 = l_n; + } + // convert to buffers + size_t lucas_seq_buffer_size = 0; + std::unique_ptr lucas_seq_buffer = + SerializeToBuffer(lucasbits, true, lucas_seq_buffer_size); + size_t lucas_deseq_buffer_size = 0; + std::unique_ptr lucas_deseq_buffer = + SerializeToBuffer(lucasbits, false, lucas_deseq_buffer_size); + RBSPStream lucas_seq_stream(lucas_seq_buffer.get(), lucas_seq_buffer_size); + RBSPStream lucas_deseq_stream(lucas_deseq_buffer.get(), + lucas_deseq_buffer_size); + l_n_minus_2 = 1; + l_n_minus_1 = 2; + int32 sev = 0; + int32 sev_n = 0; + ASSERT_TRUE(lucas_seq_stream.ReadSEV(&sev)); + ASSERT_EQ(sev, 1); + ASSERT_TRUE(lucas_deseq_stream.ReadSEV(&sev_n)); + ASSERT_EQ(sev_n, 1); + ASSERT_TRUE(lucas_seq_stream.ReadSEV(&sev)); + ASSERT_EQ(sev, -2); + ASSERT_TRUE(lucas_deseq_stream.ReadSEV(&sev_n)); + ASSERT_EQ(sev_n, -2); + for (int i = 2; i < 44; ++i) { + int32 l_n = l_n_minus_1 + l_n_minus_2; + ASSERT_TRUE(lucas_seq_stream.ReadSEV(&sev)); + ASSERT_TRUE(lucas_deseq_stream.ReadSEV(&sev_n)); + if (i % 2) { + ASSERT_EQ(-sev, l_n); + ASSERT_EQ(-sev_n, l_n); + } else { + ASSERT_EQ(sev, l_n); + ASSERT_EQ(sev_n, l_n); + } + l_n_minus_2 = l_n_minus_1; + l_n_minus_1 = l_n; + } + // subsequent calls to ReadSEV should fail + ASSERT_FALSE(lucas_seq_stream.ReadSEV(&sev)); + ASSERT_FALSE(lucas_deseq_stream.ReadSEV(&sev_n)); +} + +static const uint8 kTestRBSPExpGolombTooBig[] = { + // 15 leading zeros, should be fine + // 0000000000000001010101010101010 + // = 2^15 - 1 + read_bits(010101010101010) + // = 32768 - 1 + 10922 = 43689 unsigned, 21845 signed + // 0000 0000 0000 0001 0101 0101 0101 010+0 (first 0 of next number) + 0x00, 0x01, 0x55, 0x54, + // 31 leading zeros, should be fine + // 000000000000000000000000000000010000000000000000000000000000001 + // = 2^31 - 1 + 1 = 2147483648 unsigned, -1073741824 signed + // 0 appended on to last byte + // 0000 0000 0000 0000 0000 0000 0000 0010 0000 0000 0000 0000 0000 0000 + 0x00, 0x00, 0x03, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, + // 0000 01+00 (first 2 zeros of next number) + 0x04, + // 32 leading zeros, should not be ok + // 00000000000000000000000000000000111111111111111111111111111111111 + // = 2^32 - 1 + 2^32 = 2^33 - 1 = 8589934591 + // 00 appended on to last byte + // 0000 0000 0000 0000 0000 0000 0000 0011 1111 1111 1111 1111 1111 1111 + 0x00, 0x00, 0x00, 0x03, 0x03, 0xff, 0xff, 0xff, + // 1111 111+0 (to complete the byte) + 0xfe}; + +TEST_F(RBSPStreamTest, ReadUEVTooLarge) { + // construct a stream from the supplied test data + RBSPStream uev_too_big(kTestRBSPExpGolombTooBig, + sizeof(kTestRBSPExpGolombTooBig)); + // first call should succeed + uint32 uev = 0; + ASSERT_TRUE(uev_too_big.ReadUEV(&uev)); + ASSERT_EQ(uev, 43689); + // as should the second call + ASSERT_TRUE(uev_too_big.ReadUEV(&uev)); + ASSERT_EQ(uev, 2147483648u); + // third should fail + ASSERT_FALSE(uev_too_big.ReadUEV(&uev)); +} + +TEST_F(RBSPStreamTest, ReadSEVTooLarge) { + // construct a stream from the supplied test data + RBSPStream sev_too_big(kTestRBSPExpGolombTooBig, + sizeof(kTestRBSPExpGolombTooBig)); + // first call should succeed + int32 sev = 0; + ASSERT_TRUE(sev_too_big.ReadSEV(&sev)); + ASSERT_EQ(sev, 21845); + // as should the second call + ASSERT_TRUE(sev_too_big.ReadSEV(&sev)); + ASSERT_EQ(sev, -1073741824); + // third should fail + ASSERT_FALSE(sev_too_big.ReadSEV(&sev)); +} + +TEST_F(RBSPStreamTest, ReadBit) { + std::list padded_ones; + // build a bitfield of 1 padded by n zeros, for n in range[0, 1024] + for (int i = 0; i < 1024; i++) { + for (int j = 0; j < i; j++) { + padded_ones.push_back(false); + } + padded_ones.push_back(true); + } + // build the buffer with sequence bits and without + size_t sequence_buff_size = 0; + std::unique_ptr sequence_buff = + SerializeToBuffer(padded_ones, true, sequence_buff_size); + RBSPStream seq_stream(sequence_buff.get(), sequence_buff_size); + + size_t desequence_buff_size = 0; + std::unique_ptr desequence_buff = + SerializeToBuffer(padded_ones, false, desequence_buff_size); + RBSPStream deseq_stream(desequence_buff.get(), desequence_buff_size); + for (std::list::iterator it = padded_ones.begin(); + it != padded_ones.end(); ++it) { + uint8 bit = 0; + ASSERT_TRUE(seq_stream.ReadBit(&bit)); + ASSERT_EQ(*it, static_cast(bit)); + uint8 deseq_bit = 0; + ASSERT_TRUE(deseq_stream.ReadBit(&deseq_bit)); + ASSERT_EQ(*it, static_cast(deseq_bit)); + } + + // there should be less than a byte in the either stream + uint8 fail_byte = 0; + ASSERT_FALSE(seq_stream.ReadByte(&fail_byte)); + ASSERT_FALSE(deseq_stream.ReadByte(&fail_byte)); +} + +TEST_F(RBSPStreamTest, ReadByte) { + // build a field of 16 x (0xaa byte followed by 0 bit) + std::list aa_field; + for (int i = 0; i < 16; ++i) { + for (int j = 0; j < 8; ++j) { + aa_field.push_back(!(j % 2)); + } + aa_field.push_back(false); + } + // deseqbuff will be identical due to dense packing of 01 pattern + size_t aabuff_size = 0; + std::unique_ptr aabuff = + SerializeToBuffer(aa_field, true, aabuff_size); + RBSPStream aa_stream(aabuff.get(), aabuff_size); + for (int i = 0; i < 16; ++i) { + uint8 aa = 0; + ASSERT_TRUE(aa_stream.ReadByte(&aa)); + ASSERT_EQ(aa, 0xaa); + // read the zero separator bit + uint8 zero = 0; + ASSERT_TRUE(aa_stream.ReadBit(&zero)); + ASSERT_EQ(zero, 0); + } + + // build a field of 24 x (1 bit, 4 bytes of 0, one 03 byte, 4 bytes of 0) + std::list zero_field; + for (int i = 0; i < 24; ++i) { + zero_field.push_back(true); + for (int j = 0; j < 32; ++j) { + zero_field.push_back(false); + } + zero_field.push_back(false); + zero_field.push_back(false); + zero_field.push_back(false); + zero_field.push_back(false); + zero_field.push_back(false); + zero_field.push_back(false); + zero_field.push_back(true); + zero_field.push_back(true); + for (int j = 0; j < 32; ++j) { + zero_field.push_back(false); + } + } + size_t zseqbuff_size = 0; + std::unique_ptr zseqbuff = + SerializeToBuffer(zero_field, true, zseqbuff_size); + RBSPStream zseq_stream(zseqbuff.get(), zseqbuff_size); + size_t zdseqbuff_size = 0; + std::unique_ptr zdseqbuff = + SerializeToBuffer(zero_field, false, zdseqbuff_size); + RBSPStream zdseq_stream(zdseqbuff.get(), zdseqbuff_size); + for (int i = 0; i < 24; ++i) { + // read the leading 1 bit + uint8 seq_bit = 0; + ASSERT_TRUE(zseq_stream.ReadBit(&seq_bit)); + ASSERT_EQ(seq_bit, 1); + uint8 dseq_bit = 0; + ASSERT_TRUE(zdseq_stream.ReadBit(&dseq_bit)); + ASSERT_EQ(dseq_bit, 1); + // read 4 zeros + uint8 seq_byte = 0; + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + uint8 dseq_byte = 0; + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + // read the 3 + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0x03); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0x03); + // read the remaining 4 zeros + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + ASSERT_TRUE(zseq_stream.ReadByte(&seq_byte)); + ASSERT_EQ(seq_byte, 0); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + ASSERT_TRUE(zdseq_stream.ReadByte(&dseq_byte)); + ASSERT_EQ(dseq_byte, 0); + } +} + +TEST_F(RBSPStreamTest, ReadBits) { + // test the assertion in the ReadBits comment, as it had a bug :) + std::list seventeen_ones; + for (int i = 0; i < 17; ++i) { + seventeen_ones.push_back(true); + } + size_t seventeen_ones_size = 0; + std::unique_ptr seventeen_ones_buff = + SerializeToBuffer(seventeen_ones, false, seventeen_ones_size); + RBSPStream seventeen_ones_stream(seventeen_ones_buff.get(), + seventeen_ones_size); + uint32 seventeen_ones_word = 0; + ASSERT_TRUE(seventeen_ones_stream.ReadBits(17, &seventeen_ones_word)); + ASSERT_EQ(seventeen_ones_word, 0x0001ffff); + + // serialize all powers of two from 2^0 to 2^31 + std::list pows; + for (int i = 0; i < 32; ++i) { + pows.push_back(true); + for (int j = 0; j < i; ++j) { + pows.push_back(false); + } + } + size_t pows_size = 0; + std::unique_ptr pows_buff = SerializeToBuffer(pows, true, pows_size); + RBSPStream pows_stream(pows_buff.get(), pows_size); + // ReadBits(0) should succeed and not modify the value of the ref output or + // internal bit iterator + uint32 dont_touch = 0xfeedfeed; + ASSERT_TRUE(pows_stream.ReadBits(0, &dont_touch)); + ASSERT_EQ(dont_touch, 0xfeedfeed); + // compare deserializations + for (int i = 0; i < 32; ++i) { + uint32 bits = 0; + ASSERT_TRUE(pows_stream.ReadBits(i + 1, &bits)); + ASSERT_EQ(bits, (uint32)(1 << i)); + } +} + +TEST_F(RBSPStreamTest, SkipBytes) { + // serialize all nine-bit values from zero to 512 + std::list nines; + for (int i = 0; i < 512; ++i) { + for (int j = 8; j >= 0; --j) { + nines.push_back((i >> j) & 1); + } + } + size_t nines_size = 0; + std::unique_ptr nines_buff = + SerializeToBuffer(nines, true, nines_size); + size_t nines_deseq_size = 0; + std::unique_ptr nines_deseq_buff = + SerializeToBuffer(nines, false, nines_deseq_size); + RBSPStream nines_stream(nines_buff.get(), nines_size); + RBSPStream nines_deseq_stream(nines_deseq_buff.get(), nines_deseq_size); + // iterate through streams, skipping in one and reading in the other, always + // comparing values. + for (int i = 0; i < 512; ++i) { + if (i % 2) { + ASSERT_TRUE(nines_stream.SkipBytes(1)); + uint8 bit = 0; + ASSERT_TRUE(nines_stream.ReadBit(&bit)); + uint32 ninebits = 0; + ASSERT_TRUE(nines_deseq_stream.ReadBits(9, &ninebits)); + ASSERT_EQ(ninebits, i); + ASSERT_EQ(ninebits & 1, bit); + } else { + ASSERT_TRUE(nines_deseq_stream.SkipBytes(1)); + uint8 bit = 0; + ASSERT_TRUE(nines_deseq_stream.ReadBit(&bit)); + uint32 ninebits = 0; + ASSERT_TRUE(nines_stream.ReadBits(9, &ninebits)); + ASSERT_EQ(ninebits, i); + ASSERT_EQ(ninebits & 1, bit); + } + } + // 1 true bit followed by 1 byte with 1, followed by 1 true bit, then 2 bytes + // with 2, followed by 1 bit, then 3 bytes with 3, etc up to 256 + std::list run_length; + for (int i = 0; i < 256; ++i) { + for (int j = 0; j < i; ++j) { + for (int k = 7; k >= 0; --k) { + run_length.push_back((i >> k) & 1); + } + } + run_length.push_back(true); + } + size_t run_length_size = 0; + std::unique_ptr run_length_buff = + SerializeToBuffer(run_length, true, run_length_size); + size_t run_length_deseq_size = 0; + std::unique_ptr run_length_deseq_buff = + SerializeToBuffer(run_length, false, run_length_deseq_size); + RBSPStream run_length_stream(run_length_buff.get(), run_length_size); + RBSPStream run_length_deseq_stream(run_length_deseq_buff.get(), + run_length_deseq_size); + // read first bit, skip first byte from each stream, read next bit + uint8 bit = 0; + ASSERT_TRUE(run_length_stream.ReadBit(&bit)); + ASSERT_EQ(bit, 1); + bit = 0; + ASSERT_TRUE(run_length_deseq_stream.ReadBit(&bit)); + ASSERT_EQ(bit, 1); + ASSERT_TRUE(run_length_stream.SkipBytes(1)); + ASSERT_TRUE(run_length_deseq_stream.SkipBytes(1)); + bit = 0; + ASSERT_TRUE(run_length_stream.ReadBit(&bit)); + ASSERT_EQ(bit, 1); + bit = 0; + ASSERT_TRUE(run_length_deseq_stream.ReadBit(&bit)); + ASSERT_EQ(bit, 1); + + for (int i = 2; i < 256; ++i) { + // read first byte in seq stream, make sure it matches value + uint8 byte = 0; + ASSERT_TRUE(run_length_stream.ReadByte(&byte)); + ASSERT_EQ(byte, i); + // skip the rest of the byte field + ASSERT_TRUE(run_length_stream.SkipBytes(i - 1)); + bit = 0; + // read the separating one bit + ASSERT_TRUE(run_length_stream.ReadBit(&bit)); + ASSERT_EQ(bit, 1); + // read last byte in deseq stream, so skip bytes first + ASSERT_TRUE(run_length_deseq_stream.SkipBytes(i - 1)); + byte = 0; + ASSERT_TRUE(run_length_deseq_stream.ReadByte(&byte)); + ASSERT_EQ(byte, i); + // read the separating one bit + bit = 0; + ASSERT_TRUE(run_length_deseq_stream.ReadBit(&bit)); + ASSERT_EQ(bit, 1); + } + + // further skips should fail + ASSERT_FALSE(run_length_stream.SkipBytes(1)); + ASSERT_FALSE(run_length_deseq_stream.SkipBytes(1)); +} + +TEST_F(RBSPStreamTest, SkipBits) { + std::list one_ohs; + // encode one 1, followed by one zero, followed by 2 1s, followed by 2 zeros, + // etc + for (int i = 1; i <= 64; ++i) { + for (int j = 0; j < i; ++j) { + one_ohs.push_back(true); + } + for (int j = 0; j < i; ++j) { + one_ohs.push_back(false); + } + } + size_t skip_ones_size = 0; + std::unique_ptr skip_ones_buff = + SerializeToBuffer(one_ohs, true, skip_ones_size); + size_t skip_ohs_size = 0; + std::unique_ptr skip_ohs_buff = + SerializeToBuffer(one_ohs, false, skip_ohs_size); + RBSPStream skip_ones(skip_ones_buff.get(), skip_ones_size); + RBSPStream skip_ohs(skip_ohs_buff.get(), skip_ohs_size); + for (int i = 1; i < 64; ++i) { + // skip the ones + ASSERT_TRUE(skip_ones.SkipBits(i)); + // read the ones from the zeros stream + for (int j = 0; j < i; ++j) { + uint8 bit = 0; + ASSERT_TRUE(skip_ohs.ReadBit(&bit)); + ASSERT_EQ(bit, 1); + } + // skip the ohs + ASSERT_TRUE(skip_ohs.SkipBits(i)); + // read the ohs from the ones stream + for (int j = 0; j < i; ++j) { + uint8 bit = 0; + ASSERT_TRUE(skip_ones.ReadBit(&bit)); + ASSERT_EQ(bit, 0); + } + } +} + +} // namespace media diff --git a/third_party/blink/renderer/platform/media/web_media_player_impl.cc b/third_party/blink/renderer/platform/media/web_media_player_impl.cc index 0c8211b9a03d..87739e76856b 100644 --- a/third_party/blink/renderer/platform/media/web_media_player_impl.cc +++ b/third_party/blink/renderer/platform/media/web_media_player_impl.cc @@ -826,6 +826,13 @@ void WebMediaPlayerImpl::DoLoad(LoadType load_type, // Note: `url` may be very large, take care when making copies. loaded_url_ = GURL(url); load_type_ = load_type; + if (load_type_ == kLoadTypeURL) { + LOG(ERROR) << "Cobalt " << __func__ << " kLoadTypeURL " << loaded_url_.spec(); + } else if (load_type_ == kLoadTypeMediaSource) { + LOG(ERROR) << "Cobalt " << __func__ << " kLoadTypeMediaSource " << loaded_url_.spec(); + } else if (load_type_ == kLoadTypeMediaStream) { + LOG(ERROR) << "Cobalt " << __func__ << " kLoadTypeMediaStream " << loaded_url_.spec(); + } ReportMetrics(load_type, loaded_url_, *frame_, media_log_.get());