[android] Refine dropped frames reporting

starboard/android/shared/audio_decoder.h

@@ -68,6 +68,9 @@ class AudioDecoder
   bool InitializeCodec();
   void ProcessOutputBuffer(MediaCodecBridge* media_codec_bridge,
                            const DequeueOutputResult& output) override;
+  void OnInputBufferEnqueued(
+      MediaCodecBridge* media_codec_bridge,
+      const scoped_refptr<InputBuffer>& input_buffer) override {}
   void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) override {}
   void RefreshOutputFormat(MediaCodecBridge* media_codec_bridge) override;
   bool Tick(MediaCodecBridge* media_codec_bridge) override { return false; }

starboard/android/shared/media_decoder.cc

@@ -485,6 +485,7 @@ bool MediaDecoder::ProcessOneInputBuffer(
       status = media_codec_bridge_->QueueInputBuffer(
           dequeue_input_result.index, kNoOffset, size, pts_us, kNoBufferFlags);
     }
+    host_->OnInputBufferEnqueued(media_codec_bridge_.get(), input_buffer);
   } else {
     status = media_codec_bridge_->QueueInputBuffer(dequeue_input_result.index,
                                                    kNoOffset, size, kNoPts,

starboard/android/shared/media_decoder.h

@@ -61,6 +61,9 @@ class MediaDecoder
    public:
     virtual void ProcessOutputBuffer(MediaCodecBridge* media_codec_bridge,
                                      const DequeueOutputResult& output) = 0;
+    virtual void OnInputBufferEnqueued(
+        MediaCodecBridge* media_codec_bridge,
+        const scoped_refptr<InputBuffer>& input_buffer) = 0;
     virtual void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) = 0;
     virtual void RefreshOutputFormat(MediaCodecBridge* media_codec_bridge) = 0;
     // This function gets called frequently on the decoding thread to give the

starboard/android/shared/video_decoder.cc

@@ -300,13 +300,16 @@ class VideoRenderAlgorithmTunneled : public VideoRenderAlgorithmBase {
 
   void Render(MediaTimeProvider* media_time_provider,
               std::list<scoped_refptr<VideoFrame>>* frames,
-              VideoRendererSink::DrawFrameCB draw_frame_cb) override {}
+              VideoRendererSink::DrawFrameCB draw_frame_cb) override {
+    SB_DCHECK(media_time_provider);
+    frame_tracker_->SetMediaTimeProvider(media_time_provider);
+  }
+
   void Seek(SbTime seek_to_time) override {
     frame_tracker_->Seek(seek_to_time);
   }
-  int GetDroppedFrames() override {
-    return frame_tracker_->UpdateAndGetDroppedFrames();
-  }
+
+  int GetDroppedFrames() override { return frame_tracker_->GetDroppedFrames(); }
 
  private:
   VideoFrameTracker* frame_tracker_;
@@ -374,7 +377,7 @@ VideoDecoder::VideoDecoder(const VideoStreamInfo& video_stream_info,
   SB_DCHECK(error_message);
 
   if (tunnel_mode_audio_session_id != -1) {
-    video_frame_tracker_.reset(new VideoFrameTracker(kMaxPendingWorkSize * 2));
+    video_frame_tracker_.reset(new VideoFrameTracker());
   }
   if (force_secure_pipeline_under_tunnel_mode) {
     SB_DCHECK(tunnel_mode_audio_session_id != -1);
@@ -511,6 +514,10 @@ void VideoDecoder::WriteInputBuffers(const InputBuffers& input_buffers) {
     if (tunnel_mode_audio_session_id_ != -1) {
       Schedule(std::bind(&VideoDecoder::OnTunnelModePrerollTimeout, this),
                kInitialPrerollTimeout);
+
+      // Call Render() once to let VideoRenderAlgorithmTunneled get
+      // MediaTimeProvider.
+      sink_->Render();
     }
   }
 
@@ -774,6 +781,17 @@ void VideoDecoder::TeardownCodec() {
   }
 }
 
+void VideoDecoder::OnInputBufferEnqueued(
+    MediaCodecBridge* media_codec_bridge,
+    const scoped_refptr<InputBuffer>& input_buffer) {
+  if (tunnel_mode_audio_session_id_ == -1) {
+    return;
+  }
+
+  SB_DCHECK(input_buffer);
+  video_frame_tracker_->OnInputBufferEnqueued(input_buffer);
+}
+
 void VideoDecoder::OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) {
   if (tunnel_mode_audio_session_id_ == -1) {
     return;
@@ -816,7 +834,6 @@ void VideoDecoder::WriteInputBuffersInternal(
   if (tunnel_mode_audio_session_id_ != -1) {
     SbTime max_timestamp = input_buffers[0]->timestamp();
     for (const auto& input_buffer : input_buffers) {
-      video_frame_tracker_->OnInputBuffer(input_buffer->timestamp());
       max_timestamp = std::max(max_timestamp, input_buffer->timestamp());
     }
 
@@ -1150,7 +1167,6 @@ void VideoDecoder::OnTunnelModeFrameRendered(SbTime frame_timestamp) {
   SB_DCHECK(tunnel_mode_audio_session_id_ != -1);
 
   tunnel_mode_frame_rendered_.store(true);
-  video_frame_tracker_->OnFrameRendered(frame_timestamp);
 }
 
 void VideoDecoder::OnTunnelModePrerollTimeout() {

starboard/android/shared/video_decoder.h

@@ -109,7 +109,10 @@ class VideoDecoder
   void WriteInputBuffersInternal(const InputBuffers& input_buffers);
   void ProcessOutputBuffer(MediaCodecBridge* media_codec_bridge,
                            const DequeueOutputResult& output) override;
-  void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge);
+  void OnInputBufferEnqueued(
+      MediaCodecBridge* media_codec_bridge,
+      const scoped_refptr<InputBuffer>& input_buffer) override;
+  void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) override;
   void RefreshOutputFormat(MediaCodecBridge* media_codec_bridge) override;
   bool Tick(MediaCodecBridge* media_codec_bridge) override;
   void OnFlushing() override;

starboard/android/shared/video_frame_tracker.cc

@@ -14,131 +14,38 @@
 
 #include "starboard/android/shared/video_frame_tracker.h"
 
-#include <cmath>
-#include <cstdint>
-#include <cstdlib>
-#include <vector>
-
 #include "starboard/common/log.h"
-#include "starboard/common/mutex.h"
 #include "starboard/time.h"
 
 namespace starboard {
 namespace android {
 namespace shared {
-namespace {
-
-const SbTime kMaxAllowedSkew = 5000;
-
-}  // namespace
 
-SbTime VideoFrameTracker::seek_to_time() const {
-  return seek_to_time_;
-}
-
-void VideoFrameTracker::OnInputBuffer(SbTime timestamp) {
-  SB_DCHECK(thread_checker_.CalledOnValidThread());
+void VideoFrameTracker::OnInputBufferEnqueued(
+    const scoped_refptr<InputBuffer>& input_buffer) {
+  SB_DCHECK(input_buffer);
+  SB_DCHECK(media_time_provider_);
 
-  if (frames_to_be_rendered_.empty()) {
-    frames_to_be_rendered_.push_back(timestamp);
+  if (input_buffer->timestamp() < seek_to_time_) {
     return;
   }
 
-  if (frames_to_be_rendered_.size() > max_pending_frames_size_) {
-    // OnFrameRendered() is only available after API level 23.  Cap the size
-    // of |frames_to_be_rendered_| in case OnFrameRendered() is not available.
-    frames_to_be_rendered_.pop_front();
-  }
+  bool is_playing = true;
+  bool is_eos_played = true;
+  bool is_underflow = true;
+  double playback_rate = -1.0;
+  SbTime media_time = media_time_provider_->GetCurrentMediaTime(
+      &is_playing, &is_eos_played, &is_underflow, &playback_rate);
 
-  // Sort by |timestamp|, because |timestamp| won't be monotonic if there are
-  // B frames.
-  for (auto it = frames_to_be_rendered_.end();
-       it != frames_to_be_rendered_.begin();) {
-    it--;
-    if (*it < timestamp) {
-      frames_to_be_rendered_.emplace(++it, timestamp);
-      return;
-    } else if (*it == timestamp) {
-      SB_LOG(WARNING) << "feed video AU with same time stamp " << timestamp;
-      return;
-    }
+  if (input_buffer->timestamp() < media_time) {
+    dropped_frames_++;
   }
-
-  frames_to_be_rendered_.emplace_front(timestamp);
-}
-
-void VideoFrameTracker::OnFrameRendered(int64_t frame_timestamp) {
-  ScopedLock lock(rendered_frames_mutex_);
-  rendered_frames_on_decoder_thread_.push_back(frame_timestamp);
 }
 
 void VideoFrameTracker::Seek(SbTime seek_to_time) {
-  SB_DCHECK(thread_checker_.CalledOnValidThread());
-
-  // Ensure that all dropped frames before seeking are captured.
-  UpdateDroppedFrames();
-
-  frames_to_be_rendered_.clear();
   seek_to_time_ = seek_to_time;
 }
 
-int VideoFrameTracker::UpdateAndGetDroppedFrames() {
-  SB_DCHECK(thread_checker_.CalledOnValidThread());
-  UpdateDroppedFrames();
-  return dropped_frames_;
-}
-
-void VideoFrameTracker::UpdateDroppedFrames() {
-  SB_DCHECK(thread_checker_.CalledOnValidThread());
-
-  {
-    ScopedLock lock(rendered_frames_mutex_);
-    rendered_frames_on_tracker_thread_.swap(rendered_frames_on_decoder_thread_);
-  }
-
-  while (frames_to_be_rendered_.front() < seek_to_time_) {
-    // It is possible that the initial frame rendered time is before the
-    // seek to time, when the platform decides to render a frame earlier
-    // than the seek to time during preroll. This shouldn't be an issue
-    // after we align seek time to the next video key frame.
-    frames_to_be_rendered_.pop_front();
-  }
-
-  // Loop over all timestamps from OnFrameRendered and compare against ones from
-  // OnInputBuffer.
-  for (auto rendered_timestamp : rendered_frames_on_tracker_thread_) {
-    auto to_render_timestamp = frames_to_be_rendered_.begin();
-    // Loop over all frames to render until we've caught up to the timestamp of
-    // the last rendered frame.
-    while (to_render_timestamp != frames_to_be_rendered_.end() &&
-           !(*to_render_timestamp - rendered_timestamp > kMaxAllowedSkew)) {
-      if (std::abs(*to_render_timestamp - rendered_timestamp) <=
-          kMaxAllowedSkew) {
-        // This frame was rendered, remove it from frames_to_be_rendered_.
-        to_render_timestamp = frames_to_be_rendered_.erase(to_render_timestamp);
-      } else if (rendered_timestamp - *to_render_timestamp > kMaxAllowedSkew) {
-        // The rendered frame is too far ahead. The to_render_timestamp frame
-        // was dropped.
-        SB_LOG(WARNING) << "Video frame dropped:" << *to_render_timestamp
-                        << ", current frame timestamp:" << rendered_timestamp
-                        << ", frames in the backlog:"
-                        << frames_to_be_rendered_.size();
-        ++dropped_frames_;
-        to_render_timestamp = frames_to_be_rendered_.erase(to_render_timestamp);
-      } else {
-        // The rendered frame is too early to match the next frame to render.
-        // This could happen if a frame is reported to be rendered twice or if
-        // it is rendered more than kMaxAllowedSkew early. In the latter
-        // scenario the frame will be reported dropped in the next iteration of
-        // the outer loop.
-        ++to_render_timestamp;
-      }
-    }
-  }
-
-  rendered_frames_on_tracker_thread_.clear();
-}
-
 }  // namespace shared
 }  // namespace android
 }  // namespace starboard

starboard/android/shared/video_frame_tracker.h

@@ -15,11 +15,11 @@
 #ifndef STARBOARD_ANDROID_SHARED_VIDEO_FRAME_TRACKER_H_
 #define STARBOARD_ANDROID_SHARED_VIDEO_FRAME_TRACKER_H_
 
-#include <list>
-#include <vector>
+#include <atomic>
 
-#include "starboard/common/mutex.h"
-#include "starboard/shared/starboard/thread_checker.h"
+#include "starboard/common/ref_counted.h"
+#include "starboard/shared/starboard/player/filter/media_time_provider.h"
+#include "starboard/shared/starboard/player/input_buffer_internal.h"
 #include "starboard/time.h"
 
 namespace starboard {
@@ -28,33 +28,26 @@ namespace shared {
 
 class VideoFrameTracker {
  public:
-  explicit VideoFrameTracker(int max_pending_frames_size)
-      : max_pending_frames_size_(max_pending_frames_size) {}
+  typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer;
+  typedef ::starboard::shared::starboard::player::filter::MediaTimeProvider
+      MediaTimeProvider;
 
-  SbTime seek_to_time() const;
+  SbTime seek_to_time() const { return seek_to_time_; }
 
-  void OnInputBuffer(SbTime timestamp);
-
-  void OnFrameRendered(int64_t frame_timestamp);
+  void OnInputBufferEnqueued(const scoped_refptr<InputBuffer>& input_buffer);
 
   void Seek(SbTime seek_to_time);
 
-  int UpdateAndGetDroppedFrames();
-
- private:
-  void UpdateDroppedFrames();
-
-  ::starboard::shared::starboard::ThreadChecker thread_checker_;
-
-  std::list<SbTime> frames_to_be_rendered_;
+  int GetDroppedFrames() const { return dropped_frames_; }
 
-  const int max_pending_frames_size_;
-  int dropped_frames_ = 0;
-  SbTime seek_to_time_ = 0;
+  void SetMediaTimeProvider(MediaTimeProvider* media_time_provider) {
+    media_time_provider_ = media_time_provider;
+  }
 
-  Mutex rendered_frames_mutex_;
-  std::vector<SbTime> rendered_frames_on_tracker_thread_;
-  std::vector<SbTime> rendered_frames_on_decoder_thread_;
+ private:
+  MediaTimeProvider* media_time_provider_ = nullptr;
+  std::atomic_int dropped_frames_ = {0};
+  std::atomic_int64_t seek_to_time_ = {0};
 };
 
 }  // namespace shared