Skip to content

Commit

Permalink
[android] Refine video decoder for tunnel mode
Browse files Browse the repository at this point in the history 
1. Send placeholder frames to video renderer in tunnel mode.
2. Calculate dropped frames based on the placeholder frames.
3. Determing preroll status based on the placeholder frames.
4. Add OnFirstTunnelFrameReadyListener.
5. Remove video frame tracker as it's no longer needed.

b/175890319
b/175888740
b/302198176
b/253469969
  • Loading branch information
@jasonzhangxx
jasonzhangxx committed Oct 30, 2023
1 parent 896a0ec commit 5c92f07
Show file tree
Hide file tree
Showing 12 changed files with 143 additions and 415 deletions.
70 changes: 56 additions & 14 deletions starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import android.os.Bundle;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import dev.cobalt.util.Log;
import dev.cobalt.util.UsedByNative;
import java.nio.ByteBuffer;
Expand Down Expand Up @@ -87,6 +88,7 @@ class MediaCodecBridge {
private int mFps = 30;

private MediaCodec.OnFrameRenderedListener mTunnelModeFrameRendererListener;
private MediaCodec.OnFirstTunnelFrameReadyListener mFirstTunnelFrameReadyListener;

// Functions that require this will be called frequently in a tight loop.
// Only create one of these and reuse it to avoid excessive allocations,
Expand Down Expand Up @@ -523,22 +525,15 @@ public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
};
mMediaCodec.setCallback(mCallback);

// Listen to the frame ready events for tunnel mode.
// TODO: support OnFrameRenderedListener for non tunnel mode
if (tunnelModeAudioSessionId != -1) {
mTunnelModeFrameRendererListener =
new MediaCodec.OnFrameRenderedListener() {
@Override
public void onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime) {
synchronized (this) {
if (mNativeMediaCodecBridge == 0) {
return;
}
nativeOnMediaCodecFrameRendered(
mNativeMediaCodecBridge, presentationTimeUs, nanoTime);
}
}
};
mMediaCodec.setOnFrameRenderedListener(mTunnelModeFrameRendererListener, null);
// TODO (b/306236129): disable the listeners as frames to be discarded would invoke the
// callbacks.
// addOnFrameRenderedListener();
// if (Build.VERSION.SDK_INT >= 31) {
// addOnFirstTunnelFrameReadyListener();
// }
}
}

Expand Down Expand Up @@ -858,6 +853,10 @@ private void updateOperatingRate() {
@UsedByNative
private int flush() {
try {
// TODO: find a better place to clear listeners when tear down the codec.
mMediaCodec.setOnFrameRenderedListener(null, null);
mMediaCodec.setOnFirstTunnelFrameReadyListener(null, null);

mFlushed = true;
mMediaCodec.flush();
} catch (Exception e) {
Expand Down Expand Up @@ -1304,6 +1303,47 @@ private int getAudioFormat(int channelCount) {
}
}

private void addOnFrameRenderedListener() {
if (mMediaCodec == null) {
return;
}

mTunnelModeFrameRendererListener =
new MediaCodec.OnFrameRenderedListener() {
@Override
public void onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime) {
synchronized (this) {
if (mNativeMediaCodecBridge == 0) {
return;
}
nativeOnMediaCodecFrameRendered(
mNativeMediaCodecBridge, presentationTimeUs, nanoTime);
}
}
};
mMediaCodec.setOnFrameRenderedListener(mTunnelModeFrameRendererListener, null);
}

@RequiresApi(31)
private void addOnFirstTunnelFrameReadyListener() {
if (mMediaCodec == null) {
return;
}
mFirstTunnelFrameReadyListener =
new MediaCodec.OnFirstTunnelFrameReadyListener() {
@Override
public void onFirstTunnelFrameReady(MediaCodec codec) {
synchronized (this) {
if (mNativeMediaCodecBridge == 0) {
return;
}
nativeOnFirstTunnelFrameReady(mNativeMediaCodecBridge);
}
}
};
mMediaCodec.setOnFirstTunnelFrameReadyListener(null, mFirstTunnelFrameReadyListener);
}

private native void nativeOnMediaCodecError(
long nativeMediaCodecBridge,
boolean isRecoverable,
Expand All @@ -1325,4 +1365,6 @@ private native void nativeOnMediaCodecOutputBufferAvailable(

private native void nativeOnMediaCodecFrameRendered(
long nativeMediaCodecBridge, long presentationTimeUs, long renderAtSystemTimeNs);

private native void nativeOnFirstTunnelFrameReady(long nativeMediaCodecBridge);
}
2 changes: 0 additions & 2 deletions starboard/android/shared/BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,8 +388,6 @@ static_library("starboard_platform") {
"trace_util.h",
"video_decoder.cc",
"video_decoder.h",
"video_frame_tracker.cc",
"video_frame_tracker.h",
"video_render_algorithm.cc",
"video_render_algorithm.h",
"video_window.cc",
Expand Down
3 changes: 2 additions & 1 deletion starboard/android/shared/audio_decoder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,11 @@ class AudioDecoder
bool InitializeCodec();
void ProcessOutputBuffer(MediaCodecBridge* media_codec_bridge,
const DequeueOutputResult& output) override;
void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) override {}
void RefreshOutputFormat(MediaCodecBridge* media_codec_bridge) override;
bool Tick(MediaCodecBridge* media_codec_bridge) override { return false; }
void OnFlushing() override {}
void OnMediaCodecFrameRendered(SbTime frame_timestamp) override {}
void OnFirstTunnelFrameReady() override {}

void ReportError(SbPlayerError error, const std::string& error_message);

Expand Down
15 changes: 15 additions & 0 deletions starboard/android/shared/media_codec_bridge.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,17 @@ Java_dev_cobalt_media_MediaCodecBridge_nativeOnMediaCodecFrameRendered(
media_codec_bridge->OnMediaCodecFrameRendered(presentation_time_us);
}

extern "C" SB_EXPORT_PLATFORM void
Java_dev_cobalt_media_MediaCodecBridge_nativeOnFirstTunnelFrameReady(
JNIEnv* env,
jobject unused_this,
jlong native_media_codec_bridge) {
MediaCodecBridge* media_codec_bridge =
reinterpret_cast<MediaCodecBridge*>(native_media_codec_bridge);
SB_DCHECK(media_codec_bridge);
media_codec_bridge->OnFirstTunnelFrameReady();
}

extern "C" SB_EXPORT_PLATFORM void
Java_dev_cobalt_media_MediaCodecBridge_nativeOnMediaCodecError(
JniEnvExt* env,
Expand Down Expand Up @@ -520,6 +531,10 @@ void MediaCodecBridge::OnMediaCodecFrameRendered(SbTime frame_timestamp) {
handler_->OnMediaCodecFrameRendered(frame_timestamp);
}

void MediaCodecBridge::OnFirstTunnelFrameReady() {
handler_->OnFirstTunnelFrameReady();
}

MediaCodecBridge::MediaCodecBridge(Handler* handler) : handler_(handler) {
SB_DCHECK(handler_);
}
Expand Down
6 changes: 5 additions & 1 deletion starboard/android/shared/media_codec_bridge.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,11 @@ class MediaCodecBridge {
int64_t presentation_time_us,
int size) = 0;
virtual void OnMediaCodecOutputFormatChanged() = 0;
// This is only called on video decoder when tunnel mode is enabled.

// Functions below are only called on video decoder when tunnel mode is
// enabled.
virtual void OnMediaCodecFrameRendered(SbTime frame_timestamp) = 0;
virtual void OnFirstTunnelFrameReady() = 0;

protected:
~Handler() {}
Expand Down Expand Up @@ -212,6 +215,7 @@ class MediaCodecBridge {
int size);
void OnMediaCodecOutputFormatChanged();
void OnMediaCodecFrameRendered(SbTime frame_timestamp);
void OnFirstTunnelFrameReady();

private:
// |MediaCodecBridge|s must only be created through its factory methods.
Expand Down
19 changes: 9 additions & 10 deletions starboard/android/shared/media_decoder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,18 +115,14 @@ MediaDecoder::MediaDecoder(Host* host,
SbDrmSystem drm_system,
const SbMediaColorMetadata* color_metadata,
bool require_software_codec,
const FrameRenderedCB& frame_rendered_cb,
int tunnel_mode_audio_session_id,
bool force_big_endian_hdr_metadata,
std::string* error_message)
: media_type_(kSbMediaTypeVideo),
host_(host),
drm_system_(static_cast<DrmSystem*>(drm_system)),
frame_rendered_cb_(frame_rendered_cb),
tunnel_mode_enabled_(tunnel_mode_audio_session_id != -1),
condition_variable_(mutex_) {
SB_DCHECK(frame_rendered_cb_);

jobject j_media_crypto = drm_system_ ? drm_system_->GetMediaCrypto() : NULL;
const bool require_secured_decoder =
drm_system_ && drm_system_->require_secured_decoder();
Expand Down Expand Up @@ -340,10 +336,7 @@ void MediaDecoder::DecoderThreadFunc() {
}

bool ticked = false;
if (!tunnel_mode_enabled_) {
// Output is only processed when tunnel mode is disabled.
ticked = host_->Tick(media_codec_bridge_.get());
}
ticked = host_->Tick(media_codec_bridge_.get());

can_process_input =
pending_queue_input_buffer_task_ ||
Expand Down Expand Up @@ -489,7 +482,6 @@ bool MediaDecoder::ProcessOneInputBuffer(
status = media_codec_bridge_->QueueInputBuffer(dequeue_input_result.index,
kNoOffset, size, kNoPts,
BUFFER_FLAG_END_OF_STREAM);
host_->OnEndOfStreamWritten(media_codec_bridge_.get());
}

if (status != MEDIA_CODEC_OK) {
Expand Down Expand Up @@ -641,7 +633,14 @@ void MediaDecoder::OnMediaCodecOutputFormatChanged() {

void MediaDecoder::OnMediaCodecFrameRendered(SbTime frame_timestamp) {
SB_DCHECK(tunnel_mode_enabled_);
frame_rendered_cb_(frame_timestamp);
SB_DCHECK(host_);
host_->OnMediaCodecFrameRendered(frame_timestamp);
}

void MediaDecoder::OnFirstTunnelFrameReady() {
SB_DCHECK(tunnel_mode_enabled_);
SB_DCHECK(host_);
host_->OnFirstTunnelFrameReady();
}

} // namespace shared
Expand Down
7 changes: 4 additions & 3 deletions starboard/android/shared/media_decoder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ class MediaDecoder
public:
virtual void ProcessOutputBuffer(MediaCodecBridge* media_codec_bridge,
const DequeueOutputResult& output) = 0;
virtual void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) = 0;
virtual void RefreshOutputFormat(MediaCodecBridge* media_codec_bridge) = 0;
// This function gets called frequently on the decoding thread to give the
// Host a chance to process when the MediaDecoder is decoding.
Expand All @@ -73,6 +72,9 @@ class MediaDecoder
// before the MediaCodecBridge is flushed.
virtual void OnFlushing() = 0;

virtual void OnMediaCodecFrameRendered(SbTime frame_timestamp) = 0;
virtual void OnFirstTunnelFrameReady() = 0;

protected:
~Host() {}
};
Expand All @@ -94,7 +96,6 @@ class MediaDecoder
SbDrmSystem drm_system,
const SbMediaColorMetadata* color_metadata,
bool require_software_codec,
const FrameRenderedCB& frame_rendered_cb,
int tunnel_mode_audio_session_id,
bool force_big_endian_hdr_metadata,
std::string* error_message);
Expand Down Expand Up @@ -169,13 +170,13 @@ class MediaDecoder
int size) override;
void OnMediaCodecOutputFormatChanged() override;
void OnMediaCodecFrameRendered(SbTime frame_timestamp) override;
void OnFirstTunnelFrameReady() override;

::starboard::shared::starboard::ThreadChecker thread_checker_;

const SbMediaType media_type_;
Host* host_;
DrmSystem* const drm_system_;
const FrameRenderedCB frame_rendered_cb_;
const bool tunnel_mode_enabled_;

ErrorCB error_cb_;
Expand Down
Loading

0 comments on commit 5c92f07

Please sign in to comment.