Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[android] Refine video decoder for tunnel mode #1835

Closed
wants to merge 1 commit into from
Closed

[android] Refine video decoder for tunnel mode #1835

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 56 additions & 14 deletions starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import android.os.Bundle;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import dev.cobalt.util.Log;
import dev.cobalt.util.UsedByNative;
import java.nio.ByteBuffer;
Expand Down Expand Up @@ -87,6 +88,7 @@ class MediaCodecBridge {
private int mFps = 30;

private MediaCodec.OnFrameRenderedListener mTunnelModeFrameRendererListener;
private MediaCodec.OnFirstTunnelFrameReadyListener mFirstTunnelFrameReadyListener;

// Functions that require this will be called frequently in a tight loop.
// Only create one of these and reuse it to avoid excessive allocations,
Expand Down Expand Up @@ -523,22 +525,15 @@ public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
};
mMediaCodec.setCallback(mCallback);

// Listen to the frame ready events for tunnel mode.
// TODO: support OnFrameRenderedListener for non tunnel mode
jasonzhangxx marked this conversation as resolved.
Show resolved Hide resolved
if (tunnelModeAudioSessionId != -1) {
mTunnelModeFrameRendererListener =
new MediaCodec.OnFrameRenderedListener() {
@Override
public void onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime) {
synchronized (this) {
if (mNativeMediaCodecBridge == 0) {
return;
}
nativeOnMediaCodecFrameRendered(
mNativeMediaCodecBridge, presentationTimeUs, nanoTime);
}
}
};
mMediaCodec.setOnFrameRenderedListener(mTunnelModeFrameRendererListener, null);
// TODO (b/306236129): disable the listeners as frames to be discarded would invoke the
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are not implementing BUFFER_FLAG_DECODE_ONLY now, do we still need to remove the two listeners here?
Also nit:

  1. Remove the space after TODO
  2. Refine the commit message mentioning that OnFirstTunnelFrameReadyListener is commented out.

// callbacks.
// addOnFrameRenderedListener();
// if (Build.VERSION.SDK_INT >= 31) {
// addOnFirstTunnelFrameReadyListener();
// }
}
}

Expand Down Expand Up @@ -858,6 +853,10 @@ private void updateOperatingRate() {
@UsedByNative
private int flush() {
try {
// TODO: find a better place to clear listeners when tear down the codec.
mMediaCodec.setOnFrameRenderedListener(null, null);
mMediaCodec.setOnFirstTunnelFrameReadyListener(null, null);
Comment on lines +857 to +858
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's also comment them out, since the setting of them are commented out. Also, if we are going to set the callback to null, we should set variables like mTunnelModeFrameRendererListener to null too.


mFlushed = true;
mMediaCodec.flush();
} catch (Exception e) {
Expand Down Expand Up @@ -1304,6 +1303,47 @@ private int getAudioFormat(int channelCount) {
}
}

private void addOnFrameRenderedListener() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: "set" instead of "add".

if (mMediaCodec == null) {
return;
}

mTunnelModeFrameRendererListener =
new MediaCodec.OnFrameRenderedListener() {
@Override
public void onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime) {
synchronized (this) {
if (mNativeMediaCodecBridge == 0) {
return;
}
nativeOnMediaCodecFrameRendered(
mNativeMediaCodecBridge, presentationTimeUs, nanoTime);
}
}
};
mMediaCodec.setOnFrameRenderedListener(mTunnelModeFrameRendererListener, null);
}

@RequiresApi(31)
private void addOnFirstTunnelFrameReadyListener() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: "set" instead of "add", also append "V31" to the function name.

if (mMediaCodec == null) {
return;
}
mFirstTunnelFrameReadyListener =
new MediaCodec.OnFirstTunnelFrameReadyListener() {
@Override
public void onFirstTunnelFrameReady(MediaCodec codec) {
synchronized (this) {
if (mNativeMediaCodecBridge == 0) {
return;
}
nativeOnFirstTunnelFrameReady(mNativeMediaCodecBridge);
}
}
};
mMediaCodec.setOnFirstTunnelFrameReadyListener(null, mFirstTunnelFrameReadyListener);
}

private native void nativeOnMediaCodecError(
long nativeMediaCodecBridge,
boolean isRecoverable,
Expand All @@ -1325,4 +1365,6 @@ private native void nativeOnMediaCodecOutputBufferAvailable(

private native void nativeOnMediaCodecFrameRendered(
long nativeMediaCodecBridge, long presentationTimeUs, long renderAtSystemTimeNs);

private native void nativeOnFirstTunnelFrameReady(long nativeMediaCodecBridge);
}
2 changes: 0 additions & 2 deletions starboard/android/shared/BUILD.gn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,8 +388,6 @@ static_library("starboard_platform") {
"trace_util.h",
"video_decoder.cc",
"video_decoder.h",
"video_frame_tracker.cc",
"video_frame_tracker.h",
"video_render_algorithm.cc",
"video_render_algorithm.h",
"video_window.cc",
Expand Down
3 changes: 2 additions & 1 deletion starboard/android/shared/audio_decoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,11 @@ class AudioDecoder
bool InitializeCodec();
void ProcessOutputBuffer(MediaCodecBridge* media_codec_bridge,
const DequeueOutputResult& output) override;
void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) override {}
void RefreshOutputFormat(MediaCodecBridge* media_codec_bridge) override;
bool Tick(MediaCodecBridge* media_codec_bridge) override { return false; }
void OnFlushing() override {}
void OnMediaCodecFrameRendered(SbTime frame_timestamp) override {}
void OnFirstTunnelFrameReady() override {}
Comment on lines +74 to +75
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add NOTREACHED() here.


void ReportError(SbPlayerError error, const std::string& error_message);

Expand Down
15 changes: 15 additions & 0 deletions starboard/android/shared/media_codec_bridge.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,17 @@ Java_dev_cobalt_media_MediaCodecBridge_nativeOnMediaCodecFrameRendered(
media_codec_bridge->OnMediaCodecFrameRendered(presentation_time_us);
}

extern "C" SB_EXPORT_PLATFORM void
Java_dev_cobalt_media_MediaCodecBridge_nativeOnFirstTunnelFrameReady(
JNIEnv* env,
jobject unused_this,
jlong native_media_codec_bridge) {
MediaCodecBridge* media_codec_bridge =
reinterpret_cast<MediaCodecBridge*>(native_media_codec_bridge);
SB_DCHECK(media_codec_bridge);
media_codec_bridge->OnFirstTunnelFrameReady();
}

extern "C" SB_EXPORT_PLATFORM void
Java_dev_cobalt_media_MediaCodecBridge_nativeOnMediaCodecError(
JniEnvExt* env,
Expand Down Expand Up @@ -520,6 +531,10 @@ void MediaCodecBridge::OnMediaCodecFrameRendered(SbTime frame_timestamp) {
handler_->OnMediaCodecFrameRendered(frame_timestamp);
}

void MediaCodecBridge::OnFirstTunnelFrameReady() {
handler_->OnFirstTunnelFrameReady();
}

MediaCodecBridge::MediaCodecBridge(Handler* handler) : handler_(handler) {
SB_DCHECK(handler_);
}
Expand Down
6 changes: 5 additions & 1 deletion starboard/android/shared/media_codec_bridge.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,11 @@ class MediaCodecBridge {
int64_t presentation_time_us,
int size) = 0;
virtual void OnMediaCodecOutputFormatChanged() = 0;
// This is only called on video decoder when tunnel mode is enabled.

// Functions below are only called on video decoder when tunnel mode is
// enabled.
virtual void OnMediaCodecFrameRendered(SbTime frame_timestamp) = 0;
virtual void OnFirstTunnelFrameReady() = 0;

protected:
~Handler() {}
Expand Down Expand Up @@ -212,6 +215,7 @@ class MediaCodecBridge {
int size);
void OnMediaCodecOutputFormatChanged();
void OnMediaCodecFrameRendered(SbTime frame_timestamp);
void OnFirstTunnelFrameReady();

private:
// |MediaCodecBridge|s must only be created through its factory methods.
Expand Down
14 changes: 8 additions & 6 deletions starboard/android/shared/media_decoder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,18 +115,14 @@ MediaDecoder::MediaDecoder(Host* host,
SbDrmSystem drm_system,
const SbMediaColorMetadata* color_metadata,
bool require_software_codec,
const FrameRenderedCB& frame_rendered_cb,
int tunnel_mode_audio_session_id,
bool force_big_endian_hdr_metadata,
std::string* error_message)
: media_type_(kSbMediaTypeVideo),
host_(host),
drm_system_(static_cast<DrmSystem*>(drm_system)),
frame_rendered_cb_(frame_rendered_cb),
tunnel_mode_enabled_(tunnel_mode_audio_session_id != -1),
condition_variable_(mutex_) {
SB_DCHECK(frame_rendered_cb_);

jobject j_media_crypto = drm_system_ ? drm_system_->GetMediaCrypto() : NULL;
const bool require_secured_decoder =
drm_system_ && drm_system_->require_secured_decoder();
Expand Down Expand Up @@ -489,7 +485,6 @@ bool MediaDecoder::ProcessOneInputBuffer(
status = media_codec_bridge_->QueueInputBuffer(dequeue_input_result.index,
kNoOffset, size, kNoPts,
BUFFER_FLAG_END_OF_STREAM);
host_->OnEndOfStreamWritten(media_codec_bridge_.get());
}

if (status != MEDIA_CODEC_OK) {
Expand Down Expand Up @@ -641,7 +636,14 @@ void MediaDecoder::OnMediaCodecOutputFormatChanged() {

void MediaDecoder::OnMediaCodecFrameRendered(SbTime frame_timestamp) {
SB_DCHECK(tunnel_mode_enabled_);
frame_rendered_cb_(frame_timestamp);
SB_DCHECK(host_);
host_->OnMediaCodecFrameRendered(frame_timestamp);
}

void MediaDecoder::OnFirstTunnelFrameReady() {
SB_DCHECK(tunnel_mode_enabled_);
SB_DCHECK(host_);
host_->OnFirstTunnelFrameReady();
}

} // namespace shared
Expand Down
7 changes: 4 additions & 3 deletions starboard/android/shared/media_decoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ class MediaDecoder
public:
virtual void ProcessOutputBuffer(MediaCodecBridge* media_codec_bridge,
const DequeueOutputResult& output) = 0;
virtual void OnEndOfStreamWritten(MediaCodecBridge* media_codec_bridge) = 0;
virtual void RefreshOutputFormat(MediaCodecBridge* media_codec_bridge) = 0;
// This function gets called frequently on the decoding thread to give the
// Host a chance to process when the MediaDecoder is decoding.
Expand All @@ -73,6 +72,9 @@ class MediaDecoder
// before the MediaCodecBridge is flushed.
virtual void OnFlushing() = 0;

virtual void OnMediaCodecFrameRendered(SbTime frame_timestamp) = 0;
virtual void OnFirstTunnelFrameReady() = 0;

protected:
~Host() {}
};
Expand All @@ -94,7 +96,6 @@ class MediaDecoder
SbDrmSystem drm_system,
const SbMediaColorMetadata* color_metadata,
bool require_software_codec,
const FrameRenderedCB& frame_rendered_cb,
int tunnel_mode_audio_session_id,
bool force_big_endian_hdr_metadata,
std::string* error_message);
Expand Down Expand Up @@ -169,13 +170,13 @@ class MediaDecoder
int size) override;
void OnMediaCodecOutputFormatChanged() override;
void OnMediaCodecFrameRendered(SbTime frame_timestamp) override;
void OnFirstTunnelFrameReady() override;

::starboard::shared::starboard::ThreadChecker thread_checker_;

const SbMediaType media_type_;
Host* host_;
DrmSystem* const drm_system_;
const FrameRenderedCB frame_rendered_cb_;
const bool tunnel_mode_enabled_;

ErrorCB error_cb_;
Expand Down
Loading
Loading