Skip to content

Commit

Permalink
[android] Support KEY_MAX_INPUT_SIZE configurable from the webapp
Browse files Browse the repository at this point in the history
1. Add an h5vcc setting "Media.MaxInputSize" to allow the web app to explicitly set the MediaFormat.KEY_MAX_INPUT_SIZE.
2. This can also be set via MIME attribute "maxinputsize".
3. Use -1 for default value. It the setting value is smaller than default value, it will be ignored.

b/176923480
  • Loading branch information
borongc committed Dec 8, 2023
1 parent 7a6d3ba commit 103a31c
Show file tree
Hide file tree
Showing 29 changed files with 289 additions and 24 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
<!DOCTYPE html>
<!--
| Copyright 2023 The Cobalt Authors. All Rights Reserved.
|
| Licensed under the Apache License, Version 2.0 (the "License");
| you may not use this file except in compliance with the License.
| You may obtain a copy of the License at
|
| http://www.apache.org/licenses/LICENSE-2.0
|
| Unless required by applicable law or agreed to in writing, software
| distributed under the License is distributed on an "AS IS" BASIS,
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
| See the License for the specific language governing permissions and
| limitations under the License.
-->

<html>
<head>
<title>Configure Max Input Size</title>
<style>
body {
background-color: #0f0;
}
video {
height: 240px;
width: 426px;
}
</style>
</head>
<body>
<script type="text/javascript" src="configure-max-input-size.js"></script>
<div id="ui-layer">
<div class="item">
<video id="video1"></video>
</div>
<div class="item">
<video id="video2" muted="1" style="background-color: #4285F4">
</video>
</div>
</div>
<br>
<div id="status"></div>
</body>
</html>
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright 2023 The Cobalt Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var nextVideoElementIndex = 0;
var audioData;
var videoData;

function downloadMediaData(downloadedCallback) {
var xhr = new XMLHttpRequest;

xhr.onload = function() {
audioData = xhr.response;
console.log("Downloaded " + audioData.byteLength + " of audio data.");

xhr.onload = function() {
videoData = xhr.response;
console.log("Downloaded " + videoData.byteLength + " of video data.");
downloadedCallback();
}

xhr.open("GET", "vp9-720p.webm", true);
xhr.send();
}

xhr.open("GET", "dash-audio.mp4", true);
xhr.responseType = "arraybuffer";
xhr.send();
}

function playVideoOn(videoElement) {
var ms = new MediaSource;
ms.addEventListener('sourceopen', function() {
console.log("Creating SourceBuffer objects.");
var audioBuffer = ms.addSourceBuffer('audio/mp4; codecs="mp4a.40.2"');
var videoBuffer = ms.addSourceBuffer('video/webm; codecs="vp9"');
audioBuffer.addEventListener("updateend", function() {
audioBuffer.abort();
videoBuffer.addEventListener("updateend", function() {
setTimeout(function() {
videoBuffer.addEventListener("updateend", function() {
videoBuffer.abort();
ms.endOfStream();
videoElement.ontimeupdate = function() {
if (videoElement.currentTime > 10) {
console.log("Stop playback.");
videoElement.src = '';
videoElement.load();
videoElement.ontimeupdate = null;
}
}
console.log("Start playback.");
videoElement.play();
});
videoBuffer.appendBuffer(videoData.slice(1024));
}, 5000);
});
videoBuffer.appendBuffer(videoData.slice(0, 1024));
});
audioBuffer.appendBuffer(audioData);
});

console.log("Attaching MediaSource to video element.");
videoElement.src = URL.createObjectURL(ms);
}

function setupKeyHandler() {
const maxInputSizes = [3110500, 777000];
videoElements = document.getElementsByTagName('video');
for(let i = 0; i < videoElements.length; i++) {
if (videoElements[i].setMaxInputSize && i < maxInputSizes.length) {
videoElements[i].setMaxInputSize(maxInputSizes[i]);
}
if (i != 0 && videoElements[i].setMaxVideoCapabilities) {
videoElements[i].setMaxVideoCapabilities("width=1920; height=1080");
}
playVideoOn(videoElements[i]);
}
}

downloadMediaData(setupKeyHandler);
Binary file not shown.
Binary file not shown.
15 changes: 15 additions & 0 deletions cobalt/dom/html_media_element.cc
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ bool OriginIsSafe(loader::RequestMode request_mode, const GURL& resource_url,

HTMLMediaElement::HTMLMediaElement(Document* document, base::Token tag_name)
: HTMLElement(document, tag_name),
max_input_size_(-1),
load_state_(kWaitingForSource),
ALLOW_THIS_IN_INITIALIZER_LIST(event_queue_(this)),
playback_rate_(1.f),
Expand Down Expand Up @@ -1665,6 +1666,8 @@ std::string HTMLMediaElement::MaxVideoCapabilities() const {
return max_video_capabilities_;
}

int HTMLMediaElement::MaxInputSize() const { return max_input_size_; }

bool HTMLMediaElement::PreferDecodeToTexture() {
TRACE_EVENT0("cobalt::dom", "HTMLMediaElement::PreferDecodeToTexture()");

Expand Down Expand Up @@ -1760,5 +1763,17 @@ void HTMLMediaElement::SetMaxVideoCapabilities(
max_video_capabilities_ = max_video_capabilities;
}

void HTMLMediaElement::SetMaxInputSize(
int32 max_input_size, script::ExceptionState* exception_state) {
if (max_input_size == 0) {
web::DOMException::Raise(web::DOMException::kNotSupportedErr,
exception_state);
return;
}
LOG(ERROR) << "max_input_size is changed from " << max_input_size_ << " to "
<< max_input_size;
max_input_size_ = static_cast<int>(max_input_size);
}

} // namespace dom
} // namespace cobalt
7 changes: 7 additions & 0 deletions cobalt/dom/html_media_element.h
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,10 @@ class HTMLMediaElement : public HTMLElement,
return !max_video_capabilities_.empty();
}

// Set video max_input_size.
void SetMaxInputSize(int32 max_input_size,
script::ExceptionState* exception_state);

DEFINE_WRAPPABLE_TYPE(HTMLMediaElement);
void TraceMembers(script::Tracer* tracer) override;

Expand Down Expand Up @@ -240,6 +244,7 @@ class HTMLMediaElement : public HTMLElement,
void SourceOpened(ChunkDemuxer* chunk_demuxer) override;
std::string SourceURL() const override;
std::string MaxVideoCapabilities() const override;
int MaxInputSize() const override;
bool PreferDecodeToTexture() override;
void EncryptedMediaInitDataEncountered(
const char* init_data_type, const unsigned char* init_data,
Expand All @@ -252,6 +257,8 @@ class HTMLMediaElement : public HTMLElement,

std::string max_video_capabilities_;

int max_input_size_;

// Loading state.
enum LoadState { kWaitingForSource, kLoadingFromSrcAttr };
LoadState load_state_;
Expand Down
5 changes: 5 additions & 0 deletions cobalt/dom/html_video_element.idl
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,9 @@ interface HTMLVideoElement : HTMLMediaElement {
// is the same as the format for the string passed in to
// HTMLMediaElement.canPlayType().
[RaisesException] void setMaxVideoCapabilities(DOMString max_video_capabilities);

// Non standard interface (b/176923480). This allows web app to
// configure the maximum size of a buffer of input data on ATV.
// Use default buffer size if set to -1.
[RaisesException] void setMaxInputSize(long maxInputSize);
};
3 changes: 2 additions & 1 deletion cobalt/media/base/pipeline.h
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,8 @@ class MEDIA_EXPORT Pipeline : public base::RefCountedThreadSafe<Pipeline> {
const base::Closure& duration_change_cb,
const base::Closure& output_mode_change_cb,
const base::Closure& content_size_change_cb,
const std::string& max_video_capabilities) = 0;
const std::string& max_video_capabilities,
const int max_input_size) = 0;

#if SB_HAS(PLAYER_WITH_URL)
// Build a pipeline with an url-base player.
Expand Down
6 changes: 5 additions & 1 deletion cobalt/media/base/sbplayer_bridge.cc
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,8 @@ SbPlayerBridge::SbPlayerBridge(
SbPlayerSetBoundsHelper* set_bounds_helper, bool allow_resume_after_suspend,
SbPlayerOutputMode default_output_mode,
DecodeTargetProvider* const decode_target_provider,
const std::string& max_video_capabilities, std::string pipeline_identifier)
const std::string& max_video_capabilities, int max_input_size,
std::string pipeline_identifier)
: sbplayer_interface_(interface),
task_runner_(task_runner),
get_decode_target_graphics_context_provider_func_(
Expand All @@ -242,6 +243,7 @@ SbPlayerBridge::SbPlayerBridge(
video_config_(video_config),
decode_target_provider_(decode_target_provider),
max_video_capabilities_(max_video_capabilities),
max_input_size_(max_input_size),
cval_stats_(&interface->cval_stats_),
pipeline_identifier_(pipeline_identifier)
#if SB_HAS(PLAYER_WITH_URL)
Expand Down Expand Up @@ -765,8 +767,10 @@ void SbPlayerBridge::CreatePlayer() {
#endif // SB_API_VERSION >= 15
}
creation_param.output_mode = output_mode_;
creation_param.max_input_size = max_input_size_;
DCHECK_EQ(sbplayer_interface_->GetPreferredOutputMode(&creation_param),
output_mode_);

cval_stats_->StartTimer(MediaTiming::SbPlayerCreate, pipeline_identifier_);
player_ = sbplayer_interface_->Create(
window_, &creation_param, &SbPlayerBridge::DeallocateSampleCB,
Expand Down
4 changes: 3 additions & 1 deletion cobalt/media/base/sbplayer_bridge.h
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ class SbPlayerBridge {
bool allow_resume_after_suspend,
SbPlayerOutputMode default_output_mode,
DecodeTargetProvider* const decode_target_provider,
const std::string& max_video_capabilities,
const std::string& max_video_capabilities, int max_input_size,
std::string pipeline_identifier);

~SbPlayerBridge();
Expand Down Expand Up @@ -301,6 +301,8 @@ class SbPlayerBridge {
// A string of video maximum capabilities.
std::string max_video_capabilities_;

int max_input_size_;

// Keep track of errors during player creation.
bool is_creating_player_ = false;
std::string player_creation_error_message_;
Expand Down
7 changes: 5 additions & 2 deletions cobalt/media/base/sbplayer_pipeline.cc
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ void SbPlayerPipeline::Start(Demuxer* demuxer,
const base::Closure& duration_change_cb,
const base::Closure& output_mode_change_cb,
const base::Closure& content_size_change_cb,
const std::string& max_video_capabilities) {
const std::string& max_video_capabilities,
const int max_input_size) {
TRACE_EVENT0("cobalt::media", "SbPlayerPipeline::Start");

DCHECK(!ended_cb.is_null());
Expand All @@ -252,6 +253,7 @@ void SbPlayerPipeline::Start(Demuxer* demuxer,
parameters.output_mode_change_cb = output_mode_change_cb;
parameters.content_size_change_cb = content_size_change_cb;
parameters.max_video_capabilities = max_video_capabilities;
parameters.max_input_size = max_input_size;
#if SB_HAS(PLAYER_WITH_URL)
parameters.is_url_based = false;
#endif // SB_HAS(PLAYER_WITH_URL)
Expand Down Expand Up @@ -640,6 +642,7 @@ void SbPlayerPipeline::StartTask(StartTaskParameters parameters) {
output_mode_change_cb_ = parameters.output_mode_change_cb;
content_size_change_cb_ = parameters.content_size_change_cb;
max_video_capabilities_ = parameters.max_video_capabilities;
max_input_size_ = parameters.max_input_size;
#if SB_HAS(PLAYER_WITH_URL)
is_url_based_ = parameters.is_url_based;
if (is_url_based_) {
Expand Down Expand Up @@ -837,7 +840,7 @@ void SbPlayerPipeline::CreatePlayer(SbDrmSystem drm_system) {
audio_mime_type, video_config, video_mime_type, window_, drm_system,
this, set_bounds_helper_.get(), allow_resume_after_suspend_,
default_output_mode_, decode_target_provider_, max_video_capabilities_,
pipeline_identifier_));
max_input_size_, pipeline_identifier_));
if (player_bridge_->IsValid()) {
#if SB_API_VERSION >= 15
// TODO(b/267678497): When `player_bridge_->GetAudioConfigurations()`
Expand Down
6 changes: 5 additions & 1 deletion cobalt/media/base/sbplayer_pipeline.h
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ class MEDIA_EXPORT SbPlayerPipeline : public Pipeline,
const base::Closure& duration_change_cb,
const base::Closure& output_mode_change_cb,
const base::Closure& content_size_change_cb,
const std::string& max_video_capabilities) override;
const std::string& max_video_capabilities,
const int max_input_size) override;
#if SB_HAS(PLAYER_WITH_URL)
void Start(const SetDrmSystemReadyCB& set_drm_system_ready_cb,
const OnEncryptedMediaInitDataEncounteredCB&
Expand Down Expand Up @@ -133,6 +134,7 @@ class MEDIA_EXPORT SbPlayerPipeline : public Pipeline,
base::Closure output_mode_change_cb;
base::Closure content_size_change_cb;
std::string max_video_capabilities;
int max_input_size;
#if SB_HAS(PLAYER_WITH_URL)
std::string source_url;
bool is_url_based;
Expand Down Expand Up @@ -345,6 +347,8 @@ class MEDIA_EXPORT SbPlayerPipeline : public Pipeline,
// The maximum video playback capabilities required for the playback.
base::CVal<std::string> max_video_capabilities_;

int max_input_size_;

PlaybackStatistics playback_statistics_;

SbTimeMonotonic last_resume_time_ = -1;
Expand Down
1 change: 1 addition & 0 deletions cobalt/media/player/web_media_player.h
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ class WebMediaPlayerClient {
virtual void SourceOpened(::media::ChunkDemuxer* chunk_demuxer) = 0;
virtual std::string SourceURL() const = 0;
virtual std::string MaxVideoCapabilities() const = 0;
virtual int MaxInputSize() const = 0;

// Clients should implement this in order to indicate a preference for whether
// a video should be decoded to a texture or through a punch out system. If
Expand Down
2 changes: 1 addition & 1 deletion cobalt/media/player/web_media_player_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -842,7 +842,7 @@ void WebMediaPlayerImpl::StartPipeline(::media::Demuxer* demuxer) {
BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnDurationChanged),
BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnOutputModeChanged),
BIND_TO_RENDER_LOOP(&WebMediaPlayerImpl::OnContentSizeChanged),
GetClient()->MaxVideoCapabilities());
GetClient()->MaxVideoCapabilities(), GetClient()->MaxInputSize());
}

void WebMediaPlayerImpl::SetNetworkState(WebMediaPlayer::NetworkState state) {
Expand Down
1 change: 1 addition & 0 deletions cobalt/media/sandbox/web_media_player_helper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ class WebMediaPlayerHelper::WebMediaPlayerClientStub
}
std::string SourceURL() const override { return ""; }
std::string MaxVideoCapabilities() const override { return std::string(); };
int MaxInputSize() const override { return -1; };

void EncryptedMediaInitDataEncountered(const char*, const unsigned char*,
unsigned) override {}
Expand Down
Loading

0 comments on commit 103a31c

Please sign in to comment.