Skip to content

Commit

Permalink
[Proposal]: Add experimental.enableResolutionChecks loadVideo option
Browse files Browse the repository at this point in the history
This is a Proposal to optionally move some "resolution checks" that
were historically on the application-side on the RxPlayer side.

Some devices aren't able to play video content with a higher resolution
than e.g. what their screen size can display: some will fail on error,
others will just infinitely freeze etc.

Many of those devices have vendor-specific API that indicated whether a
device was e.g. compatible to 4k content, 1080p content etc.

Until now we told the application developers to do the Representation
filtering themselves with the help of the `representationFilter` option
from the `loadVideo` API. This is because of the vast numbers of devices
out there, each with their own API, and because we didn't want to take
the risk of having false negatives if it turned out some of those
API were not always right.

However, many of those devices are very popular (Lg and Samsung TVs,
game consoles). Thus I'm wondering if it would be better that we
provide some of those resolution checks ourselves, to lower the
efforts an application have to make to rely on the RxPlayer on those
common devices.

For now I added an experimental `loadVideo` option:
`experimental.enableResolutionChecks`, that has to be set to `true`
to enable the behavior. The long term idea would be that for devices
where it seems 100% accurate, we would enable the check by default.

It will directly filter out resolutions that are too high, unless all
resolutions are too high on the current video track in which case it
will disable the check (as a security).
In other cases, it follows the exact same rules than the
`isCodecSupported` and `decipherable` properties of `Representation`.
  • Loading branch information
peaBerberian committed Sep 5, 2024
1 parent 1746fe4 commit fc7b62f
Show file tree
Hide file tree
Showing 22 changed files with 458 additions and 112 deletions.
9 changes: 9 additions & 0 deletions src/compat/browser_detection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ let isWebOs2021 = false;
/** `true` specifically for WebOS 2022 version. */
let isWebOs2022 = false;

/** `true` specifically for Hisense TVs. */
let isHisense = false;

/** `true` for Panasonic devices. */
let isPanasonic = false;

Expand Down Expand Up @@ -158,6 +161,11 @@ let isXbox = false;
) {
isWebOs2021 = true;
}
} else if (
navigator.userAgent.indexOf("Hisense") !== -1 &&
navigator.userAgent.indexOf("VIDAA") !== -1
) {
isHisense = true;
} else if (
navigator.userAgent.indexOf("NETTV") !== -1 &&
navigator.userAgent.indexOf("Philips") !== -1
Expand All @@ -175,6 +183,7 @@ export {
isIE11,
isIEOrEdge,
isFirefox,
isHisense,
isPanasonic,
isPhilipsNetTv,
isPlayStation4,
Expand Down
163 changes: 163 additions & 0 deletions src/compat/max_resolution_detection.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
import log from "../log";
import globalScope from "../utils/global_scope";
import isNullOrUndefined from "../utils/is_null_or_undefined";
import { isHisense, isTizen, isWebOs } from "./browser_detection";

interface IWebOsDeviceCallback {
modelName: string;
modelNameAscii: string;
version: string;
versionMajor: number;
versionMinor: number;
versionDot: number;
sdkVersion: string;
screenWidth: number;
screenHeight: number;
uhd?: boolean;
}

type IVendorGlobalScope = typeof globalScope & {
/** On Hisense TVs. */
// eslint-disable-next-line @typescript-eslint/naming-convention
Hisense_Get4KSupportState?: (() => boolean) | null | undefined;
/** On Tizen devices. */
webapis?:
| {
productinfo?:
| {
is8KPanelSupported?: (() => boolean) | null | undefined;
isUdPanelSupported?: (() => boolean) | null | undefined;
}
| null
| undefined;
}
| null
| undefined;
/** On LG TVs. */
// eslint-disable-next-line @typescript-eslint/naming-convention
PalmSystem?:
| {
deviceInfo?: string | null | undefined;
}
| null
| undefined;
/** Also on LG TVs. */
webOS?:
| {
deviceInfo:
| ((cb: (arg: IWebOsDeviceCallback) => void) => void)
| null
| undefined;
}
| null
| undefined;
};
const global: IVendorGlobalScope = globalScope;

export default function getMaxSupportedResolution(): {
width?: number | undefined;
height: number | undefined;
} {
try {
if (isHisense) {
if (
navigator.userAgent.indexOf(";FHD") >= 0 ||
navigator.userAgent.indexOf("/FHD") >= 0
) {
return {
height: 1080,
width: undefined,
};
}
if (
navigator.userAgent.indexOf(";HD") >= 0 ||
navigator.userAgent.indexOf("/HD") >= 0
) {
return {
height: 720,
width: undefined,
};
}
// Found in VIDAA Web developer documentation
if (
"Hisense_Get4KSupportState" in global &&
typeof global.Hisense_Get4KSupportState === "function"
) {
if (global.Hisense_Get4KSupportState()) {
return {
height: undefined,
width: undefined,
};
}
}
}

if (isTizen) {
if (
!isNullOrUndefined(global.webapis) &&
!isNullOrUndefined(global.webapis.productinfo)
) {
if (typeof global.webapis.productinfo.is8KPanelSupported === "function") {
return {
height: undefined,
width: undefined,
};
}
if (typeof global.webapis.productinfo.isUdPanelSupported === "function") {
return {
height: 3840,
width: 2160,
};
}
}
}

if (isWebOs) {
let deviceInfo: IWebOsDeviceCallback | null | undefined;
if (
!isNullOrUndefined(global.PalmSystem) &&
typeof global.PalmSystem.deviceInfo === "string"
) {
deviceInfo = JSON.parse(global.PalmSystem.deviceInfo) as IWebOsDeviceCallback;
}
if (
!isNullOrUndefined(global.webOS) &&
typeof global.webOS.deviceInfo === "function"
) {
global.webOS.deviceInfo((info: IWebOsDeviceCallback) => {
deviceInfo = info;
});
}
if (!isNullOrUndefined(deviceInfo)) {
if (deviceInfo.uhd === true) {
return {
width: undefined,
height: undefined,
};
}
if (
"screenWidth" in deviceInfo &&
typeof deviceInfo.screenWidth === "number" &&
deviceInfo.screenWidth <= 1920 &&
"screenHeight" in deviceInfo &&
typeof deviceInfo.screenHeight === "number" &&
deviceInfo.screenHeight <= 1080
) {
return {
width: 1920,
height: 1080,
};
}
}
}
} catch (err) {
log.error(
"Compat: Error when trying to call vendor API",
err instanceof Error ? err : new Error("Unknown Error"),
);
}
return {
height: undefined,
width: undefined,
};
}
3 changes: 2 additions & 1 deletion src/core/cmcd/cmcd_data_builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,8 @@ export default class CmcdDataBuilder {
props.tb = content.adaptation.representations.reduce(
(acc: number | undefined, representation: IRepresentation) => {
if (
representation.isSupported !== true ||
representation.isCodecSupported !== true ||
representation.isResolutionSupported === false ||
representation.decipherable === false
) {
return acc;
Expand Down
3 changes: 2 additions & 1 deletion src/core/stream/adaptation/adaptation_stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,8 @@ export default function AdaptationStream(
(r) =>
arrayIncludes(initialRepIds, r.id) &&
r.decipherable !== false &&
r.isSupported !== false,
r.isCodecSupported !== false &&
r.isResolutionSupported !== false,
);

/** Emit the list of Representation for the adaptive logic. */
Expand Down
6 changes: 5 additions & 1 deletion src/core/stream/period/period_stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,11 @@ function createOrReuseSegmentSink(
*/
function getFirstDeclaredMimeType(adaptation: IAdaptation): string {
const representations = adaptation.representations.filter((r) => {
return r.isSupported === true && r.decipherable !== false;
return (
r.isCodecSupported === true &&
r.decipherable !== false &&
r.isResolutionSupported !== false
);
});
if (representations.length === 0) {
const noRepErr = new MediaError(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,9 @@ export default function getAdaptationSwitchStrategy(
function hasCompatibleCodec(adaptation: IAdaptation, segmentSinkCodec: string): boolean {
return adaptation.representations.some(
(rep) =>
rep.isSupported === true &&
rep.isCodecSupported === true &&
rep.decipherable !== false &&
rep.isResolutionSupported !== false &&
areCodecsCompatible(rep.getMimeTypeString(), segmentSinkCodec),
);
}
7 changes: 4 additions & 3 deletions src/main_thread/api/debug/modules/general_info.ts
Original file line number Diff line number Diff line change
Expand Up @@ -177,15 +177,16 @@ export default function constructDebugGeneralInfo(
adaptations?.video?.representations.map((r) => {
return (
String(r.bitrate ?? "N/A") +
(r.isSupported !== false ? "" : " U!") +
(r.decipherable !== false ? "" : " E!")
(r.isCodecSupported !== false ? "" : " U!") +
(r.decipherable !== false ? "" : " E!") +
(r.isResolutionSupported !== false ? "" : " R!")
);
}) ?? [];
const audioBitratesStr =
adaptations?.audio?.representations.map((r) => {
return (
String(r.bitrate ?? "N/A") +
(r.isSupported !== false ? "" : " U!") +
(r.isCodecSupported !== false ? "" : " U!") +
(r.decipherable !== false ? "" : " E!")
);
}) ?? [];
Expand Down
30 changes: 30 additions & 0 deletions src/main_thread/api/option_utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,29 +65,55 @@ export interface IParsedConstructorOptions {
/**
* Base type which the types for the parsed options of the RxPlayer's
* `loadVideo` method exend.
* @see ILoadVideoOptions
*/
interface IParsedLoadVideoOptionsBase {
/** @see ILoadVideoOptions.url */
url: string | undefined;
/** @see ILoadVideoOptions.transport */
transport: string;
/** @see ILoadVideoOptions.autoPlay */
autoPlay: boolean;
/** @see ILoadVideoOptions.initialManifest */
initialManifest: ILoadedManifestFormat | undefined;
/** @see ILoadVideoOptions.keySystems */
keySystems: IKeySystemOption[];
/** @see ILoadVideoOptions.lowLatencyMode */
lowLatencyMode: boolean;
/** @see ILoadVideoOptions.minimumManifestUpdateInterval */
minimumManifestUpdateInterval: number;
/** @see ILoadVideoOptions.requestConfig */
requestConfig: IRequestConfig;
/** @see ILoadVideoOptions.startAt */
startAt: IParsedStartAtOption | undefined;
/** @see ILoadVideoOptions.enableFastSwitching */
enableFastSwitching: boolean;
/** @see ILoadVideoOptions.defaultAudioTrackSwitchingMode */
defaultAudioTrackSwitchingMode: IAudioTrackSwitchingMode | undefined;
/** @see ILoadVideoOptions.onCodecSwitch */
onCodecSwitch: "continue" | "reload";
/** @see ILoadVideoOptions.checkMediaSegmentIntegrity */
checkMediaSegmentIntegrity?: boolean | undefined;
/** @see ILoadVideoOptions.checkManifestIntegrity */
checkManifestIntegrity?: boolean | undefined;
/** @see ILoadVideoOptions.manifestLoader */
manifestLoader?: IManifestLoader | undefined;
/** @see ILoadVideoOptions.referenceDateTime */
referenceDateTime?: number | undefined;
/** @see ILoadVideoOptions.representationFilter */
representationFilter?: IRepresentationFilter | string | undefined;
/** @see ILoadVideoOptions.segmentLoader */
segmentLoader?: ISegmentLoader | undefined;
/** @see ILoadVideoOptions.serverSyncInfos */
serverSyncInfos?: IServerSyncInfos | undefined;
/** @see ILoadVideoOptions.mode */
mode: IRxPlayerMode;
/** @see ILoadVideoOptions.cmcd */
cmcd: ICmcdOptions | undefined;
/** @see ILoadVideoOptions.experimentalOptions */
experimentalOptions: {
enableResolutionChecks: boolean;
};
__priv_manifestUpdateUrl?: string | undefined;
__priv_patchLastSegmentInSidx?: boolean | undefined;
}
Expand Down Expand Up @@ -461,6 +487,10 @@ function parseLoadVideoOptions(options: ILoadVideoOptions): IParsedLoadVideoOpti
mode,
url,
cmcd: options.cmcd,
experimentalOptions: {
enableResolutionChecks:
options.experimentalOptions?.enableResolutionChecks === true,
},
};
}

Expand Down
1 change: 1 addition & 0 deletions src/main_thread/api/public_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -952,6 +952,7 @@ class Player extends EventEmitter<IPublicAPIEvent> {
manifestLoader: undefined,
segmentLoader: undefined,
representationFilter: options.representationFilter,
enableResolutionChecks: options.experimentalOptions.enableResolutionChecks,
__priv_manifestUpdateUrl,
__priv_patchLastSegmentInSidx,
};
Expand Down
18 changes: 9 additions & 9 deletions src/main_thread/init/utils/update_manifest_codec_support.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { ContentDecryptorState } from "../../decrypt";
/**
* Returns a list of all codecs that the support is not known yet on the given
* Manifest.
* If a representation with (`isSupported`) is undefined, we consider the
* If a representation with (`isCodecSupported`) is undefined, we consider the
* codec support as unknown.
*
* This function iterates through all periods, adaptations, and representations,
Expand All @@ -36,7 +36,7 @@ export function getCodecsWithUnknownSupport(
continue;
}
for (const representation of adaptation.representations) {
if (representation.isSupported === undefined) {
if (representation.isCodecSupported === undefined) {
codecsWithUnknownSupport.push({
mimeType: representation.mimeType ?? "",
codec: representation.codecs?.[0] ?? "",
Expand Down Expand Up @@ -131,8 +131,8 @@ export function updateManifestCodecSupport(
let hasSupportedCodec: boolean = false;
let hasCodecWithUndefinedSupport: boolean = false;
adaptation.representations.forEach((representation) => {
if (representation.isSupported !== undefined) {
if (representation.isSupported) {
if (representation.isCodecSupported !== undefined) {
if (representation.isCodecSupported) {
hasSupportedCodec = true;
}
// We already knew the support for that one, continue to next one
Expand All @@ -148,16 +148,16 @@ export function updateManifestCodecSupport(
for (const codec of codecs) {
const codecSupportInfo = efficientlyGetCodecSupport(mimeType, codec);
if (!isEncrypted) {
representation.isSupported = codecSupportInfo.isSupportedClear;
representation.isCodecSupported = codecSupportInfo.isSupportedClear;
} else if (
representation.isSupported !== codecSupportInfo.isSupportedEncrypted
representation.isCodecSupported !== codecSupportInfo.isSupportedEncrypted
) {
representation.isSupported = codecSupportInfo.isSupportedEncrypted;
representation.isCodecSupported = codecSupportInfo.isSupportedEncrypted;
}

if (representation.isSupported === undefined) {
if (representation.isCodecSupported === undefined) {
hasCodecWithUndefinedSupport = true;
} else if (representation.isSupported) {
} else if (representation.isCodecSupported) {
hasSupportedCodec = true;
representation.codecs = [codec];

Expand Down
Loading

0 comments on commit fc7b62f

Please sign in to comment.