Merge pull request #1686 from twilio/prep_for_release_2.19.0

Prep for release 2.19.0

CHANGELOG.md

@@ -2,6 +2,47 @@ The Twilio Programmable Video SDKs use [Semantic Versioning](http://www.semver.o
 
 **Version 1.x reached End of Life on September 8th, 2021.** See the changelog entry [here](https://www.twilio.com/changelog/end-of-life-complete-for-unsupported-versions-of-the-programmable-video-sdk). Support for the 1.x version ended on December 4th, 2020.
 
+2.19.0 (January 31, 2022)
+=========================
+
+New Features
+------------
+
+- This release introduces a new feature **Adaptive Simulcast**. This opt-in feature can be enabled by setting `preferredVideoCodecs="auto"` in ConnectOptions. When joining a group room with this feature enabled, the SDK will use VP8 simulcast, and will enable/disable simulcast layers dynamically, thus improving bandwidth and CPU usage for the publishing client. It works best when used along with `Client Track Switch Off Control` and `Video Content Preferences`. These two flags allow the SFU to determine which simulcast layers are needed, thus allowing it to disable the layers not needed on publisher side. This feature cannot be used alongside `maxVideoBitrate`.
+
+If your application is currently using VP8 simulcast we recommend that you switch to this option.
+
+Example:
+
+```ts
+const { connect } = require('twilio-video');
+
+const room = await connect(token, {
+  preferredVideoCodecs: 'auto',
+  bandwidthProfile: {
+    video: {
+      contentPreferencesMode: 'auto',
+      clientTrackSwitchOffControl: 'auto'
+    }
+  }
+});
+```
+
+Known Limitations
+-----------------
+
+- Specifying `preferredVideoCodecs="auto"` will revert to unicast in the following cases:
+  - The publisher is using Firefox.
+  - The publisher has preferred the H264 codec.
+  - The Room is configured to support only the H264 codec.
+  - Peer-to-Peer Rooms
+- When the participant is being recorded, the SFU will not disable any simulcast layers of the participant's VideoTrack.
+
+Bug Fixes
+---------
+
+- Fixed a bug where `clientTrackSwitchOffControl` and `contentPreferencesMode` sometimes did not work as expected during network glitches. (VIDEO-7654)
+
 2.18.3 (January 4, 2022)
 ========================
 

README.md

@@ -74,7 +74,7 @@ Releases of twilio-video.js are hosted on a CDN, and you can include these
 directly in your web app using a <script> tag.
 
 ```html
-<script src="//sdk.twilio.com/js/video/releases/2.18.3/twilio-video.min.js"></script>
+<script src="//sdk.twilio.com/js/video/releases/2.19.0/twilio-video.min.js"></script>
 
 ```
 

lib/connect.js

@@ -221,6 +221,18 @@ function connect(token, options) {
   // Additionally, the options that are no longer needed will be removed.
   deprecateOptions(options, log, deprecatedConnectOptionsProps);
 
+  const adaptiveSimulcast = options.preferredVideoCodecs === 'auto';
+  if (adaptiveSimulcast) {
+    // NOTE(mpatwardhan): enable adaptiveSimulcast.
+    options.preferredVideoCodecs = [{ codec: 'VP8', simulcast: true, adaptiveSimulcast: true }];
+  }
+
+  if (options.maxVideoBitrate && adaptiveSimulcast) {
+    log.error('ConnectOptions "maxVideoBitrate" is not compatible with "preferredVideoCodecs=auto"');
+    return CancelablePromise.reject(E.ILLEGAL_INVOKE('connect',
+      'ConnectOptions "maxVideoBitrate" is not compatible with "preferredVideoCodecs=auto"'));
+  }
+
   options = Object.assign({
     automaticSubscription: true,
     createLocalTracks,
@@ -359,7 +371,7 @@ function connect(token, options) {
   const encodingParameters = new EncodingParametersImpl({
     maxAudioBitrate: options.maxAudioBitrate,
     maxVideoBitrate: options.maxVideoBitrate
-  });
+  }, adaptiveSimulcast);
 
   const preferredCodecs = {
     audio: options.preferredAudioCodecs.map(normalizeCodecSettings),
@@ -455,6 +467,7 @@ function connect(token, options) {
  *   effect for fixed bitrate codecs; Based on our tests, Chrome, Firefox and Safari
  *   all seem to support an average bitrate range of 20000 bps (20 kbps) to
  *   8000000 bps (8 mbps) for a 720p VideoTrack
+ *   This parameter must not be set when when preferredVideoCodecs is set to `auto`.
  * @property {?string} [name=null] - Set to connect to a {@link Room} by name
  * @property {boolean|NetworkQualityConfiguration} [networkQuality=false] - Whether to enable the Network
  *   Quality API or not. This only takes effect in Group Rooms. Pass a {@link NetworkQualityConfiguration}
@@ -470,8 +483,11 @@ function connect(token, options) {
  *   for the list of supported signaling regions.
  * @property {Array<AudioCodec|AudioCodecSettings>} [preferredAudioCodecs=[]] - Preferred audio codecs;
  *  An empty array preserves the current audio codec preference order.
- * @property {Array<VideoCodec|VideoCodecSettings>} [preferredVideoCodecs=[]] -
- *  Preferred video codecs; An empty array preserves the current video codec
+ * @property {Array<VideoCodec|VideoCodecSettings>|VideoEncodingMode} [preferredVideoCodecs=[]] -
+ *  Preferred video codecs; when set to 'VideoEncodingMode.Auto', SDK manages the video codec,
+ *  by preferring VP8 simulcast in group rooms. It also enables adaptive simulcast, which allows SDK
+ *  to turn off simulcast layers that are not needed for efficient bandwidth and CPU usage.
+ *  An empty array preserves the current video codec.
  *  preference order. If you want to set a preferred video codec on a Group Room,
  *  you will need to create the Room using the REST API and set the
  *  <code>VideoCodecs</code> property.
@@ -620,6 +636,16 @@ const AudioCodec = {
   PCMU: 'PCMU'
 };
 
+/**
+ * Names of the supported VideoEncodingMode.
+ * @enum {string}
+ */
+// eslint-disable-next-line
+const VideoEncodingMode = {
+  Auto: 'auto',
+};
+
+
 /**
  * Names of the supported video codecs.
  * @enum {string}

lib/encodingparameters.js

@@ -15,8 +15,9 @@ class EncodingParametersImpl extends EventEmitter {
   /**
    * Construct an {@link EncodingParametersImpl}.
    * @param {EncodingParamters} encodingParameters - Initial {@link EncodingParameters}
+   * @param {Boolean} adaptiveSimulcast - true if adaptive simulcast was enabled by connect options.
    */
-  constructor(encodingParameters) {
+  constructor(encodingParameters, adaptiveSimulcast) {
     super();
 
     encodingParameters = Object.assign({
@@ -32,6 +33,9 @@ class EncodingParametersImpl extends EventEmitter {
       maxVideoBitrate: {
         value: encodingParameters.maxVideoBitrate,
         writable: true
+      },
+      adaptiveSimulcast: {
+        value: adaptiveSimulcast
       }
     });
   }

lib/localparticipant.js

@@ -534,6 +534,11 @@ class LocalParticipant extends Participant {
     }
 
     if (encodingParameters) {
+      if (this._signaling.getParameters().adaptiveSimulcast && encodingParameters.maxVideoBitrate) {
+        // eslint-disable-next-line new-cap
+        throw E.INVALID_TYPE('encodingParameters', 'encodingParameters.maxVideoBitrate is not compatible with "preferredVideoCodecs=auto"');
+      }
+
       ['maxAudioBitrate', 'maxVideoBitrate'].forEach(prop => {
         if (typeof encodingParameters[prop] !== 'undefined'
           && typeof encodingParameters[prop] !== 'number'

lib/media/track/sender.js

@@ -5,6 +5,7 @@ const MediaTrackTransceiver = require('./transceiver');
 /**
  * A {@link MediaTrackSender} represents one or more local RTCRtpSenders.
  * @extends MediaTrackTransceiver
+ * @emits MediaTrackSender#replaced
  */
 class MediaTrackSender extends MediaTrackTransceiver {
   /**
@@ -20,6 +21,9 @@ class MediaTrackSender extends MediaTrackTransceiver {
       _senders: {
         value: new Set()
       },
+      _senderToPublisherHintCallbacks: {
+        value: new Map()
+      },
       isPublishing: {
         get() {
           return !!this._clones.size;
@@ -58,7 +62,7 @@ class MediaTrackSender extends MediaTrackTransceiver {
     return Promise.all(clones.map(clone => {
       return clone.setMediaStreamTrack(mediaStreamTrack.clone());
     }).concat(senders.map(sender => {
-      return sender.replaceTrack(mediaStreamTrack);
+      return this._replaceTrack(sender, mediaStreamTrack);
     }))).finally(() => {
       this._track = mediaStreamTrack;
     });
@@ -67,10 +71,14 @@ class MediaTrackSender extends MediaTrackTransceiver {
   /**
    * Add an RTCRtpSender.
    * @param {RTCRtpSender} sender
+   * @param {?()=>Promise<string>} publisherHintCallback
    * @returns {this}
    */
-  addSender(sender) {
+  addSender(sender, publisherHintCallback) {
     this._senders.add(sender);
+    if (publisherHintCallback) {
+      this._senderToPublisherHintCallbacks.set(sender, publisherHintCallback);
+    }
     return this;
   }
 
@@ -81,8 +89,34 @@ class MediaTrackSender extends MediaTrackTransceiver {
    */
   removeSender(sender) {
     this._senders.delete(sender);
+    this._senderToPublisherHintCallbacks.delete(sender);
     return this;
   }
+
+  /**
+   * Applies given encodings, or resets encodings if none specified.
+   * @param {Array<{enabled: boolean, layer_index: number}>|null} encodings
+   * @returns {Promise<string>}
+   */
+  setPublisherHint(encodings) {
+    // Note(mpatwardhan): since publisher hint applies only to group rooms we only look at 1st call callback.
+    const [publisherHintCallback] = Array.from(this._senderToPublisherHintCallbacks.values());
+    return publisherHintCallback ? publisherHintCallback(encodings) : Promise.resolve('COULD_NOT_APPLY_HINT');
+  }
+
+  _replaceTrack(sender, mediaStreamTrack) {
+    return sender.replaceTrack(mediaStreamTrack).then(replaceTrackResult => {
+      // clear any publisherHints and apply default encodings.
+      this.setPublisherHint(null).catch(() => {});
+      this.emit('replaced');
+      return replaceTrackResult;
+    });
+  }
 }
 
+/**
+ * The {@link MediaTrackSender} replaced the underlying mediaStreamTrack
+ * @event MediaTrackSender#replaced
+ */
+
 module.exports = MediaTrackSender;

lib/signaling/v2/cancelableroomsignalingpromise.js

@@ -19,6 +19,7 @@ function createCancelableRoomSignalingPromise(token, wsServer, localParticipant,
     Transport: DefaultTransport
   }, options);
 
+  const adaptiveSimulcast = preferredCodecs.video[0] &&  preferredCodecs.video[0].adaptiveSimulcast === true;
   const { PeerConnectionManager, RoomV2, Transport, iceServers, log } = options;
   const peerConnectionManager = new PeerConnectionManager(encodingParameters, preferredCodecs, options);
   const trackSenders = flatMap(localParticipant.tracks, trackV2 => [trackV2.trackTransceiver]);
@@ -79,6 +80,7 @@ function createCancelableRoomSignalingPromise(token, wsServer, localParticipant,
       (options.clientTrackSwitchOffControl !== 'disabled' || options.contentPreferencesMode !== 'disabled');
 
     const transportOptions = Object.assign({
+      adaptiveSimulcast,
       automaticSubscription,
       dominantSpeaker,
       environment,

lib/signaling/v2/localparticipant.js

@@ -2,7 +2,9 @@
 
 const LocalParticipantSignaling = require('../localparticipant');
 const LocalTrackPublicationV2 = require('./localtrackpublication');
-const { isDeepEqual } = require('../../util');
+const { DEFAULT_LOG_LEVEL } = require('../../util/constants');
+const Log = require('../../util/log');
+const { buildLogLevels, isDeepEqual } = require('../../util');
 
 /**
  * @extends ParticipantSignaling
@@ -20,10 +22,14 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
    */
   constructor(encodingParameters, networkQualityConfiguration, options) {
     options = Object.assign({
+      logLevel: DEFAULT_LOG_LEVEL,
       LocalTrackPublicationV2
     }, options);
 
     super();
+
+    const logLevels = buildLogLevels(options.logLevel);
+
     Object.defineProperties(this, {
       _bandwidthProfile: {
         value: null,
@@ -42,6 +48,11 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
       _LocalTrackPublicationV2: {
         value: options.LocalTrackPublicationV2
       },
+      _log: {
+        value: options.log
+          ? options.log.createLog('default', this)
+          : new Log('default', this, logLevels, options.loggerName)
+      },
       _publishedRevision: {
         writable: true,
         value: 0
@@ -85,6 +96,10 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
     });
   }
 
+  toString() {
+    return `[LocalParticipantSignaling: ${this.sid}]`;
+  }
+
   /**
    * Set the signalingRegion.
    * @param {string} signalingRegion.
@@ -110,6 +125,14 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
     }
   }
 
+  /**
+   * returns current {@link EncodingParametersImpl}.
+   * @returns {EncodingParametersImpl}
+   */
+  getParameters() {
+    return this._encodingParameters;
+  }
+
   /**
    * Set the {@link EncodingParameters}.
    * @param {?EncodingParameters} encodingParameters
@@ -249,8 +272,25 @@ class LocalParticipantV2 extends LocalParticipantSignaling {
   setNetworkQualityConfiguration(networkQualityConfiguration) {
     this.networkQualityConfiguration.update(networkQualityConfiguration);
   }
+
+  /**
+   * updates encodings for simulcast layers.
+   * @param {Track.SID} trackSid
+   * @param {Array<{enabled: boolean, layer_index: number}>} encodings
+   * @returns {Promise<string>} string indicating result of the operation. can be one of
+   *  "OK", "INVALID_HINT", "COULD_NOT_APPLY_HINT", "UNKNOWN_TRACK"
+   */
+  setPublisherHint(trackSid, encodings) {
+    const trackSignaling = Array.from(this.tracks.values()).find(trackPub => trackPub.sid === trackSid);
+    if (!trackSignaling) {
+      this._log.warn(`track:${trackSid} not found`);
+      return Promise.resolve('UNKNOWN_TRACK');
+    }
+    return trackSignaling.trackTransceiver.setPublisherHint(encodings);
+  }
 }
 
+
 /**
  * @interface Published
  * @property {number} revision