Merge pull request #1407 from twilio/prep_2.13.0

Prep for 2.13.0 release
twilio · Mar 3, 2021 · fdaa7fe · fdaa7fe
2 parents e5cbcc2 + beb953f
commit fdaa7fe
Show file tree

Hide file tree

Showing 24 changed files with 1,727 additions and 26 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,73 @@ The Twilio Programmable Video SDKs use [Semantic Versioning](http://www.semver.o
 
 **Support for the 1.x version ended on December 4th, 2020**. Check [this guide](https://www.twilio.com/docs/video/migrating-1x-2x) to plan your migration to the latest 2.x version.
 
+2.13.0 (March 3, 2021)
+======================
+
+New Features
+------------
+
+**Video Processor API Pilot (Chrome only)**
+- You can now register a `VideoProcessor` with a VideoTrack in order to process its video frames. In a LocalVideoTrack, video frames are processed before being sent to the encoder. In a RemoteVideoTrack, video frames are processed before being sent to the attached `<video>` element(s). The `VideoProcessor` should implement the interface shown below. (VIDEO-3560, VIDEO-3561)
+
+  ```ts
+  abstract class VideoProcessor {
+    abstract processFrame(inputFrame: OffscreenCanvas)
+      : Promise<OffscreenCanvas | null>
+      | OffscreenCanvas | null;
+  }
+  ```
+
+  A VideoTrack provides new methods [addProcessor](https://media.twiliocdn.com/sdk/js/video/releases/2.13.0/docs/VideoTrack.html#addProcessor) and [removeProcessor](https://media.twiliocdn.com/sdk/js/video/releases/2.13.0/docs/VideoTrack.html#removeProcessor) which can be used to add and remove a VideoProcessor. It also provides a new property `processor` which points to the current VideoProcessor being used by the VideoTrack. For example, you can toggle a blur filter on a LocalVideoTrack as shown below.
+
+  ```ts
+  import { createLocalVideoTrack } from 'twilio-video';
+
+  class BlurVideoProcessor {
+    private readonly _outputFrameCtx: CanvasRenderingContext2D;
+    private readonly _outputFrame: OffscreenCanvas;
+
+    constructor(width: number, height: number, blurRadius: number) {
+      this._outputFrame = new OffscreenCanvas(width, height);
+      this._outputFrameCtx = this._outputFrame.getContext('2d');
+      this._outputFrameCtx.filter = `blur(${blurRadius}px)`;
+    }
+
+    processFrame(inputFrame: OffscreenCanvas) {
+      this._outputFrameCtx.drawImage(inputFrame, 0, 0);
+      return this._outputFrame;
+    }
+  }
+
+  // Local video track
+  createLocalVideoTrack({
+    width: 1280,
+    height: 720
+  }).then(track => {
+    const processor = new BlurVideoProcessor(1280, 720, 5);
+    document.getElementById('preview').appendChild(track.attach());
+    document.getElementById('toggle-blur').onclick = () => track.processor
+      ? track.removeProcessor(processor)
+      : track.addProcessor(processor);
+  });
+
+  ```
+
+  You can also toggle a blur filter on a RemoteVideoTrack as shown below.
+
+  ```js
+  room.on('trackSubscribed', track => {
+    if (track.kind === 'video') {
+      const { width, height } = track.dimensions;
+      const processor = new BlurVideoProcessor(width, height, 3);
+      document.getElementById('preview-remote').appendChild(track.attach());
+      document.getElementById('toggle-blur-remote').onclick = () => track.processor
+        ? track.removeProcessor(processor)
+        : track.addProcessor(processor);
+    }
+  });
+  ```
+
 2.12.0 (Feb 10, 2020)
 =====================
 

diff --git a/README.md b/README.md
@@ -77,7 +77,7 @@ Releases of twilio-video.js are hosted on a CDN, and you can include these
 directly in your web app using a &lt;script&gt; tag.
 
 ```html
-<script src="//media.twiliocdn.com/sdk/js/video/releases/2.12.0/twilio-video.min.js"></script>
+<script src="//media.twiliocdn.com/sdk/js/video/releases/2.13.0/twilio-video.min.js"></script>
 
 ```
 

diff --git a/lib/localparticipant.js b/lib/localparticipant.js
@@ -313,7 +313,13 @@ class LocalParticipant extends Participant {
           self._addTrackPublication(localTrackPublication);
         }
 
-        if (self._signaling.state === 'connected') {
+        const { state } = self._signaling;
+        // NOTE(csantos): For tracks created before joining a room or already joined but about to publish it
+        if (localTrack.processedTrack && (state === 'connected' || state === 'connecting')) {
+          localTrack._captureFrames();
+          localTrack._setSenderMediaStreamTrack(true);
+        }
+        if (state === 'connected') {
           setTimeout(() => {
             self.emit('trackPublished', localTrackPublication);
           });

diff --git a/lib/media/track/audiotrack.js b/lib/media/track/audiotrack.js
@@ -12,6 +12,8 @@ const MediaTrack = require('./mediatrack');
  *   enabled; if the {@link AudioTrack} is not enabled, it is "muted"
  * @property {Track.Kind} kind - "audio"
  * @property {MediaStreamTrack} mediaStreamTrack - An audio MediaStreamTrack
+ * @property {?MediaStreamTrack} processedTrack - The source of processed audio samples.
+ * It is always null as audio processing is not currently supported.
  * @emits AudioTrack#disabled
  * @emits AudioTrack#enabled
  * @emits AudioTrack#started

diff --git a/lib/media/track/localmediatrack.js b/lib/media/track/localmediatrack.js
@@ -189,9 +189,13 @@ function mixinLocalMediaTrack(AudioOrVideoTrack) {
       // stopped, this should fire a "stopped" event.
       this._stop();
 
-      return this._trackSender.setMediaStreamTrack(mediaStreamTrack).catch(error => {
+      // NOTE(csantos): If there's an unprocessedTrack, this means RTCRtpSender has
+      // the processedTrack already set, we don't want to replace that.
+      return (this._unprocessedTrack ? Promise.resolve().then(() => {
+        this._unprocessedTrack = mediaStreamTrack;
+      }) : this._trackSender.setMediaStreamTrack(mediaStreamTrack).catch(error => {
         this._log.warn('setMediaStreamTrack failed:', { error, mediaStreamTrack });
-      }).then(() => {
+      })).then(() => {
         this._initialize();
         this._getAllAttachedElements().forEach(el => this._attach(el));
       });

diff --git a/lib/media/track/localvideotrack.js b/lib/media/track/localvideotrack.js
@@ -62,37 +62,153 @@ class LocalVideoTrack extends LocalMediaVideoTrack {
     return `[LocalVideoTrack #${this._instanceId}: ${this.id}]`;
   }
 
+  /**
+   * @private
+   */
+  _canCaptureFrames() {
+    return super._canCaptureFrames.call(this, this._trackSender.isPublishing);
+  }
+
   /**
    * @private
    */
   _end() {
     return super._end.apply(this, arguments);
   }
 
+  /**
+   * @private
+   */
+  _setSenderMediaStreamTrack(useProcessed) {
+    const unprocessedTrack = this.mediaStreamTrack;
+    const mediaStreamTrack = useProcessed ? this.processedTrack : unprocessedTrack;
+
+    return this._trackSender.setMediaStreamTrack(mediaStreamTrack)
+      .catch(error => this._log.warn(
+        'setMediaStreamTrack failed on LocalVideoTrack RTCRtpSender', { error, mediaStreamTrack }))
+      .then(() => {
+        this._unprocessedTrack = useProcessed ? unprocessedTrack : null;
+      });
+  }
+
+  /**
+   * Add a {@link VideoProcessor} to allow for custom processing of video frames belonging to a VideoTrack.
+   * Only Chrome supports this as of now. Calling this API from a non-supported browser will result in a log warning.
+   * @param {VideoProcessor} processor - The {@link VideoProcessor} to use.
+   * @returns {this}
+   * @example
+   * class GrayScaleProcessor {
+   *   constructor(percentage) {
+   *     this.outputFrame = new OffscreenCanvas(0, 0);
+   *     this.percentage = percentage;
+   *   }
+   *   processFrame(inputFrame) {
+   *     this.outputFrame.width = inputFrame.width;
+   *     this.outputFrame.height = inputFrame.height;
+   *
+   *     const context = this.outputFrame.getContext('2d');
+   *     context.filter = `grayscale(${this.percentage}%)`;
+   *     context.drawImage(inputFrame, 0, 0, inputFrame.width, inputFrame.height);
+   *     return this.outputFrame;
+   *   }
+   * }
+   *
+   * const localVideoTrack = Array.from(room.localParticipant.videoTracks.values())[0].track;
+   * localVideoTrack.addProcessor(new GrayScaleProcessor(100));
+   */
+  addProcessor() {
+    this._log.debug('Adding VideoProcessor to the LocalVideoTrack');
+    const result = super.addProcessor.apply(this, arguments);
+
+    if (!this.processedTrack) {
+      return this._log.warn('Unable to add a VideoProcessor to the LocalVideoTrack');
+    }
+
+    this._log.debug('Updating LocalVideoTrack\'s MediaStreamTrack with the processed MediaStreamTrack', this.processedTrack);
+    this._setSenderMediaStreamTrack(true);
+
+    return result;
+  }
+
+  /**
+   * Remove the previously added {@link VideoProcessor} using `addProcessor` API.
+   * @param {VideoProcessor} processor - The {@link VideoProcessor} to remove.
+   * @returns {this}
+   * @example
+   * class GrayScaleProcessor {
+   *   constructor(percentage) {
+   *     this.outputFrame = new OffscreenCanvas(0, 0);
+   *     this.percentage = percentage;
+   *   }
+   *   processFrame(inputFrame) {
+   *     this.outputFrame.width = inputFrame.width;
+   *     this.outputFrame.height = inputFrame.height;
+   *
+   *     const context = this.outputFrame.getContext('2d');
+   *     context.filter = `grayscale(${this.percentage}%)`;
+   *     context.drawImage(inputFrame, 0, 0, inputFrame.width, inputFrame.height);
+   *     return this.outputFrame;
+   *   }
+   * }
+   *
+   * const localVideoTrack = Array.from(room.localParticipant.videoTracks.values())[0].track;
+   * const grayScaleProcessor = new GrayScaleProcessor(100);
+   * localVideoTrack.addProcessor(grayScaleProcessor);
+   *
+   * document.getElementById('remove-button').onclick = () => localVideoTrack.removeProcessor(grayScaleProcessor);
+   */
+  removeProcessor() {
+    this._log.debug('Removing VideoProcessor from the LocalVideoTrack');
+    const result = super.removeProcessor.apply(this, arguments);
+
+    this._log.debug('Updating LocalVideoTrack\'s MediaStreamTrack with the original MediaStreamTrack');
+    this._setSenderMediaStreamTrack()
+      .then(() => this._updateElementsMediaStreamTrack());
+
+    return result;
+  }
+
   /**
    * Disable the {@link LocalVideoTrack}. This is effectively "pause".
+   * If a {@link VideoProcessor} is added, then `processedTrack` is disabled as well.
    * @returns {this}
    * @fires VideoTrack#disabled
    */
   disable() {
-    return super.disable.apply(this, arguments);
+    const result = super.disable.apply(this, arguments);
+    if (this.processedTrack) {
+      this.processedTrack.enabled = false;
+    }
+    return result;
   }
 
   /**
    * Enable the {@link LocalVideoTrack}. This is effectively "unpause".
+   * If a {@link VideoProcessor} is added, then `processedTrack` is enabled as well.
    * @returns {this}
    * @fires VideoTrack#enabled
   *//**
    * Enable or disable the {@link LocalVideoTrack}. This is effectively "unpause"
-   * or "pause".
+   * or "pause". If a {@link VideoProcessor} is added,
+   * then `processedTrack` is enabled or disabled as well.
    * @param {boolean} [enabled] - Specify false to pause the
    *   {@link LocalVideoTrack}
    * @returns {this}
    * @fires VideoTrack#disabled
    * @fires VideoTrack#enabled
    */
-  enable() {
-    return super.enable.apply(this, arguments);
+  enable(enabled = true) {
+    const result = super.enable.apply(this, arguments);
+    if (this.processedTrack) {
+      this.processedTrack.enabled = enabled;
+
+      if (enabled) {
+        this._captureFrames();
+        this._log.debug('Updating LocalVideoTrack\'s MediaStreamTrack with the processed MediaStreamTrack', this.processedTrack);
+        this._setSenderMediaStreamTrack(true);
+      }
+    }
+    return result;
   }
 
   /**
@@ -134,7 +250,13 @@ class LocalVideoTrack extends LocalMediaVideoTrack {
       this._workaroundSilentLocalVideoCleanup();
       this._workaroundSilentLocalVideoCleanup = null;
     }
+
     const promise = super.restart.apply(this, arguments);
+    if (this.processor) {
+      promise.then(() => {
+        this._restartProcessor();
+      });
+    }
 
     if (this._workaroundSilentLocalVideo) {
       promise.finally(() => {

diff --git a/lib/media/track/mediatrack.js b/lib/media/track/mediatrack.js
@@ -72,6 +72,10 @@ class MediaTrack extends Track {
         value: options.workaroundWebKitBug212780
           || options.playPausedElementsIfNotBackgrounded
       },
+      _unprocessedTrack: {
+        value: null,
+        writable: true
+      },
       _MediaStream: {
         value: options.MediaStream
       },
@@ -84,8 +88,13 @@ class MediaTrack extends Track {
       mediaStreamTrack: {
         enumerable: true,
         get() {
-          return mediaTrackTransceiver.track;
+          return this._unprocessedTrack || mediaTrackTransceiver.track;
         }
+      },
+      processedTrack: {
+        enumerable: true,
+        value: null,
+        writable: true
       }
     });
 
@@ -122,7 +131,10 @@ class MediaTrack extends Track {
     if (this._dummyEl) {
       this._dummyEl.muted = true;
       this._dummyEl.oncanplay = this._start.bind(this, this._dummyEl);
-      this._attach(this._dummyEl);
+
+      // NOTE(csantos): We always want to attach the original mediaStreamTrack for dummyEl
+      this._attach(this._dummyEl, this.mediaStreamTrack);
+
       this._attachments.delete(this._dummyEl);
     }
   }
@@ -158,22 +170,27 @@ class MediaTrack extends Track {
   }
 
   /**
+   * Attach the provided MediaStreamTrack to the media element.
+   * @param el - The media element to attach to
+   * @param mediaStreamTrack - The MediaStreamTrack to attach. If this is
+   * not provided, it uses the processedTrack if it exists
+   * or it defaults to the current mediaStreamTrack
    * @private
    */
-  _attach(el) {
+  _attach(el, mediaStreamTrack = this.processedTrack || this.mediaStreamTrack) {
     let mediaStream = el.srcObject;
     if (!(mediaStream instanceof this._MediaStream)) {
       mediaStream = new this._MediaStream();
     }
 
-    const getTracks = this.mediaStreamTrack.kind === 'audio'
+    const getTracks = mediaStreamTrack.kind === 'audio'
       ? 'getAudioTracks'
       : 'getVideoTracks';
 
-    mediaStream[getTracks]().forEach(mediaStreamTrack => {
-      mediaStream.removeTrack(mediaStreamTrack);
+    mediaStream[getTracks]().forEach(track => {
+      mediaStream.removeTrack(track);
     });
-    mediaStream.addTrack(this.mediaStreamTrack);
+    mediaStream.addTrack(mediaStreamTrack);
 
     // NOTE(mpatwardhan): resetting `srcObject` here, causes flicker (JSDK-2641), but it lets us
     // to sidestep the a chrome bug: https://bugs.chromium.org/p/chromium/issues/detail?id=1052353
@@ -202,6 +219,14 @@ class MediaTrack extends Track {
     return el;
   }
 
+  /**
+   * @private
+   */
+  _updateElementsMediaStreamTrack() {
+    this._log.debug('Reattaching all elements to update mediaStreamTrack');
+    this._getAllAttachedElements().forEach(el => this._attach(el));
+  }
+
   /**
    * @private
    */
@@ -244,7 +269,7 @@ class MediaTrack extends Track {
 
     const mediaStream = el.srcObject;
     if (mediaStream instanceof this._MediaStream) {
-      mediaStream.removeTrack(this.mediaStreamTrack);
+      mediaStream.removeTrack(this.processedTrack || this.mediaStreamTrack);
     }
 
     this._attachments.delete(el);