Merge pull request #1928 from twilio/merge-2-25-0-master

Merge 2.25.0 changes to master.

CHANGELOG.md

@@ -2,6 +2,67 @@ The Twilio Programmable Video SDKs use [Semantic Versioning](http://www.semver.o
 
 **Version 1.x reached End of Life on September 8th, 2021.** See the changelog entry [here](https://www.twilio.com/changelog/end-of-life-complete-for-unsupported-versions-of-the-programmable-video-sdk). Support for the 1.x version ended on December 4th, 2020.
 
+2.25.0 (November 14, 2022)
+==========================
+
+New Features
+------------
+
+### Auto-switch default audio input devices
+
+This release adds a new feature that preserves audio continuity in situations where end-users change the default audio input device.
+A LocalAudioTrack is said to be capturing audio from the default audio input device if:
+
+- it was created using the MediaTrackConstraints `{ audio: true }`, or
+- it was created using the MediaTrackConstraints `{ audio: { deviceId: 'foo' } }`, and "foo" is not available, or
+- it was created using the MediaTrackConstraints `{ audio: { deviceId: { ideal: 'foo' } } }` and "foo" is not available
+
+In previous versions of the SDK, if the default device changed (ex: a bluetooth headset is connected to a mac or windows laptop),
+the LocalAudioTrack continued to capture audio from the old default device (ex: the laptop microphone). Now, a LocalAudioTrack
+will switch automatically from the old default audio input device to the new default audio input device (ex: from the laptop microphone to the headset microphone).
+This feature is controlled by a new [CreateLocalAudioTrackOptions](https://sdk.twilio.com/js/video/releases/2.25.0/docs/global.html#CreateLocalAudioTrackOptions)
+property `defaultDeviceCaptureMode`, which defaults to `auto` (new behavior) or can be set to `manual` (old behavior).
+
+The application can decide to capture audio from a specific audio input device by creating a LocalAudioTrack:
+
+- using the MediaTrackConstraints `{ audio: { deviceId: 'foo' } }`, and "foo" is available, or
+- using the MediaTrackConstraints `{ audio: { deviceId: { ideal: 'foo' } } }` and "foo" is available, or
+- using the MediaTrackConstraints `{ audio: { deviceId: { exact: 'foo' } } }` and "foo" is available 
+
+In this case, the LocalAudioTrack DOES NOT switch to another audio input device if the current audio input device is no
+longer available. See below for the behavior of this property based on how the LocalAudioTrack is created. (VIDEO-11701)
+
+```js
+const { connect, createLocalAudioTrack, createLocalTracks } = require('twilio-video');
+
+// Auto-switch default audio input devices: option 1
+const audioTrack = await createLocalAudioTrack();
+
+// Auto-switch default audio input devices: option 2
+const audioTrack1 = await createLocalAudioTrack({ defaultDeviceCaptureMode: 'auto' });
+
+// Auto-switch default audio input devices: option 3
+const [audioTrack3] = await createLocalTracks({ audio: true });
+
+// Auto-switch default audio input devices: option 4
+const [audioTrack4] = await createLocalTracks({ audio: { defaultDeviceCaptureMode: 'auto' } });
+
+// Auto-switch default audio input devices: option 5
+const room1 = await connect({ audio: true });
+
+// Auto-switch default audio input devices: option 6
+const room2 = await connect({ audio: { defaultDeviceCaptureMode: 'auto' } });
+
+// Disable auto-switch default audio input devices
+const room = await createLocalAudioTrack({ defaultDeviceCaptureMode: 'manual' });
+```
+
+**Limitations**
+
+- This feature is not enabled on iOS as it is natively supported.
+- Due to this [WebKit bug](https://bugs.webkit.org/show_bug.cgi?id=232835), MacOS Safari Participants may lose their local audio after switching between default audio input devices two-three times.
+- This feature is not supported on Android Chrome, as it does not support the [MediaDevices.ondevicechange](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/devicechange_event#browser_compatibility) event.
+
 2.24.3 (October 10, 2022)
 =========================
 

README.md

@@ -74,8 +74,7 @@ Releases of twilio-video.js are hosted on a CDN, and you can include these
 directly in your web app using a <script> tag.
 
 ```html
-<script src="//sdk.twilio.com/js/video/releases/2.24.3/twilio-video.min.js"></script>
-
+<script src="//sdk.twilio.com/js/video/releases/2.25.0/twilio-video.min.js"></script>
 ```
 
 Using this method, twilio-video.js will set a browser global:

lib/createlocaltrack.js

@@ -124,8 +124,16 @@ const NoiseCancellationVendor = {
  * @property {boolean} [workaroundWebKitBug180748=false] - setting this
  *   attempts to workaround WebKit Bug 180748, where, in Safari, getUserMedia may return a silent audio
  *   MediaStreamTrack.
- * @property {NoiseCancellationOptions} [noiseCancellationOptions] - This property enables using 3rd party plugins
- *   for noise cancellation. This is a beta feature.
+ * @property {DefaultDeviceCaptureMode} [defaultDeviceCaptureMode="auto"] - This optional property only applies if the
+ *   {@link LocalAudioTrack} is capturing from the default audio input device connected to a desktop or laptop. When the
+ *   property is set to "auto", the LocalAudioTrack restarts whenever the default audio input device changes, in order to
+ *   capture audio from the new default audio input device. For example, when a bluetooth audio headset is connected to a
+ *   Macbook, the LocalAudioTrack will start capturing audio from the headset microphone. When the headset is disconnected,
+ *   the LocalAudioTrack will start capturing audio from the Macbook microphone. When the property is set to "manual", the
+ *   LocalAudioTrack continues to capture from the same audio input device even after the default audio input device changes.
+ *   When the property is not specified, it defaults to "auto".
+ * @property {NoiseCancellationOptions} [noiseCancellationOptions] - This optional property enables using 3rd party plugins
+ *   for noise cancellation.
  */
 
 /**

lib/createlocaltracks.ts

@@ -1,7 +1,15 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 'use strict';
 
-import { CreateLocalTrackOptions, CreateLocalTracksOptions, LocalTrack, NoiseCancellationOptions } from '../tsdef/types';
+import {
+  CreateLocalAudioTrackOptions,
+  CreateLocalTrackOptions,
+  CreateLocalTracksOptions,
+  DefaultDeviceCaptureMode,
+  LocalTrack,
+  NoiseCancellationOptions
+} from '../tsdef';
+
 import { applyNoiseCancellation } from './media/track/noisecancellationimpl';
 
 const { buildLogLevels } = require('./util');
@@ -14,7 +22,7 @@ const {
 } = require('./media/track/es5');
 
 const Log = require('./util/log');
-const { DEFAULT_LOG_LEVEL, DEFAULT_LOGGER_NAME } = require('./util/constants');
+const { DEFAULT_LOG_LEVEL, DEFAULT_LOGGER_NAME, typeErrors: { INVALID_VALUE } } = require('./util/constants');
 const workaround180748 = require('./webaudio/workaround180748');
 
 // This is used to make out which createLocalTracks() call a particular Log
@@ -24,7 +32,8 @@ let createLocalTrackCalls = 0;
 
 
 type ExtraLocalTrackOption = CreateLocalTrackOptions & { isCreatedByCreateLocalTracks?: boolean };
-type ExtraLocalTrackOptions = { audio: ExtraLocalTrackOption; video: ExtraLocalTrackOption; };
+type ExtraLocalAudioTrackOption = ExtraLocalTrackOption & { defaultDeviceCaptureMode? : DefaultDeviceCaptureMode };
+type ExtraLocalTrackOptions = { audio: ExtraLocalAudioTrackOption; video: ExtraLocalTrackOption; };
 
 interface InternalOptions extends CreateLocalTracksOptions {
   getUserMedia: any;
@@ -138,7 +147,7 @@ export async function createLocalTracks(options?: CreateLocalTracksOptions): Pro
   const extraLocalTrackOptions: ExtraLocalTrackOptions = {
     audio: typeof fullOptions.audio === 'object' && fullOptions.audio.name
       ? { name: fullOptions.audio.name }
-      : {},
+      : { defaultDeviceCaptureMode: 'auto' },
     video: typeof fullOptions.video === 'object' && fullOptions.video.name
       ? { name: fullOptions.video.name }
       : {}
@@ -147,14 +156,26 @@ export async function createLocalTracks(options?: CreateLocalTracksOptions): Pro
   extraLocalTrackOptions.audio.isCreatedByCreateLocalTracks = true;
   extraLocalTrackOptions.video.isCreatedByCreateLocalTracks = true;
 
-  if (typeof fullOptions.audio === 'object' && typeof fullOptions.audio.workaroundWebKitBug1208516 === 'boolean') {
-    extraLocalTrackOptions.audio.workaroundWebKitBug1208516 = fullOptions.audio.workaroundWebKitBug1208516;
-  }
+  let noiseCancellationOptions: NoiseCancellationOptions | undefined;
 
-  let noiseCancellationOptions: NoiseCancellationOptions|undefined;
-  if (typeof fullOptions.audio === 'object' && 'noiseCancellationOptions' in fullOptions.audio) {
-    noiseCancellationOptions = fullOptions.audio.noiseCancellationOptions;
-    delete fullOptions.audio.noiseCancellationOptions;
+  if (typeof fullOptions.audio === 'object') {
+    if (typeof fullOptions.audio.workaroundWebKitBug1208516 === 'boolean') {
+      extraLocalTrackOptions.audio.workaroundWebKitBug1208516 = fullOptions.audio.workaroundWebKitBug1208516;
+    }
+
+    if ('noiseCancellationOptions' in fullOptions.audio) {
+      noiseCancellationOptions = fullOptions.audio.noiseCancellationOptions;
+      delete fullOptions.audio.noiseCancellationOptions;
+    }
+
+    if (!('defaultDeviceCaptureMode' in fullOptions.audio)) {
+      extraLocalTrackOptions.audio.defaultDeviceCaptureMode = 'auto';
+    } else if (['auto', 'manual'].every(mode => mode !== (fullOptions.audio as CreateLocalAudioTrackOptions).defaultDeviceCaptureMode)) {
+      // eslint-disable-next-line new-cap
+      throw INVALID_VALUE('CreateLocalAudioTrackOptions.defaultDeviceCaptureMode', ['auto', 'manual']);
+    } else {
+      extraLocalTrackOptions.audio.defaultDeviceCaptureMode = fullOptions.audio.defaultDeviceCaptureMode;
+    }
   }
 
   if (typeof fullOptions.video === 'object' && typeof fullOptions.video.workaroundWebKitBug1208516 === 'boolean') {

lib/media/track/localaudiotrack.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const { isIOS } = require('../../util/browserdetection');
 const AudioTrack = require('./audiotrack');
 const mixinLocalMediaTrack = require('./localmediatrack');
 
@@ -33,13 +34,63 @@ class LocalAudioTrack extends LocalMediaAudioTrack {
     const noiseCancellation = options?.noiseCancellation || null;
     super(mediaStreamTrack, options);
 
+    const { _log: log } = this;
+    const { label: defaultDeviceLabel = '' } = mediaStreamTrack;
+    const { deviceId: defaultDeviceId = '', groupId: defaultGroupId = '' } = mediaStreamTrack.getSettings();
+
     Object.defineProperties(this, {
+      _currentDefaultDeviceInfo: {
+        value: { deviceId: defaultDeviceId, groupId: defaultGroupId, label: defaultDeviceLabel },
+        writable: true
+      },
+      _defaultDeviceCaptureMode: {
+        value: !isIOS()
+          && this._isCreatedByCreateLocalTracks
+          && typeof navigator === 'object'
+          && typeof navigator.mediaDevices === 'object'
+          && typeof navigator.mediaDevices.addEventListener === 'function'
+          && typeof navigator.mediaDevices.enumerateDevices === 'function'
+          ? options?.defaultDeviceCaptureMode || 'auto'
+          : 'manual'
+      },
+      _onDeviceChange: {
+        value: () => {
+          navigator.mediaDevices.enumerateDevices().then(deviceInfos => {
+            // NOTE(mmalavalli): In Chrome, when the default device changes, and we restart the LocalAudioTrack with
+            // device ID "default", it will not switch to the new default device unless all LocalAudioTracks capturing
+            // from the old default device are stopped. So, we restart the LocalAudioTrack with the actual device ID of
+            // the new default device instead.
+            const defaultDeviceInfo = deviceInfos.find(({ deviceId, kind }) => {
+              return kind === 'audioinput' && deviceId !== 'default';
+            });
+
+            if (defaultDeviceInfo && ['deviceId', 'groupId'].some(prop => {
+              return defaultDeviceInfo[prop] !== this._currentDefaultDeviceInfo[prop];
+            })) {
+              log.info('Default device changed, restarting the LocalAudioTrack');
+              log.debug(`Old default device: "${this._currentDefaultDeviceInfo.deviceId}" => "${this._currentDefaultDeviceInfo.label}"`);
+              log.debug(`New default device: "${defaultDeviceInfo.deviceId}" => "${defaultDeviceInfo.label}"`);
+              this._currentDefaultDeviceInfo = defaultDeviceInfo;
+              this._restartDefaultDevice().catch(error => log.warn(`Failed to restart: ${error.message}`));
+            }
+          }, error => {
+            log.warn(`Failed to run enumerateDevices(): ${error.message}`);
+          });
+        }
+      },
+      _restartOnDefaultDeviceChangeCleanup: {
+        value: null,
+        writable: true
+      },
       noiseCancellation: {
         enumerable: true,
         value: noiseCancellation,
         writable: false
       },
     });
+
+    log.debug('defaultDeviceCaptureMode:', this._defaultDeviceCaptureMode);
+    this._maybeRestartOnDefaultDeviceChange();
   }
 
   toString() {
@@ -59,6 +110,49 @@ class LocalAudioTrack extends LocalMediaAudioTrack {
     return super._end.apply(this, arguments);
   }
 
+  /**
+   * @private
+   */
+  _maybeRestartOnDefaultDeviceChange() {
+    const { _constraints: constraints, _defaultDeviceCaptureMode: defaultDeviceCaptureMode, _log: log } = this;
+    const mediaStreamTrack = this.noiseCancellation ? this.noiseCancellation.sourceTrack : this.mediaStreamTrack;
+    const { deviceId } = mediaStreamTrack.getSettings();
+
+    const isNotEqualToCapturedDeviceIdOrEqualToDefault = requestedDeviceId => {
+      return requestedDeviceId !== deviceId || requestedDeviceId === 'default';
+    };
+
+    const isCapturingFromDefaultDevice = (function checkIfCapturingFromDefaultDevice(deviceIdConstraint = {}) {
+      if (typeof deviceIdConstraint === 'string') {
+        return isNotEqualToCapturedDeviceIdOrEqualToDefault(deviceIdConstraint);
+      } else if (Array.isArray(deviceIdConstraint)) {
+        return deviceIdConstraint.every(isNotEqualToCapturedDeviceIdOrEqualToDefault);
+      } else if (deviceIdConstraint.exact) {
+        return checkIfCapturingFromDefaultDevice(deviceIdConstraint.exact);
+      } else if (deviceIdConstraint.ideal) {
+        return checkIfCapturingFromDefaultDevice(deviceIdConstraint.ideal);
+      }
+      return true;
+    }(constraints.deviceId));
+
+    if (defaultDeviceCaptureMode === 'auto' && isCapturingFromDefaultDevice) {
+      if (!this._restartOnDefaultDeviceChangeCleanup) {
+        log.info('LocalAudioTrack will be restarted if the default device changes');
+        navigator.mediaDevices.addEventListener('devicechange', this._onDeviceChange);
+        this._restartOnDefaultDeviceChangeCleanup = () => {
+          log.info('Cleaning up the listener to restart the LocalAudioTrack if the default device changes');
+          navigator.mediaDevices.removeEventListener('devicechange', this._onDeviceChange);
+          this._restartOnDefaultDeviceChangeCleanup = null;
+        };
+      }
+    } else {
+      log.info('LocalAudioTrack will NOT be restarted if the default device changes');
+      if (this._restartOnDefaultDeviceChangeCleanup) {
+        this._restartOnDefaultDeviceChangeCleanup();
+      }
+    }
+  }
+
   /**
    * @private
    */
@@ -73,6 +167,21 @@ class LocalAudioTrack extends LocalMediaAudioTrack {
     return super._reacquireTrack.call(this, constraints);
   }
 
+  /**
+   * @private
+   */
+  _restartDefaultDevice() {
+    const constraints = Object.assign({}, this._constraints);
+    const restartConstraints = Object.assign({}, constraints, { deviceId: this._currentDefaultDeviceInfo.deviceId });
+    return this.restart(restartConstraints).then(() => {
+      // NOTE(mmalavalli): Since we used the new default device's ID while restarting the LocalAudioTrack,
+      // we reset the constraints to the original constraints so that the default device detection logic in
+      // _maybeRestartOnDefaultDeviceChange() still works.
+      this._constraints = constraints;
+      this._maybeRestartOnDefaultDeviceChange();
+    });
+  }
+
   /**
    * Disable the {@link LocalAudioTrack}. This is effectively "mute".
    * @returns {this}
@@ -147,6 +256,9 @@ class LocalAudioTrack extends LocalMediaAudioTrack {
     if (this.noiseCancellation) {
       this.noiseCancellation.stop();
     }
+    if (this._restartOnDefaultDeviceChangeCleanup) {
+      this._restartOnDefaultDeviceChangeCleanup();
+    }
     return super.stop.apply(this, arguments);
   }
 }

lib/media/track/localmediatrack.js

@@ -2,7 +2,7 @@
 'use strict';
 
 const { getUserMedia } = require('../../webrtc');
-const { guessBrowser, isIOSChrome } = require('../../webrtc/util');
+const { isIOS } = require('../../util/browserdetection');
 
 const { capitalize, defer, isUserMediaTrack, waitForSometime, waitForEvent } = require('../../util');
 const { typeErrors: { ILLEGAL_INVOKE } } = require('../../util/constants');
@@ -29,10 +29,7 @@ function mixinLocalMediaTrack(AudioOrVideoTrack) {
      * @param {LocalTrackOptions} [options] - {@link LocalTrack} options
      */
     constructor(mediaStreamTrack, options) {
-      // NOTE(mpatwardhan): by default workaround for WebKitBug1208516 will be enabled on Safari browsers
-      // although the bug is seen  mainly on iOS devices, we do not have a reliable way to tell iOS from MacOs
-      // userAgent on iOS pretends its macOs if Safari is set to request desktop pages.
-      const workaroundWebKitBug1208516 = (guessBrowser() === 'safari' || isIOSChrome())
+      const workaroundWebKitBug1208516 = isIOS()
         && isUserMediaTrack(mediaStreamTrack)
         && typeof document === 'object'
         && typeof document.addEventListener === 'function'

lib/media/track/localvideotrack.js

@@ -1,7 +1,6 @@
 'use strict';
 
-const { guessBrowser, isIOSChrome } = require('../../webrtc/util');
-
+const { isIOS } = require('../../util/browserdetection');
 const detectSilentVideo = require('../../util/detectsilentvideo');
 const mixinLocalMediaTrack = require('./localmediatrack');
 const VideoTrack = require('./videotrack');
@@ -32,7 +31,7 @@ class LocalVideoTrack extends LocalMediaVideoTrack {
    */
   constructor(mediaStreamTrack, options) {
     options = Object.assign({
-      workaroundSilentLocalVideo: (guessBrowser() === 'safari' || isIOSChrome())
+      workaroundSilentLocalVideo: isIOS()
         && isUserMediaTrack(mediaStreamTrack)
         && typeof document !== 'undefined'
         && typeof document.createElement === 'function'

lib/media/track/mediatrack.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { guessBrowser, isIOSChrome } = require('../../webrtc/util');
+const { isIOS } = require('../../util/browserdetection');
 const { MediaStream } = require('../../webrtc');
 
 const { waitForEvent, waitForSometime } = require('../../util');
@@ -32,7 +32,7 @@ class MediaTrack extends Track {
    */
   constructor(mediaTrackTransceiver, options) {
     options = Object.assign({
-      playPausedElementsIfNotBackgrounded: (guessBrowser() === 'safari' || isIOSChrome())
+      playPausedElementsIfNotBackgrounded: isIOS()
         && typeof document === 'object'
         && typeof document.addEventListener === 'function'
         && typeof document.visibilityState === 'string'