feat: speaking while muted notification (#1011)

The new device API now able to detect is the user is speaking while
their mic is turned off.

The feature is not available on RN, but @santhoshvai, @khushal87 or
@vishalnarkhede please test that nothing breaks for RN.

---------

Co-authored-by: Oliver Lazoroski <[email protected]>

packages/client/docusaurus/docs/javascript/02-guides/04-camera-and-microphone.mdx

@@ -178,18 +178,50 @@ Follow our [Playing Video and Audio guide](../../guides/playing-video-and-audio/
 
 #### Lobby preview
 
-This is how you can show a visual representation about the sound level coming from the user's selected microphone:
+On lobby screens a common UX pattern is to show a visual indicator of the detected audio levels coming from the selected microphone. The client exposes the `createSoundDetector` utility method to help implement this functionality. Here is an example of how you can do that:
+
+```html
+<progress id="volume" max="100" min="0"></progress>
+```
 
 ```typescript
-// Code example coming soon 🏗️
+import { createSoundDetector } from '@stream-io/video-client';
+
+let cleanup: Function | undefined;
+call.microphone.state.mediaStream$.subscribe(async (mediaStream) => {
+  const progressBar = document.getElementById('volume') as HTMLProgressElement;
+  if (mediaStream) {
+    cleanup = createSoundDetector(
+      mediaStream,
+      (event) => {
+        progressBar.value = event.audioLevel;
+      },
+      { detectionFrequencyInMs: 100 },
+    );
+  } else {
+    await cleanup?.();
+    progressBar.value = 0;
+  }
+});
 ```
 
 ### Speaking while muted notification
 
+When the microphone is disabled, the client will automatically start monitoring audio levels, to detect if the user is speaking.
+
+This is how you can subscribe to these notifications:
+
 ```typescript
-// This feature is coming soon 🏗️
+call.microphone.state.speakingWhileMuted; // current value
+call.microphone.state.speakingWhileMuted$.subscribe((isSpeaking) => {
+  if (isSpeaking) {
+    console.log(`You're muted, unmute yourself to speak`);
+  }
+}); // Reactive value
 ```
 
+The notification is automatically disabled if the user doesn't have the permission to send audio.
+
 ## Speaker management
 
 ### List and select devices

packages/client/src/Call.ts

@@ -116,10 +116,12 @@ import {
 } from './coordinator/connection/types';
 import { getClientDetails, getSdkInfo } from './client-details';
 import { getLogger } from './logger';
-import { CameraManager } from './devices/CameraManager';
-import { MicrophoneManager } from './devices/MicrophoneManager';
-import { CameraDirection } from './devices/CameraManagerState';
-import { SpeakerManager } from './devices/SpeakerManager';
+import {
+  CameraDirection,
+  CameraManager,
+  MicrophoneManager,
+  SpeakerManager,
+} from './devices';
 
 /**
  * An object representation of a `Call`.

packages/client/src/devices/InputMediaDeviceManager.ts

@@ -141,7 +141,7 @@ export abstract class InputMediaDeviceManager<
 
   protected abstract getTrack(): undefined | MediaStreamTrack;
 
-  private async muteStream(stopTracks: boolean = true) {
+  protected async muteStream(stopTracks: boolean = true) {
     if (!this.state.mediaStream) {
       return;
     }
@@ -154,7 +154,7 @@ export abstract class InputMediaDeviceManager<
       // @ts-expect-error release() is present in react-native-webrtc and must be called to dispose the stream
       if (typeof this.state.mediaStream.release === 'function') {
         // @ts-expect-error
-        this.state.mediaStream.release();
+        stream.release();
       }
       this.state.setMediaStream(undefined);
     }
@@ -191,7 +191,7 @@ export abstract class InputMediaDeviceManager<
     stopTracks ? this.stopTrack() : this.muteTrack();
   }
 
-  private async unmuteStream() {
+  protected async unmuteStream() {
     this.logger('debug', 'Starting stream');
     let stream: MediaStream;
     if (this.state.mediaStream && this.getTrack()?.readyState === 'live') {
@@ -207,6 +207,8 @@ export abstract class InputMediaDeviceManager<
     if (this.call.state.callingState === CallingState.JOINED) {
       await this.publishStream(stream);
     }
-    this.state.setMediaStream(stream);
+    if (this.state.mediaStream !== stream) {
+      this.state.setMediaStream(stream);
+    }
   }
 }

packages/client/src/devices/MicrophoneManager.ts

@@ -1,13 +1,42 @@
-import { Observable } from 'rxjs';
+import { combineLatest, Observable } from 'rxjs';
 import { Call } from '../Call';
 import { InputMediaDeviceManager } from './InputMediaDeviceManager';
 import { MicrophoneManagerState } from './MicrophoneManagerState';
 import { getAudioDevices, getAudioStream } from './devices';
 import { TrackType } from '../gen/video/sfu/models/models';
+import { createSoundDetector } from '../helpers/sound-detector';
+import { isReactNative } from '../helpers/platforms';
+import { OwnCapability } from '../gen/coordinator';
+import { CallingState } from '../store';
 
 export class MicrophoneManager extends InputMediaDeviceManager<MicrophoneManagerState> {
+  private soundDetectorCleanup?: Function;
+
   constructor(call: Call) {
     super(call, new MicrophoneManagerState(), TrackType.AUDIO);
+
+    combineLatest([
+      this.call.state.callingState$,
+      this.call.state.ownCapabilities$,
+      this.state.selectedDevice$,
+      this.state.status$,
+    ]).subscribe(async ([callingState, ownCapabilities, deviceId, status]) => {
+      if (callingState !== CallingState.JOINED) {
+        if (callingState === CallingState.LEFT) {
+          await this.stopSpeakingWhileMutedDetection();
+        }
+        return;
+      }
+      if (ownCapabilities.includes(OwnCapability.SEND_AUDIO)) {
+        if (status === 'disabled') {
+          await this.startSpeakingWhileMutedDetection(deviceId);
+        } else {
+          await this.stopSpeakingWhileMutedDetection();
+        }
+      } else {
+        await this.stopSpeakingWhileMutedDetection();
+      }
+    });
   }
 
   protected getDevices(): Observable<MediaDeviceInfo[]> {
@@ -28,4 +57,30 @@ export class MicrophoneManager extends InputMediaDeviceManager<MicrophoneManager
   protected getTrack() {
     return this.state.mediaStream?.getAudioTracks()[0];
   }
+
+  private async startSpeakingWhileMutedDetection(deviceId?: string) {
+    if (isReactNative()) {
+      return;
+    }
+    await this.stopSpeakingWhileMutedDetection();
+    // Need to start a new stream that's not connected to publisher
+    const stream = await this.getStream({
+      deviceId,
+    });
+    this.soundDetectorCleanup = createSoundDetector(stream, (event) => {
+      this.state.setSpeakingWhileMuted(event.isSoundDetected);
+    });
+  }
+
+  private async stopSpeakingWhileMutedDetection() {
+    if (isReactNative() || !this.soundDetectorCleanup) {
+      return;
+    }
+    this.state.setSpeakingWhileMuted(false);
+    try {
+      await this.soundDetectorCleanup();
+    } finally {
+      this.soundDetectorCleanup = undefined;
+    }
+  }
 }

packages/client/src/devices/MicrophoneManagerState.ts

@@ -1,8 +1,38 @@
+import { BehaviorSubject, Observable, distinctUntilChanged } from 'rxjs';
 import { InputMediaDeviceManagerState } from './InputMediaDeviceManagerState';
 
 export class MicrophoneManagerState extends InputMediaDeviceManagerState {
+  private speakingWhileMutedSubject = new BehaviorSubject<boolean>(false);
+
+  /**
+   * An Observable that emits `true` if the user's microphone is muted but they'are speaking.
+   *
+   * This feature is not available in the React Native SDK.
+   */
+  speakingWhileMuted$: Observable<boolean>;
+
   constructor() {
     super('disable-tracks');
+
+    this.speakingWhileMuted$ = this.speakingWhileMutedSubject
+      .asObservable()
+      .pipe(distinctUntilChanged());
+  }
+
+  /**
+   * `true` if the user's microphone is muted but they'are speaking.
+   *
+   * This feature is not available in the React Native SDK.
+   */
+  get speakingWhileMuted() {
+    return this.getCurrentValue(this.speakingWhileMuted$);
+  }
+
+  /**
+   * @internal
+   */
+  setSpeakingWhileMuted(isSpeaking: boolean) {
+    this.setCurrentValue(this.speakingWhileMutedSubject, isSpeaking);
   }
 
   protected getDeviceIdFromStream(stream: MediaStream): string | undefined {

packages/client/src/devices/__tests__/CameraManager.test.ts

@@ -2,7 +2,7 @@ import { Call } from '../../Call';
 import { StreamClient } from '../../coordinator/connection/client';
 import { CallingState, StreamVideoWriteableStateStore } from '../../store';
 
-import { afterEach, beforeEach, describe, vi, it, expect, Mock } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, Mock, vi } from 'vitest';
 import { mockCall, mockVideoDevices, mockVideoStream } from './mocks';
 import { getVideoStream } from '../devices';
 import { TrackType } from '../../gen/video/sfu/models/models';
@@ -67,8 +67,7 @@ describe('CameraManager', () => {
   });
 
   it('publish stream', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
 
     await manager.enable();
 
@@ -78,8 +77,7 @@ describe('CameraManager', () => {
   });
 
   it('stop publish stream', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
     await manager.enable();
 
     await manager.disable();

packages/client/src/devices/__tests__/InputMediaDeviceManager.test.ts

@@ -2,7 +2,7 @@ import { Call } from '../../Call';
 import { StreamClient } from '../../coordinator/connection/client';
 import { CallingState, StreamVideoWriteableStateStore } from '../../store';
 
-import { afterEach, beforeEach, describe, vi, it, expect } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { mockCall, mockVideoDevices, mockVideoStream } from './mocks';
 import { InputMediaDeviceManager } from '../InputMediaDeviceManager';
 import { InputMediaDeviceManagerState } from '../InputMediaDeviceManagerState';
@@ -67,8 +67,7 @@ describe('InputMediaDeviceManager.test', () => {
   });
 
   it('enable device - after joined to call', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
 
     await manager.enable();
 
@@ -93,8 +92,7 @@ describe('InputMediaDeviceManager.test', () => {
   });
 
   it('disable device - after joined to call', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
     await manager.enable();
 
     await manager.disable();
@@ -136,8 +134,7 @@ describe('InputMediaDeviceManager.test', () => {
   });
 
   it('select device when status is enabled and in call', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
     await manager.enable();
 
     const deviceId = mockVideoDevices[1].deviceId;