diff --git a/examples/prebuilt-react-integration/index.html b/examples/prebuilt-react-integration/index.html
index 895cbcd8ce..aee5d151bb 100644
--- a/examples/prebuilt-react-integration/index.html
+++ b/examples/prebuilt-react-integration/index.html
@@ -1,4 +1,4 @@
-
+
diff --git a/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts b/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts
index e387fa4991..d7f4200922 100644
--- a/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts
+++ b/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts
@@ -256,6 +256,18 @@ export default class AnalyticsEventFactory {
level: AnalyticsEventLevel.INFO,
});
}
+
+ static interruption(started: boolean, type: string, deviceInfo: Partial) {
+ return new AnalyticsEvent({
+ name: `${started ? 'interruption.start' : 'interruption.stop'}`,
+ level: AnalyticsEventLevel.INFO,
+ properties: {
+ type,
+ ...deviceInfo,
+ },
+ });
+ }
+
private static eventNameFor(name: string, ok: boolean) {
const suffix = ok ? 'success' : 'failed';
return `${name}.${suffix}`;
diff --git a/packages/hms-video-store/src/analytics/AnalyticsTransport.ts b/packages/hms-video-store/src/analytics/AnalyticsTransport.ts
index 7f3dde0637..77a2955d6e 100644
--- a/packages/hms-video-store/src/analytics/AnalyticsTransport.ts
+++ b/packages/hms-video-store/src/analytics/AnalyticsTransport.ts
@@ -9,7 +9,30 @@ export abstract class AnalyticsTransport {
abstract failedEvents: Queue;
private readonly TAG = '[AnalyticsTransport]';
+ private eventCount = 0;
+ private lastResetTime: number = Date.now();
+ private readonly MAX_EVENTS_PER_MINUTE: number = 200;
+ private readonly RESET_INTERVAL_MS: number = 60000;
+
+ private checkRateLimit() {
+ const now = Date.now();
+ if (now - this.lastResetTime >= this.RESET_INTERVAL_MS) {
+ this.eventCount = 0;
+ this.lastResetTime = now;
+ }
+ if (this.eventCount >= this.MAX_EVENTS_PER_MINUTE) {
+ throw new Error('Too many events being sent, please check the implementation.');
+ }
+ this.eventCount++;
+ }
+
sendEvent(event: AnalyticsEvent) {
+ try {
+ this.checkRateLimit();
+ } catch (e) {
+ HMSLogger.w(this.TAG, 'Rate limit exceeded', e);
+ throw e;
+ }
try {
this.sendSingleEvent(event);
this.flushFailedEvents();
diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts
index 6d1e667888..a5e2b90b76 100644
--- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts
+++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts
@@ -295,29 +295,15 @@ export class AudioSinkManager {
if ('ondevicechange' in navigator.mediaDevices) {
return;
}
- let bluetoothDevice: InputDeviceInfo | null = null;
- let speakerPhone: InputDeviceInfo | null = null;
- let wired: InputDeviceInfo | null = null;
- let earpiece: InputDeviceInfo | null = null;
-
- for (const device of this.deviceManager.audioInput) {
- const label = device.label.toLowerCase();
- if (label.includes('speakerphone')) {
- speakerPhone = device;
- } else if (label.includes('wired')) {
- wired = device;
- } else if (label.includes('bluetooth')) {
- bluetoothDevice = device;
- } else if (label.includes('earpiece')) {
- earpiece = device;
- }
- }
+ const { bluetoothDevice, earpiece, speakerPhone, wired } = this.deviceManager.categorizeAudioInputDevices();
const localAudioTrack = this.store.getLocalPeer()?.audioTrack;
if (localAudioTrack && earpiece) {
- const externalDeviceID = bluetoothDevice?.deviceId || wired?.deviceId || speakerPhone?.deviceId;
+ const manualSelection = this.deviceManager.getManuallySelectedAudioDevice();
+ const externalDeviceID =
+ manualSelection?.deviceId || bluetoothDevice?.deviceId || wired?.deviceId || speakerPhone?.deviceId;
HMSLogger.d(this.TAG, 'externalDeviceID', externalDeviceID);
// already selected appropriate device
- if (localAudioTrack.settings.deviceId === externalDeviceID) {
+ if (localAudioTrack.settings.deviceId === externalDeviceID && this.earpieceSelected) {
return;
}
if (!this.earpieceSelected) {
diff --git a/packages/hms-video-store/src/connection/HMSConnection.ts b/packages/hms-video-store/src/connection/HMSConnection.ts
index aa8619b67d..cc29dd1581 100644
--- a/packages/hms-video-store/src/connection/HMSConnection.ts
+++ b/packages/hms-video-store/src/connection/HMSConnection.ts
@@ -28,6 +28,8 @@ export default abstract class HMSConnection {
* - [HMSSubscribeConnection] clears this list as soon as we call [addIceCandidate]
*/
readonly candidates = new Array();
+ // @ts-ignore
+ private sfuNodeId?: string;
selectedCandidatePair?: RTCIceCandidatePair;
@@ -48,6 +50,10 @@ export default abstract class HMSConnection {
return this.role === HMSConnectionRole.Publish ? HMSAction.PUBLISH : HMSAction.SUBSCRIBE;
}
+ setSfuNodeId(nodeId?: string) {
+ this.sfuNodeId = nodeId;
+ }
+
addTransceiver(track: MediaStreamTrack, init: RTCRtpTransceiverInit): RTCRtpTransceiver {
return this.nativeConnection.addTransceiver(track, init);
}
@@ -198,7 +204,7 @@ export default abstract class HMSConnection {
return await this.nativeConnection.getStats();
}
- async close() {
+ close() {
this.nativeConnection.close();
}
diff --git a/packages/hms-video-store/src/connection/publish/publishConnection.ts b/packages/hms-video-store/src/connection/publish/publishConnection.ts
index 042410914c..422c7eb8c8 100644
--- a/packages/hms-video-store/src/connection/publish/publishConnection.ts
+++ b/packages/hms-video-store/src/connection/publish/publishConnection.ts
@@ -32,7 +32,6 @@ export default class HMSPublishConnection extends HMSConnection {
this.observer.onIceConnectionChange(this.nativeConnection.iceConnectionState);
};
- // @TODO(eswar): Remove this. Use iceconnectionstate change with interval and threshold.
this.nativeConnection.onconnectionstatechange = () => {
this.observer.onConnectionStateChange(this.nativeConnection.connectionState);
@@ -51,6 +50,11 @@ export default class HMSPublishConnection extends HMSConnection {
};
}
+ close() {
+ super.close();
+ this.channel.close();
+ }
+
initAfterJoin() {
this.nativeConnection.onnegotiationneeded = async () => {
HMSLogger.d(this.TAG, `onnegotiationneeded`);
diff --git a/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts b/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts
index 2758b7f290..3f95f886b2 100644
--- a/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts
+++ b/packages/hms-video-store/src/connection/subscribe/subscribeConnection.ts
@@ -154,8 +154,8 @@ export default class HMSSubscribeConnection extends HMSConnection {
return this.sendMessage(request, id);
}
- async close() {
- await super.close();
+ close() {
+ super.close();
this.apiChannel?.close();
}
diff --git a/packages/hms-video-store/src/device-manager/DeviceManager.ts b/packages/hms-video-store/src/device-manager/DeviceManager.ts
index 049f7af74d..86e908780d 100644
--- a/packages/hms-video-store/src/device-manager/DeviceManager.ts
+++ b/packages/hms-video-store/src/device-manager/DeviceManager.ts
@@ -4,6 +4,7 @@ import { ErrorFactory } from '../error/ErrorFactory';
import { HMSException } from '../error/HMSException';
import { EventBus } from '../events/EventBus';
import { DeviceMap, HMSDeviceChangeEvent, SelectedDevices } from '../interfaces';
+import { isIOS } from '../internal';
import { HMSAudioTrackSettingsBuilder, HMSVideoTrackSettingsBuilder } from '../media/settings';
import { HMSLocalAudioTrack, HMSLocalTrack, HMSLocalVideoTrack } from '../media/tracks';
import { Store } from '../sdk/store';
@@ -205,7 +206,11 @@ export class DeviceManager implements HMSDeviceManager {
}
const audioDeviceId = this.store.getConfig()?.settings?.audioInputDeviceId;
if (!audioDeviceId && localPeer?.audioTrack) {
- await localPeer.audioTrack.setSettings({ deviceId: this.audioInput[0]?.deviceId }, true);
+ const getInitialDeviceId = () => {
+ const nonIPhoneDevice = this.audioInput.find(device => !device.label.toLowerCase().includes('iphone'));
+ return isIOS() && nonIPhoneDevice ? nonIPhoneDevice?.deviceId : this.getNewAudioInputDevice()?.deviceId;
+ };
+ await localPeer.audioTrack.setSettings({ deviceId: getInitialDeviceId() }, true);
}
};
@@ -232,16 +237,12 @@ export class DeviceManager implements HMSDeviceManager {
* @returns {MediaDeviceInfo}
*/
getNewAudioInputDevice() {
- const localPeer = this.store.getLocalPeer();
- const audioTrack = localPeer?.audioTrack;
- const manualSelection = this.audioInput.find(
- device => device.deviceId === audioTrack?.getManuallySelectedDeviceId(),
- );
+ const manualSelection = this.getManuallySelectedAudioDevice();
if (manualSelection) {
return manualSelection;
}
// if manually selected device is not available, reset on the track
- audioTrack?.resetManuallySelectedDeviceId();
+ this.store.getLocalPeer()?.audioTrack?.resetManuallySelectedDeviceId();
const defaultDevice = this.audioInput.find(device => device.deviceId === 'default');
if (defaultDevice) {
// Selecting a non-default device so that the deviceId comparision does not give
@@ -416,6 +417,35 @@ export class DeviceManager implements HMSDeviceManager {
}
};
+ getManuallySelectedAudioDevice() {
+ const localPeer = this.store.getLocalPeer();
+ const audioTrack = localPeer?.audioTrack;
+ return this.audioInput.find(device => device.deviceId === audioTrack?.getManuallySelectedDeviceId());
+ }
+
+ // specifically used for mweb
+ categorizeAudioInputDevices() {
+ let bluetoothDevice: InputDeviceInfo | null = null;
+ let speakerPhone: InputDeviceInfo | null = null;
+ let wired: InputDeviceInfo | null = null;
+ let earpiece: InputDeviceInfo | null = null;
+
+ for (const device of this.audioInput) {
+ const label = device.label.toLowerCase();
+ if (label.includes('speakerphone')) {
+ speakerPhone = device;
+ } else if (label.includes('wired')) {
+ wired = device;
+ } else if (/airpods|buds|wireless|bluetooth/gi.test(label)) {
+ bluetoothDevice = device;
+ } else if (label.includes('earpiece')) {
+ earpiece = device;
+ }
+ }
+
+ return { bluetoothDevice, speakerPhone, wired, earpiece };
+ }
+
// eslint-disable-next-line complexity
private getAudioOutputDeviceMatchingInput(inputDevice?: MediaDeviceInfo) {
const blacklist = this.store.getConfig()?.settings?.speakerAutoSelectionBlacklist || [];
diff --git a/packages/hms-video-store/src/interfaces/room.ts b/packages/hms-video-store/src/interfaces/room.ts
index db861e0725..aa7ecafefb 100644
--- a/packages/hms-video-store/src/interfaces/room.ts
+++ b/packages/hms-video-store/src/interfaces/room.ts
@@ -33,13 +33,8 @@ export interface HMSRoom {
description?: string;
max_size?: number;
large_room_optimization?: boolean;
- /**
- * @alpha
- */
isEffectsEnabled?: boolean;
- /**
- * @alpha
- */
+ isVBEnabled?: boolean;
effectsKey?: string;
isHipaaEnabled?: boolean;
isNoiseCancellationEnabled?: boolean;
diff --git a/packages/hms-video-store/src/interfaces/update-listener.ts b/packages/hms-video-store/src/interfaces/update-listener.ts
index 315713cb34..adbe9973ad 100644
--- a/packages/hms-video-store/src/interfaces/update-listener.ts
+++ b/packages/hms-video-store/src/interfaces/update-listener.ts
@@ -83,6 +83,7 @@ export interface HMSUpdateListener extends DeviceChangeListener, SessionStoreLis
onError(error: HMSException): void;
onReconnecting(error: HMSException): void;
onReconnected(): void;
+ onSFUMigration?: () => void;
onRoleChangeRequest(request: HMSRoleChangeRequest): void;
onRoleUpdate(newRole: string): void;
onChangeTrackStateRequest(request: HMSChangeTrackStateRequest): void;
diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts
index 2e3883660c..5c9c4785ca 100644
--- a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts
+++ b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts
@@ -9,7 +9,6 @@ import { HMSAudioPlugin, HMSPluginSupportResult } from '../../plugins';
import { HMSAudioPluginsManager } from '../../plugins/audio';
import Room from '../../sdk/models/HMSRoom';
import HMSLogger from '../../utils/logger';
-import { isBrowser, isIOS } from '../../utils/support';
import { getAudioTrack, isEmptyTrack } from '../../utils/track';
import { TrackAudioLevelMonitor } from '../../utils/track-audio-level-monitor';
import { HMSAudioTrackSettings, HMSAudioTrackSettingsBuilder } from '../settings';
@@ -47,7 +46,7 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
source: string,
private eventBus: EventBus,
settings: HMSAudioTrackSettings = new HMSAudioTrackSettingsBuilder().build(),
- room?: Room,
+ private room?: Room,
) {
super(stream, track, source);
stream.tracks.push(this);
@@ -60,11 +59,31 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
}
this.pluginsManager = new HMSAudioPluginsManager(this, eventBus, room);
this.setFirstTrackId(track.id);
- if (isIOS() && isBrowser) {
+ if (source === 'regular') {
document.addEventListener('visibilitychange', this.handleVisibilityChange);
}
}
+ clone(stream: HMSLocalStream) {
+ const track = new HMSLocalAudioTrack(
+ stream,
+ this.nativeTrack.clone(),
+ this.source!,
+ this.eventBus,
+ this.settings,
+ this.room,
+ );
+
+ if (this.pluginsManager.pluginsMap.size > 0) {
+ this.pluginsManager.pluginsMap.forEach(value => {
+ track
+ .addPlugin(value)
+ .catch((e: Error) => HMSLogger.e(this.TAG, 'Plugin add failed while migrating', value, e));
+ });
+ }
+ return track;
+ }
+
getManuallySelectedDeviceId() {
return this.manuallySelectedDeviceId;
}
@@ -73,9 +92,30 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
this.manuallySelectedDeviceId = undefined;
}
+ private isTrackNotPublishing = () => {
+ return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted;
+ };
+
private handleVisibilityChange = async () => {
- if (document.visibilityState === 'visible') {
+ // track state is fine do nothing
+ if (!this.isTrackNotPublishing()) {
+ HMSLogger.d(this.TAG, `visibiltiy: ${document.visibilityState}`, `${this}`);
+ return;
+ }
+ if (document.visibilityState === 'hidden') {
+ this.eventBus.analytics.publish(
+ this.sendInterruptionEvent({
+ started: true,
+ }),
+ );
+ } else {
+ HMSLogger.d(this.TAG, 'On visibile replacing track as it is not publishing');
await this.replaceTrackWith(this.settings);
+ this.eventBus.analytics.publish(
+ this.sendInterruptionEvent({
+ started: false,
+ }),
+ );
}
};
@@ -230,9 +270,7 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
this.processedTrack?.stop();
this.isPublished = false;
this.destroyAudioLevelMonitor();
- if (isIOS() && isBrowser) {
- document.removeEventListener('visibilitychange', this.handleVisibilityChange);
- }
+ document.removeEventListener('visibilitychange', this.handleVisibilityChange);
}
/**
diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts
index 0a19048514..f578b2f2c4 100644
--- a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts
+++ b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts
@@ -70,7 +70,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
source: string,
private eventBus: EventBus,
settings: HMSVideoTrackSettings = new HMSVideoTrackSettingsBuilder().build(),
- room?: Room,
+ private room?: Room,
) {
super(stream, track, source);
stream.tracks.push(this);
@@ -84,11 +84,33 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
this.pluginsManager = new HMSVideoPluginsManager(this, eventBus);
this.mediaStreamPluginsManager = new HMSMediaStreamPluginsManager(eventBus, room);
this.setFirstTrackId(this.trackId);
- if (isBrowser && isMobile()) {
+ if (isBrowser && source === 'regular' && isMobile()) {
document.addEventListener('visibilitychange', this.handleVisibilityChange);
}
}
+ clone(stream: HMSLocalStream) {
+ const track = new HMSLocalVideoTrack(
+ stream,
+ this.nativeTrack.clone(),
+ this.source!,
+ this.eventBus,
+ this.settings,
+ this.room,
+ );
+ if (this.pluginsManager.pluginsMap.size > 0) {
+ this.pluginsManager.pluginsMap.forEach(value => {
+ track
+ .addPlugin(value)
+ .catch((e: Error) => HMSLogger.e(this.TAG, 'Plugin add failed while migrating', value, e));
+ });
+ }
+ if (this.mediaStreamPluginsManager.plugins.size > 0) {
+ track.addStreamPlugins(Array.from(this.mediaStreamPluginsManager.plugins));
+ }
+ return track;
+ }
+
/** @internal */
setSimulcastDefinitons(definitions: HMSSimulcastLayerDefinition[]) {
this._layerDefinitions = definitions;
@@ -144,6 +166,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
} else {
await this.setProcessedTrack();
}
+ this.videoHandler.updateSinks();
} catch (e) {
console.error('error in processing plugin(s)', e);
}
@@ -494,14 +517,38 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
await this.replaceSenderTrack(this.processedTrack || this.nativeTrack);
};
+ // eslint-disable-next-line complexity
private handleVisibilityChange = async () => {
- if (document.visibilityState === 'hidden' && this.source === 'regular') {
+ if (document.visibilityState === 'hidden') {
this.enabledStateBeforeBackground = this.enabled;
- this.nativeTrack.enabled = false;
- this.replaceSenderTrack(this.nativeTrack);
+ if (this.enabled) {
+ const track = await this.replaceTrackWithBlank();
+ await this.replaceSender(track, this.enabled);
+ this.nativeTrack?.stop();
+ this.nativeTrack = track;
+ } else {
+ await this.replaceSender(this.nativeTrack, false);
+ }
+ // started interruption event
+ this.eventBus.analytics.publish(
+ this.sendInterruptionEvent({
+ started: true,
+ }),
+ );
} else {
- this.nativeTrack.enabled = this.enabledStateBeforeBackground;
- this.replaceSenderTrack(this.processedTrack || this.nativeTrack);
+ HMSLogger.d(this.TAG, 'visibility visibile, restoring track state', this.enabledStateBeforeBackground);
+ if (this.enabledStateBeforeBackground) {
+ await this.setEnabled(true);
+ } else {
+ this.nativeTrack.enabled = this.enabledStateBeforeBackground;
+ await this.replaceSender(this.nativeTrack, this.enabledStateBeforeBackground);
+ }
+ // started interruption event
+ this.eventBus.analytics.publish(
+ this.sendInterruptionEvent({
+ started: false,
+ }),
+ );
}
this.eventBus.localVideoEnabled.publish({ enabled: this.nativeTrack.enabled, track: this });
};
diff --git a/packages/hms-video-store/src/media/tracks/HMSTrack.ts b/packages/hms-video-store/src/media/tracks/HMSTrack.ts
index b3c5574bb1..470a3eae88 100644
--- a/packages/hms-video-store/src/media/tracks/HMSTrack.ts
+++ b/packages/hms-video-store/src/media/tracks/HMSTrack.ts
@@ -1,4 +1,5 @@
import { HMSTrackType } from './HMSTrackType';
+import AnalyticsEventFactory from '../../analytics/AnalyticsEventFactory';
import { stringifyMediaStreamTrack } from '../../utils/json';
import HMSLogger from '../../utils/logger';
import { HMSMediaStream } from '../streams';
@@ -84,7 +85,16 @@ export abstract class HMSTrack {
protected setFirstTrackId(trackId: string) {
this.firstTrackId = trackId;
}
-
+ /**
+ * @internal
+ * It will send event to analytics when interruption start/stop
+ */
+ sendInterruptionEvent({ started, isRemoteAudio = false }: { started: boolean; isRemoteAudio?: boolean }) {
+ return AnalyticsEventFactory.interruption(started, isRemoteAudio ? 'remote.audio' : this.type, {
+ deviceId: this.nativeTrack.getSettings().deviceId,
+ groupId: this.nativeTrack.getSettings().groupId,
+ });
+ }
/**
* @internal
* take care of -
diff --git a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts
index 28616e29da..c5a287dde2 100644
--- a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts
+++ b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts
@@ -35,5 +35,6 @@ export enum HMSNotificationMethod {
POLL_STATS = 'on-poll-stats',
ROOM_INFO = 'room-info',
SESSION_INFO = 'session-info',
+ NODE_INFO = 'node-info',
WHITEBOARD_UPDATE = 'on-whiteboard-update',
}
diff --git a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts
index 9350179bc2..11fffbb254 100644
--- a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts
+++ b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts
@@ -393,3 +393,7 @@ export interface WhiteboardInfo {
state?: string;
attributes?: Array<{ name: string; value: unknown }>;
}
+
+export interface NodeInfo {
+ sfu_node_id: string;
+}
diff --git a/packages/hms-video-store/src/notification-manager/NotificationManager.ts b/packages/hms-video-store/src/notification-manager/NotificationManager.ts
index cb6922b20a..64fcf9c314 100644
--- a/packages/hms-video-store/src/notification-manager/NotificationManager.ts
+++ b/packages/hms-video-store/src/notification-manager/NotificationManager.ts
@@ -14,6 +14,7 @@ import { WhiteboardManager } from './managers/WhiteboardManager';
import { HMSNotificationMethod } from './HMSNotificationMethod';
import {
ConnectionQualityList,
+ NodeInfo,
OnTrackLayerUpdateNotification,
PolicyParams,
SpeakerList,
@@ -168,6 +169,10 @@ export class NotificationManager {
this.policyChangeManager.handlePolicyChange(notification as PolicyParams);
break;
+ case HMSNotificationMethod.NODE_INFO:
+ this.transport.setSFUNodeId((notification as NodeInfo).sfu_node_id);
+ break;
+
default:
break;
}
diff --git a/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts b/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts
index f84d7a2b5a..d3feeba285 100644
--- a/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts
+++ b/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts
@@ -56,12 +56,11 @@ export class TrackManager {
};
handleTrackRemovedPermanently = (notification: TrackStateNotification) => {
- HMSLogger.d(this.TAG, `ONTRACKREMOVE`, notification);
+ HMSLogger.d(this.TAG, `ONTRACKREMOVE permanently`, notification);
const trackIds = Object.keys(notification.tracks);
trackIds.forEach(trackId => {
const trackStateEntry = this.store.getTrackState(trackId);
-
if (!trackStateEntry) {
return;
}
diff --git a/packages/hms-video-store/src/notification-manager/notification-manager.test.ts b/packages/hms-video-store/src/notification-manager/notification-manager.test.ts
index b1344e1f8b..8b33228517 100644
--- a/packages/hms-video-store/src/notification-manager/notification-manager.test.ts
+++ b/packages/hms-video-store/src/notification-manager/notification-manager.test.ts
@@ -11,6 +11,7 @@ import HMSRoom from '../sdk/models/HMSRoom';
import { HMSRemotePeer } from '../sdk/models/peer';
import { Store } from '../sdk/store';
import HMSTransport from '../transport';
+import ITransportObserver from '../transport/ITransportObserver';
let joinHandler: jest.Mock;
let previewHandler: jest.Mock;
@@ -37,6 +38,9 @@ const store: Store = new Store();
let notificationManager: NotificationManager;
let eventBus: EventBus;
let transport: HMSTransport;
+let deviceManager: DeviceManager;
+let analyticsTimer: AnalyticsTimer;
+let observer: ITransportObserver;
beforeEach(() => {
joinHandler = jest.fn();
@@ -58,6 +62,16 @@ beforeEach(() => {
pollsUpdateHandler = jest.fn();
whiteboardUpdateHandler = jest.fn();
eventBus = new EventBus();
+ deviceManager = new DeviceManager(store, eventBus);
+ analyticsTimer = new AnalyticsTimer();
+ observer = {
+ onNotification: jest.fn(),
+ onTrackAdd: jest.fn(),
+ onTrackRemove: jest.fn(),
+ onFailure: jest.fn(),
+ onStateChange: jest.fn(),
+ onConnected: jest.fn(),
+ };
const mockMediaStream = {
id: 'native-stream-id',
getVideoTracks: jest.fn(() => [
@@ -83,19 +97,12 @@ beforeEach(() => {
global.HTMLCanvasElement.prototype.captureStream = jest.fn().mockImplementation(() => mockMediaStream);
transport = new HMSTransport(
- {
- onNotification: jest.fn(),
- onTrackAdd: jest.fn(),
- onTrackRemove: jest.fn(),
- onFailure: jest.fn(),
- onStateChange: jest.fn(),
- onConnected: jest.fn(),
- },
- new DeviceManager(store, eventBus),
+ observer,
+ deviceManager,
store,
eventBus,
new AnalyticsEventsService(store),
- new AnalyticsTimer(),
+ analyticsTimer,
new PluginUsageTracker(eventBus),
);
store.setRoom(new HMSRoom('1234'));
@@ -120,6 +127,8 @@ beforeEach(() => {
onWhiteboardUpdate: whiteboardUpdateHandler,
};
+ transport.setListener(listener);
+
audioListener = { onAudioLevelUpdate: audioUpdateHandler };
notificationManager = new NotificationManager(store, eventBus, transport, listener, audioListener);
diff --git a/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts b/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts
index 649d556e75..75100bb9a5 100644
--- a/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts
+++ b/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts
@@ -32,7 +32,7 @@ export class HMSAudioPluginsManager {
private readonly TAG = '[AudioPluginsManager]';
private readonly hmsTrack: HMSLocalAudioTrack;
// Map maintains the insertion order
- private readonly pluginsMap: Map;
+ readonly pluginsMap: Map;
private audioContext?: AudioContext;
private sourceNode?: MediaStreamAudioSourceNode;
diff --git a/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts b/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts
index d6d5669ecc..1991355757 100644
--- a/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts
+++ b/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts
@@ -8,7 +8,7 @@ import HMSLogger from '../../utils/logger';
export class HMSMediaStreamPluginsManager {
private readonly TAG = '[MediaStreamPluginsManager]';
private analytics: VideoPluginsAnalytics;
- private plugins: Set;
+ readonly plugins: Set;
private room?: Room;
constructor(eventBus: EventBus, room?: Room) {
diff --git a/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts b/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts
index c976dc610a..2a018cbfce 100644
--- a/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts
+++ b/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts
@@ -47,7 +47,7 @@ export class HMSVideoPluginsManager {
private pluginsLoopRunning = false;
private pluginsLoopState: 'paused' | 'running' = 'paused';
private readonly hmsTrack: HMSLocalVideoTrack;
- private readonly pluginsMap: Map; // plugin names to their instance mapping
+ readonly pluginsMap: Map; // plugin names to their instance mapping
private inputVideo?: HTMLVideoElement;
private inputCanvas?: CanvasElement;
private outputCanvas?: CanvasElement;
diff --git a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts
index 6f1d22e6f6..094ff6aac5 100644
--- a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts
+++ b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts
@@ -862,6 +862,7 @@ export class HMSSDKActions {
type: HMSPeerType.REGULAR,
});
testStore.addPeer(localPeer);
+ analyticsTimer = new AnalyticsTimer();
});
it('instantiates without any issues', () => {
@@ -243,7 +245,7 @@ describe('LocalTrackManager', () => {
testObserver,
new DeviceManager(testStore, testEventBus),
testEventBus,
- new AnalyticsTimer(),
+ analyticsTimer,
);
expect(manager).toBeDefined();
});
@@ -254,7 +256,7 @@ describe('LocalTrackManager', () => {
testObserver,
new DeviceManager(testStore, testEventBus),
testEventBus,
- new AnalyticsTimer(),
+ analyticsTimer,
);
testStore.setKnownRoles(policyParams);
await manager.getTracksToPublish({});
@@ -276,7 +278,7 @@ describe('LocalTrackManager', () => {
testObserver,
new DeviceManager(testStore, testEventBus),
testEventBus,
- new AnalyticsTimer(),
+ analyticsTimer,
);
global.navigator.mediaDevices.getUserMedia = mockDenyGetUserMedia as any;
testStore.setKnownRoles(policyParams);
@@ -436,7 +438,7 @@ describe('LocalTrackManager', () => {
testObserver,
new DeviceManager(testStore, testEventBus),
testEventBus,
- new AnalyticsTimer(),
+ analyticsTimer,
);
testStore.setKnownRoles(policyParams);
const tracksToPublish = await manager.getTracksToPublish({});
@@ -465,7 +467,7 @@ describe('LocalTrackManager', () => {
testObserver,
new DeviceManager(testStore, testEventBus),
testEventBus,
- new AnalyticsTimer(),
+ analyticsTimer,
);
testStore.setKnownRoles(policyParams);
const tracksToPublish = await manager.getTracksToPublish({});
diff --git a/packages/hms-video-store/src/sdk/index.ts b/packages/hms-video-store/src/sdk/index.ts
index c99eb87ef6..5db170b812 100644
--- a/packages/hms-video-store/src/sdk/index.ts
+++ b/packages/hms-video-store/src/sdk/index.ts
@@ -169,6 +169,7 @@ export class HMSSdk implements HMSInterface {
this.notificationManager?.setListener(this.listener);
this.audioSinkManager.setListener(this.listener);
this.interactivityCenter.setListener(this.listener);
+ this.transport.setListener(this.listener);
return;
}
@@ -428,16 +429,9 @@ export class HMSSdk implements HMSInterface {
resolve();
};
- const errorHandler = (ex?: HMSException) => {
- this.analyticsTimer.end(TimedEvent.PREVIEW);
- ex && this.errorListener?.onError(ex);
- this.sendPreviewAnalyticsEvent(ex);
- this.sdkState.isPreviewInProgress = false;
- reject(ex as HMSException);
- };
-
this.eventBus.policyChange.subscribeOnce(policyHandler);
- this.eventBus.leave.subscribeOnce(errorHandler);
+ this.eventBus.leave.subscribeOnce(this.handlePreviewError);
+ this.eventBus.leave.subscribeOnce(ex => reject(ex as HMSException));
this.transport
.preview(
@@ -457,10 +451,20 @@ export class HMSSdk implements HMSInterface {
});
}
})
- .catch(errorHandler);
+ .catch(ex => {
+ this.handlePreviewError(ex);
+ reject(ex);
+ });
});
}
+ private handlePreviewError = (ex?: HMSException) => {
+ this.analyticsTimer.end(TimedEvent.PREVIEW);
+ ex && this.errorListener?.onError(ex);
+ this.sendPreviewAnalyticsEvent(ex);
+ this.sdkState.isPreviewInProgress = false;
+ };
+
private async midCallPreview(asRole?: string, settings?: InitialSettings): Promise {
if (!this.localPeer || this.transportState !== TransportState.Joined) {
throw ErrorFactory.GenericErrors.NotConnected(HMSAction.VALIDATION, 'Not connected - midCallPreview');
@@ -538,6 +542,8 @@ export class HMSSdk implements HMSInterface {
throw ErrorFactory.GenericErrors.NotReady(HMSAction.JOIN, "Preview is in progress, can't join");
}
+ // remove terminal error handling from preview(do not send preview.failed after join on disconnect)
+ this.eventBus?.leave?.unsubscribe(this.handlePreviewError);
this.analyticsTimer.start(TimedEvent.JOIN);
this.sdkState.isJoinInProgress = true;
@@ -838,6 +844,9 @@ export class HMSSdk implements HMSInterface {
});
return;
}
+ this.transport.setOnScreenshareStop(() => {
+ this.stopEndedScreenshare(onStop);
+ });
await this.transport.publish(tracks);
tracks.forEach(track => {
track.peerId = this.localPeer?.peerId;
diff --git a/packages/hms-video-store/src/sdk/models/HMSRoom.ts b/packages/hms-video-store/src/sdk/models/HMSRoom.ts
index c0aca24bdf..3438b97867 100644
--- a/packages/hms-video-store/src/sdk/models/HMSRoom.ts
+++ b/packages/hms-video-store/src/sdk/models/HMSRoom.ts
@@ -15,13 +15,8 @@ export default class Room implements HMSRoom {
max_size?: number;
large_room_optimization?: boolean;
transcriptions?: HMSTranscriptionInfo[] = [];
- /**
- * @alpha
- */
isEffectsEnabled?: boolean;
- /**
- * @alpha
- */
+ isVBEnabled?: boolean;
effectsKey?: string;
isHipaaEnabled?: boolean;
isNoiseCancellationEnabled?: boolean;
diff --git a/packages/hms-video-store/src/sdk/store/Store.ts b/packages/hms-video-store/src/sdk/store/Store.ts
index 320310813c..c08e55bbe8 100644
--- a/packages/hms-video-store/src/sdk/store/Store.ts
+++ b/packages/hms-video-store/src/sdk/store/Store.ts
@@ -75,6 +75,15 @@ class Store {
this.simulcastEnabled = enabled;
}
+ removeRemoteTracks() {
+ this.tracks.forEach(track => {
+ if (track instanceof HMSRemoteAudioTrack || track instanceof HMSRemoteVideoTrack) {
+ this.removeTrack(track);
+ delete this.peerTrackStates[track.trackId];
+ }
+ });
+ }
+
getEnv() {
return this.env;
}
@@ -280,6 +289,10 @@ class Store {
this.peerTrackStates[trackStateEntry.trackInfo.track_id] = trackStateEntry;
}
+ removeTrackState(trackId: string) {
+ delete this.peerTrackStates[trackId];
+ }
+
removePeer(peerId: string) {
if (this.localPeerId === peerId) {
this.localPeerId = undefined;
diff --git a/packages/hms-video-store/src/selectors/selectors.ts b/packages/hms-video-store/src/selectors/selectors.ts
index 0bbac939d3..7de9ef82c9 100644
--- a/packages/hms-video-store/src/selectors/selectors.ts
+++ b/packages/hms-video-store/src/selectors/selectors.ts
@@ -457,6 +457,7 @@ export const selectSessionId = createSelector(selectRoom, room => room.sessionId
export const selectRoomStartTime = createSelector(selectRoom, room => room.startedAt);
export const selectIsLargeRoom = createSelector(selectRoom, room => !!room.isLargeRoom);
export const selectIsEffectsEnabled = createSelector(selectRoom, room => !!room.isEffectsEnabled);
+export const selectIsVBEnabled = createSelector(selectRoom, room => !!room.isVBEnabled);
export const selectEffectsKey = createSelector(selectRoom, room => room.effectsKey);
export const selectTemplateAppData = (store: HMSStore) => store.templateAppData;
diff --git a/packages/hms-video-store/src/signal/init/models.ts b/packages/hms-video-store/src/signal/init/models.ts
index 7467ab7812..a4a8ca7db5 100644
--- a/packages/hms-video-store/src/signal/init/models.ts
+++ b/packages/hms-video-store/src/signal/init/models.ts
@@ -58,6 +58,7 @@ export enum InitFlags {
FLAG_DISABLE_VIDEO_TRACK_AUTO_UNSUBSCRIBE = 'disableVideoTrackAutoUnsubscribe',
FLAG_WHITEBOARD_ENABLED = 'whiteboardEnabled',
FLAG_EFFECTS_SDK_ENABLED = 'effectsSDKEnabled',
+ FLAG_VB_ENABLED = 'vb',
FLAG_HIPAA_ENABLED = 'hipaa',
FLAG_NOISE_CANCELLATION = 'noiseCancellation',
FLAG_SCALE_SCREENSHARE_BASED_ON_PIXELS = 'scaleScreenshareBasedOnPixels',
diff --git a/packages/hms-video-store/src/signal/jsonrpc/index.ts b/packages/hms-video-store/src/signal/jsonrpc/index.ts
index e00fea019d..51e357ce56 100644
--- a/packages/hms-video-store/src/signal/jsonrpc/index.ts
+++ b/packages/hms-video-store/src/signal/jsonrpc/index.ts
@@ -91,6 +91,7 @@ export default class JsonRpcSignal {
private _isConnected = false;
private id = 0;
+ private sfuNodeId: string | undefined;
private onCloseHandler: (event: CloseEvent) => void = () => {};
@@ -98,6 +99,10 @@ export default class JsonRpcSignal {
return this._isConnected;
}
+ setSfuNodeId(sfuNodeId?: string) {
+ this.sfuNodeId = sfuNodeId;
+ }
+
public setIsConnected(newValue: boolean, reason = '') {
HMSLogger.d(this.TAG, `isConnected set id: ${this.id}, oldValue: ${this._isConnected}, newValue: ${newValue}`);
if (this._isConnected === newValue) {
@@ -244,7 +249,7 @@ export default class JsonRpcSignal {
simulcast: boolean,
onDemandTracks: boolean,
offer?: RTCSessionDescriptionInit,
- ): Promise {
+ ): Promise {
if (!this.isConnected) {
throw ErrorFactory.WebSocketConnectionErrors.WebSocketConnectionLost(
HMSAction.JOIN,
@@ -260,7 +265,10 @@ export default class JsonRpcSignal {
simulcast,
onDemandTracks,
};
- const response: RTCSessionDescriptionInit = await this.internalCall(HMSSignalMethod.JOIN, params);
+ const response: RTCSessionDescriptionInit & { sfu_node_id: string | undefined } = await this.internalCall(
+ HMSSignalMethod.JOIN,
+ params,
+ );
this.isJoinCompleted = true;
this.pendingTrickle.forEach(({ target, candidate }) => this.trickle(target, candidate));
@@ -272,7 +280,7 @@ export default class JsonRpcSignal {
trickle(target: HMSConnectionRole, candidate: RTCIceCandidateInit) {
if (this.isJoinCompleted) {
- this.notify(HMSSignalMethod.TRICKLE, { target, candidate });
+ this.notify(HMSSignalMethod.TRICKLE, { target, candidate, sfu_node_id: this.sfuNodeId });
} else {
this.pendingTrickle.push({ target, candidate });
}
@@ -282,12 +290,13 @@ export default class JsonRpcSignal {
const response = await this.call(HMSSignalMethod.OFFER, {
desc,
tracks: Object.fromEntries(tracks),
+ sfu_node_id: this.sfuNodeId,
});
return response as RTCSessionDescriptionInit;
}
answer(desc: RTCSessionDescriptionInit) {
- this.notify(HMSSignalMethod.ANSWER, { desc });
+ this.notify(HMSSignalMethod.ANSWER, { desc, sfu_node_id: this.sfuNodeId });
}
trackUpdate(tracks: Map) {
diff --git a/packages/hms-video-store/src/transport/index.ts b/packages/hms-video-store/src/transport/index.ts
index 54cc1848bc..6591b48499 100644
--- a/packages/hms-video-store/src/transport/index.ts
+++ b/packages/hms-video-store/src/transport/index.ts
@@ -24,7 +24,7 @@ import { ErrorFactory } from '../error/ErrorFactory';
import { HMSAction } from '../error/HMSAction';
import { HMSException } from '../error/HMSException';
import { EventBus } from '../events/EventBus';
-import { HMSICEServer, HMSRole } from '../interfaces';
+import { HMSICEServer, HMSRole, HMSUpdateListener } from '../interfaces';
import { HMSLocalStream } from '../media/streams/HMSLocalStream';
import { HMSLocalTrack, HMSLocalVideoTrack, HMSTrack } from '../media/tracks';
import { TrackState } from '../notification-manager';
@@ -80,7 +80,11 @@ export default class HMSTransport {
private subscribeStatsAnalytics?: SubscribeStatsAnalytics;
private maxSubscribeBitrate = 0;
private connectivityListener?: HMSDiagnosticsConnectivityListener;
+ private sfuNodeId?: string;
joinRetryCount = 0;
+ private publishDisconnectTimer = 0;
+ private listener?: HMSUpdateListener;
+ private onScreenshareStop = () => {};
constructor(
private observer: ITransportObserver,
@@ -122,12 +126,24 @@ export default class HMSTransport {
*/
private readonly callbacks = new Map();
+ setListener = (listener: HMSUpdateListener) => {
+ this.listener = listener;
+ };
+
+ setOnScreenshareStop = (onStop: () => void) => {
+ this.onScreenshareStop = onStop;
+ };
+
private signalObserver: ISignalEventsObserver = {
- onOffer: async (jsep: RTCSessionDescriptionInit) => {
+ onOffer: async (jsep: RTCSessionDescriptionInit & { sfu_node_id: string }) => {
try {
if (!this.subscribeConnection) {
return;
}
+ if (this.sfuNodeId !== jsep.sfu_node_id) {
+ HMSLogger.d(TAG, 'ignoring old offer');
+ return;
+ }
await this.subscribeConnection.setRemoteDescription(jsep);
HMSLogger.d(
TAG,
@@ -149,7 +165,7 @@ export default class HMSTransport {
if (err instanceof HMSException) {
ex = err;
} else {
- ex = ErrorFactory.GenericErrors.Unknown(HMSAction.PUBLISH, (err as Error).message);
+ ex = ErrorFactory.GenericErrors.Unknown(HMSAction.SUBSCRIBE, (err as Error).message);
}
this.observer.onFailure(ex);
this.eventBus.analytics.publish(AnalyticsEventFactory.subscribeFail(ex));
@@ -218,189 +234,6 @@ export default class HMSTransport {
private publishDtlsStateTimer = 0;
private lastPublishDtlsState: RTCDtlsTransportState = 'new';
- private publishConnectionObserver: IPublishConnectionObserver = {
- onRenegotiationNeeded: async () => {
- await this.performPublishRenegotiation();
- },
-
- // eslint-disable-next-line complexity
- onDTLSTransportStateChange: (state?: RTCDtlsTransportState) => {
- const log = state === 'failed' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
- log(TAG, `Publisher on dtls transport state change: ${state}`);
-
- if (!state || this.lastPublishDtlsState === state) {
- return;
- }
-
- this.lastPublishDtlsState = state;
- if (this.publishDtlsStateTimer !== 0) {
- clearTimeout(this.publishDtlsStateTimer);
- this.publishDtlsStateTimer = 0;
- }
-
- if (state !== 'connecting' && state !== 'failed') {
- return;
- }
-
- const timeout = this.initConfig?.config?.dtlsStateTimeouts?.[state];
- if (!timeout || timeout <= 0) {
- return;
- }
-
- // if we're in connecting check again after timeout
- // hotfix: mitigate https://100ms.atlassian.net/browse/LIVE-1924
- this.publishDtlsStateTimer = window.setTimeout(() => {
- const newState = this.publishConnection?.nativeConnection.connectionState;
- if (newState && state && newState === state) {
- // stuck in either `connecting` or `failed` state for long time
- const err = ErrorFactory.WebrtcErrors.ICEFailure(
- HMSAction.PUBLISH,
- `DTLS transport state ${state} timeout:${timeout}ms`,
- true,
- );
- this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(err));
- this.observer.onFailure(err);
- }
- }, timeout);
- },
-
- onDTLSTransportError: (error: Error) => {
- HMSLogger.e(TAG, `onDTLSTransportError ${error.name} ${error.message}`, error);
- this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(error));
- },
-
- onIceConnectionChange: async (newState: RTCIceConnectionState) => {
- const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
- log(TAG, `Publish ice connection state change: ${newState}`);
-
- // @TODO: Uncomment this and remove connectionstatechange
- if (newState === 'failed') {
- // await this.handleIceConnectionFailure(HMSConnectionRole.Publish);
- }
- },
-
- // @TODO(eswar): Remove this. Use iceconnectionstate change with interval and threshold.
- onConnectionStateChange: async (newState: RTCPeerConnectionState) => {
- const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
- log(TAG, `Publish connection state change: ${newState}`);
-
- if (newState === 'connected') {
- this.connectivityListener?.onICESuccess(true);
- this.publishConnection?.handleSelectedIceCandidatePairs();
- }
-
- if (newState === 'disconnected') {
- // if state stays disconnected for 5 seconds, retry
- setTimeout(() => {
- if (this.publishConnection?.connectionState === 'disconnected') {
- this.handleIceConnectionFailure(
- HMSConnectionRole.Publish,
- ErrorFactory.WebrtcErrors.ICEDisconnected(
- HMSAction.PUBLISH,
- `local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`,
- ),
- );
- }
- }, ICE_DISCONNECTION_TIMEOUT);
- }
-
- if (newState === 'failed') {
- await this.handleIceConnectionFailure(
- HMSConnectionRole.Publish,
- ErrorFactory.WebrtcErrors.ICEFailure(
- HMSAction.PUBLISH,
- `local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`,
- ),
- );
- }
- },
-
- onIceCandidate: candidate => {
- this.connectivityListener?.onICECandidate(candidate, true);
- },
-
- onSelectedCandidatePairChange: candidatePair => {
- this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, true);
- },
- };
-
- private subscribeConnectionObserver: ISubscribeConnectionObserver = {
- onApiChannelMessage: (message: string) => {
- this.observer.onNotification(JSON.parse(message));
- },
-
- onTrackAdd: (track: HMSTrack) => {
- HMSLogger.d(TAG, '[Subscribe] onTrackAdd', `${track}`);
- this.observer.onTrackAdd(track);
- },
-
- onTrackRemove: (track: HMSTrack) => {
- HMSLogger.d(TAG, '[Subscribe] onTrackRemove', `${track}`);
- this.observer.onTrackRemove(track);
- },
-
- onIceConnectionChange: async (newState: RTCIceConnectionState) => {
- const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
- log(TAG, `Subscribe ice connection state change: ${newState}`);
-
- if (newState === 'failed') {
- // await this.handleIceConnectionFailure(HMSConnectionRole.Subscribe);
- }
-
- if (newState === 'connected') {
- const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
- this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
-
- this.connectivityListener?.onICESuccess(false);
- if (callback) {
- callback.promise.resolve(true);
- }
- }
- },
-
- // @TODO(eswar): Remove this. Use iceconnectionstate change with interval and threshold.
- onConnectionStateChange: async (newState: RTCPeerConnectionState) => {
- const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
- log(TAG, `Subscribe connection state change: ${newState}`);
-
- if (newState === 'failed') {
- await this.handleIceConnectionFailure(
- HMSConnectionRole.Subscribe,
- ErrorFactory.WebrtcErrors.ICEFailure(
- HMSAction.SUBSCRIBE,
- `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`,
- ),
- );
- }
-
- if (newState === 'disconnected') {
- setTimeout(() => {
- if (this.subscribeConnection?.connectionState === 'disconnected') {
- this.handleIceConnectionFailure(
- HMSConnectionRole.Subscribe,
- ErrorFactory.WebrtcErrors.ICEDisconnected(
- HMSAction.SUBSCRIBE,
- `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`,
- ),
- );
- }
- }, ICE_DISCONNECTION_TIMEOUT);
- }
-
- if (newState === 'connected') {
- this.handleSubscribeConnectionConnected();
- }
- },
-
- onIceCandidate: candidate => {
- this.connectivityListener?.onICECandidate(candidate, false);
- },
-
- onSelectedCandidatePairChange: candidatePair => {
- this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, false);
- },
- };
-
getWebrtcInternals() {
return this.webrtcInternals;
}
@@ -533,8 +366,7 @@ export default class HMSTransport {
this.publishStatsAnalytics?.stop();
this.subscribeStatsAnalytics?.stop();
this.webrtcInternals?.cleanup();
- await this.publishConnection?.close();
- await this.subscribeConnection?.close();
+ this.clearPeerConnections();
if (notifyServer) {
try {
this.signal.leave();
@@ -591,6 +423,90 @@ export default class HMSTransport {
}
}
+ setSFUNodeId(id?: string) {
+ this.signal.setSfuNodeId(id);
+ if (!this.sfuNodeId) {
+ this.sfuNodeId = id;
+ this.publishConnection?.setSfuNodeId(id);
+ this.subscribeConnection?.setSfuNodeId(id);
+ } else if (this.sfuNodeId !== id) {
+ this.sfuNodeId = id;
+ this.handleSFUMigration();
+ }
+ }
+
+ // eslint-disable-next-line complexity
+ async handleSFUMigration() {
+ HMSLogger.time('sfu migration');
+ this.clearPeerConnections();
+ const peers = this.store.getPeerMap();
+ this.store.removeRemoteTracks();
+ for (const peerId in peers) {
+ const peer = peers[peerId];
+ if (peer.isLocal) {
+ continue;
+ }
+ peer.audioTrack = undefined;
+ peer.videoTrack = undefined;
+ peer.auxiliaryTracks = [];
+ }
+
+ const localPeer = this.store.getLocalPeer();
+ if (!localPeer) {
+ return;
+ }
+ this.createPeerConnections();
+ this.trackStates.clear();
+ await this.negotiateOnFirstPublish();
+ const streamMap = new Map();
+ if (localPeer.audioTrack) {
+ const stream = localPeer.audioTrack.stream as HMSLocalStream;
+ if (!streamMap.get(stream.id)) {
+ streamMap.set(stream.id, new HMSLocalStream(new MediaStream()));
+ }
+ const newTrack = localPeer.audioTrack.clone(streamMap.get(stream.id)!);
+ this.store.removeTrack(localPeer.audioTrack);
+ localPeer.audioTrack.cleanup();
+ await this.publishTrack(newTrack);
+ localPeer.audioTrack = newTrack;
+ }
+
+ if (localPeer.videoTrack) {
+ const stream = localPeer.videoTrack.stream as HMSLocalStream;
+ if (!streamMap.get(stream.id)) {
+ streamMap.set(stream.id, new HMSLocalStream(new MediaStream()));
+ }
+ this.store.removeTrack(localPeer.videoTrack);
+ const newTrack = localPeer.videoTrack.clone(streamMap.get(stream.id)!);
+ localPeer.videoTrack.cleanup();
+ await this.publishTrack(newTrack);
+ localPeer.videoTrack = newTrack;
+ }
+
+ const auxTracks = [];
+ while (localPeer.auxiliaryTracks.length > 0) {
+ const track = localPeer.auxiliaryTracks.shift();
+ if (track) {
+ const stream = track.stream as HMSLocalStream;
+ if (!streamMap.get(stream.id)) {
+ streamMap.set(stream.id, new HMSLocalStream(new MediaStream()));
+ }
+ this.store.removeTrack(track);
+ const newTrack = track.clone(streamMap.get(stream.id)!);
+ if (newTrack.type === 'video' && newTrack.source === 'screen') {
+ newTrack.nativeTrack.addEventListener('ended', this.onScreenshareStop);
+ }
+ track.cleanup();
+ await this.publishTrack(newTrack);
+ auxTracks.push(newTrack);
+ }
+ }
+ localPeer.auxiliaryTracks = auxTracks;
+ streamMap.clear();
+ this.listener?.onSFUMigration?.();
+ HMSLogger.timeEnd('sfu migration');
+ }
+
/**
* TODO: check if track.publishedTrackId be used instead of the hack to match with track with same type and
* source. The hack won't work if there are multiple tracks with same source and type.
@@ -685,6 +601,18 @@ export default class HMSTransport {
HMSLogger.d(TAG, `✅ unpublishTrack: trackId=${track.trackId}`, this.callbacks);
}
+ private async clearPeerConnections() {
+ clearTimeout(this.publishDtlsStateTimer);
+ this.publishDtlsStateTimer = 0;
+ clearTimeout(this.publishDisconnectTimer);
+ this.publishDisconnectTimer = 0;
+ this.lastPublishDtlsState = 'new';
+ this.publishConnection?.close();
+ this.subscribeConnection?.close();
+ this.publishConnection = null;
+ this.subscribeConnection = null;
+ }
+
private waitForLocalRoleAvailability() {
if (this.store.hasRoleDetailsArrived()) {
return;
@@ -716,11 +644,186 @@ export default class HMSTransport {
private createPeerConnections() {
if (this.initConfig) {
+ const publishConnectionObserver: IPublishConnectionObserver = {
+ onRenegotiationNeeded: async () => {
+ await this.performPublishRenegotiation();
+ },
+
+ // eslint-disable-next-line complexity
+ onDTLSTransportStateChange: (state?: RTCDtlsTransportState) => {
+ const log = state === 'failed' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
+ log(TAG, `Publisher on dtls transport state change: ${state}`);
+
+ if (!state || this.lastPublishDtlsState === state) {
+ return;
+ }
+
+ this.lastPublishDtlsState = state;
+ if (this.publishDtlsStateTimer !== 0) {
+ clearTimeout(this.publishDtlsStateTimer);
+ this.publishDtlsStateTimer = 0;
+ }
+
+ if (state !== 'connecting' && state !== 'failed') {
+ return;
+ }
+
+ const timeout = this.initConfig?.config?.dtlsStateTimeouts?.[state];
+ if (!timeout || timeout <= 0) {
+ return;
+ }
+
+ // if we're in connecting check again after timeout
+ // hotfix: mitigate https://100ms.atlassian.net/browse/LIVE-1924
+ this.publishDtlsStateTimer = window.setTimeout(() => {
+ const newState = this.publishConnection?.nativeConnection.connectionState;
+ if (newState && state && newState === state) {
+ // stuck in either `connecting` or `failed` state for long time
+ const err = ErrorFactory.WebrtcErrors.ICEFailure(
+ HMSAction.PUBLISH,
+ `DTLS transport state ${state} timeout:${timeout}ms`,
+ true,
+ );
+ this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(err));
+ this.observer.onFailure(err);
+ }
+ }, timeout);
+ },
+
+ onDTLSTransportError: (error: Error) => {
+ HMSLogger.e(TAG, `onDTLSTransportError ${error.name} ${error.message}`, error);
+ this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(error));
+ },
+
+ onIceConnectionChange: async (newState: RTCIceConnectionState) => {
+ const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
+ log(TAG, `Publish ice connection state change: ${newState}`);
+ },
+
+ onConnectionStateChange: async (newState: RTCPeerConnectionState) => {
+ const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
+ log(TAG, `Publish connection state change: ${newState}`);
+ if (newState === 'new') {
+ return;
+ }
+
+ if (newState === 'connected') {
+ this.connectivityListener?.onICESuccess(true);
+ this.publishConnection?.handleSelectedIceCandidatePairs();
+ } else if (newState === 'failed') {
+ await this.handleIceConnectionFailure(
+ HMSConnectionRole.Publish,
+ ErrorFactory.WebrtcErrors.ICEFailure(
+ HMSAction.PUBLISH,
+ `local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`,
+ ),
+ );
+ } else {
+ this.publishDisconnectTimer = window.setTimeout(() => {
+ if (this.publishConnection?.connectionState !== 'connected') {
+ this.handleIceConnectionFailure(
+ HMSConnectionRole.Publish,
+ ErrorFactory.WebrtcErrors.ICEDisconnected(
+ HMSAction.PUBLISH,
+ `local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`,
+ ),
+ );
+ }
+ }, ICE_DISCONNECTION_TIMEOUT);
+ }
+ },
+
+ onIceCandidate: candidate => {
+ this.connectivityListener?.onICECandidate(candidate, true);
+ },
+
+ onSelectedCandidatePairChange: candidatePair => {
+ this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, true);
+ },
+ };
+
+ const subscribeConnectionObserver: ISubscribeConnectionObserver = {
+ onApiChannelMessage: (message: string) => {
+ this.observer.onNotification(JSON.parse(message));
+ },
+
+ onTrackAdd: (track: HMSTrack) => {
+ HMSLogger.d(TAG, '[Subscribe] onTrackAdd', `${track}`);
+ this.observer.onTrackAdd(track);
+ },
+
+ onTrackRemove: (track: HMSTrack) => {
+ HMSLogger.d(TAG, '[Subscribe] onTrackRemove', `${track}`);
+ this.observer.onTrackRemove(track);
+ },
+
+ onIceConnectionChange: async (newState: RTCIceConnectionState) => {
+ const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
+ log(TAG, `Subscribe ice connection state change: ${newState}`);
+
+ // if (newState === 'failed') {
+ // // await this.handleIceConnectionFailure(HMSConnectionRole.Subscribe);
+ // }
+
+ if (newState === 'connected') {
+ const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
+ this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
+
+ this.connectivityListener?.onICESuccess(false);
+ if (callback) {
+ callback.promise.resolve(true);
+ }
+ }
+ },
+
+ onConnectionStateChange: async (newState: RTCPeerConnectionState) => {
+ const log = newState === 'disconnected' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
+ log(TAG, `Subscribe connection state change: ${newState}`);
+
+ if (newState === 'failed') {
+ await this.handleIceConnectionFailure(
+ HMSConnectionRole.Subscribe,
+ ErrorFactory.WebrtcErrors.ICEFailure(
+ HMSAction.SUBSCRIBE,
+ `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`,
+ ),
+ );
+ } else if (newState === 'disconnected') {
+ setTimeout(() => {
+ if (this.subscribeConnection?.connectionState === 'disconnected') {
+ this.handleIceConnectionFailure(
+ HMSConnectionRole.Subscribe,
+ ErrorFactory.WebrtcErrors.ICEDisconnected(
+ HMSAction.SUBSCRIBE,
+ `local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`,
+ ),
+ );
+ }
+ }, ICE_DISCONNECTION_TIMEOUT);
+ } else if (newState === 'connected') {
+ this.subscribeConnection?.handleSelectedIceCandidatePairs();
+ const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
+ this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
+
+ if (callback) {
+ callback.promise.resolve(true);
+ }
+ }
+ },
+
+ onIceCandidate: candidate => {
+ this.connectivityListener?.onICECandidate(candidate, false);
+ },
+
+ onSelectedCandidatePairChange: candidatePair => {
+ this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, false);
+ },
+ };
if (!this.publishConnection) {
this.publishConnection = new HMSPublishConnection(
this.signal,
this.initConfig.rtcConfiguration,
- this.publishConnectionObserver,
+ publishConnectionObserver,
);
}
@@ -729,7 +832,7 @@ export default class HMSTransport {
this.signal,
this.initConfig.rtcConfiguration,
this.isFlagEnabled.bind(this),
- this.subscribeConnectionObserver,
+ subscribeConnectionObserver,
);
}
}
@@ -816,6 +919,7 @@ export default class HMSTransport {
onDemandTracks,
offer,
);
+ this.setSFUNodeId(answer?.sfu_node_id);
await this.publishConnection.setRemoteDescription(answer);
for (const candidate of this.publishConnection.candidates) {
await this.publishConnection.addIceCandidate(candidate);
@@ -838,6 +942,7 @@ export default class HMSTransport {
simulcast,
onDemandTracks,
);
+ this.setSFUNodeId(response?.sfu_node_id);
return !!response;
}
@@ -850,16 +955,24 @@ export default class HMSTransport {
HMSLogger.e(TAG, 'Publish peer connection not found, cannot negotiate');
return false;
}
- const offer = await this.publishConnection.createOffer(this.trackStates);
- await this.publishConnection.setLocalDescription(offer);
- const answer = await this.signal.offer(offer, this.trackStates);
- await this.publishConnection.setRemoteDescription(answer);
- for (const candidate of this.publishConnection.candidates) {
- await this.publishConnection.addIceCandidate(candidate);
- }
+ try {
+ const offer = await this.publishConnection.createOffer(this.trackStates);
+ await this.publishConnection.setLocalDescription(offer);
+ const answer = await this.signal.offer(offer, this.trackStates);
+ await this.publishConnection.setRemoteDescription(answer);
+ for (const candidate of this.publishConnection.candidates) {
+ await this.publishConnection.addIceCandidate(candidate);
+ }
- this.publishConnection.initAfterJoin();
- return !!answer;
+ this.publishConnection.initAfterJoin();
+ return !!answer;
+ } catch (ex) {
+ // resolve for now as this might happen during migration
+ if (ex instanceof HMSException && ex.code === 421) {
+ return true;
+ }
+ throw ex;
+ }
}
private async performPublishRenegotiation(constraints?: RTCOfferOptions) {
@@ -892,7 +1005,12 @@ export default class HMSTransport {
ex = ErrorFactory.GenericErrors.Unknown(HMSAction.PUBLISH, (err as Error).message);
}
- callback!.promise.reject(ex);
+ // resolve for now as this might happen during migration
+ if (ex.code === 421) {
+ callback!.promise.resolve(true);
+ } else {
+ callback!.promise.reject(ex);
+ }
HMSLogger.d(TAG, `[role=PUBLISH] onRenegotiationNeeded FAILED ❌`);
}
}
@@ -944,6 +1062,7 @@ export default class HMSTransport {
if (room) {
room.effectsKey = this.initConfig.config.vb?.effectsKey;
room.isEffectsEnabled = this.isFlagEnabled(InitFlags.FLAG_EFFECTS_SDK_ENABLED);
+ room.isVBEnabled = this.isFlagEnabled(InitFlags.FLAG_VB_ENABLED);
room.isHipaaEnabled = this.isFlagEnabled(InitFlags.FLAG_HIPAA_ENABLED);
room.isNoiseCancellationEnabled = this.isFlagEnabled(InitFlags.FLAG_NOISE_CANCELLATION);
}
@@ -1081,15 +1200,7 @@ export default class HMSTransport {
* Do iceRestart only if not connected
*/
if (this.publishConnection) {
- const p = new Promise((resolve, reject) => {
- this.callbacks.set(RENEGOTIATION_CALLBACK_ID, {
- promise: { resolve, reject },
- action: HMSAction.RESTART_ICE,
- extra: {},
- });
- });
await this.performPublishRenegotiation({ iceRestart: this.publishConnection.connectionState !== 'connected' });
- await p;
}
return true;
@@ -1139,16 +1250,6 @@ export default class HMSTransport {
return ok;
};
- private handleSubscribeConnectionConnected() {
- this.subscribeConnection?.handleSelectedIceCandidatePairs();
- const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
- this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
-
- if (callback) {
- callback.promise.resolve(true);
- }
- }
-
private setTransportStateForConnect() {
if (this.state === TransportState.Failed) {
this.state = TransportState.Disconnected;
diff --git a/packages/hms-video-store/src/utils/constants.ts b/packages/hms-video-store/src/utils/constants.ts
index 5372663c7b..b31ecbfc32 100644
--- a/packages/hms-video-store/src/utils/constants.ts
+++ b/packages/hms-video-store/src/utils/constants.ts
@@ -59,7 +59,7 @@ export const HMSEvents = {
export const PROTOCOL_VERSION = '2.5';
-export const PROTOCOL_SPEC = '20240521';
+export const PROTOCOL_SPEC = '20240720';
export const HAND_RAISE_GROUP_NAME = '_handraise';
diff --git a/packages/hms-virtual-background/src/HMSVBPlugin.ts b/packages/hms-virtual-background/src/HMSVBPlugin.ts
index b94ce59494..92deea4aaa 100644
--- a/packages/hms-virtual-background/src/HMSVBPlugin.ts
+++ b/packages/hms-virtual-background/src/HMSVBPlugin.ts
@@ -42,9 +42,13 @@ export class HMSVBPlugin implements HMSVideoPlugin {
return this.checkSupport().isSupported;
}
+ isBlurSupported(): boolean {
+ return 'filter' in CanvasRenderingContext2D.prototype;
+ }
+
checkSupport(): HMSPluginSupportResult {
const browserResult = {} as HMSPluginSupportResult;
- if (['Chrome', 'Firefox', 'Edg', 'Edge'].some(value => navigator.userAgent.indexOf(value) !== -1)) {
+ if (['Chrome', 'Firefox', 'Edg', 'Edge', 'Safari'].some(value => navigator.userAgent.indexOf(value) !== -1)) {
browserResult.isSupported = true;
} else {
browserResult.isSupported = false;
diff --git a/packages/roomkit-react/package.json b/packages/roomkit-react/package.json
index 01cc71ef1a..e988cacc32 100644
--- a/packages/roomkit-react/package.json
+++ b/packages/roomkit-react/package.json
@@ -81,7 +81,7 @@
"@100mslive/hms-whiteboard": "0.0.6",
"@100mslive/react-icons": "0.10.16",
"@100mslive/react-sdk": "0.10.16",
- "@100mslive/types-prebuilt": "0.12.9",
+ "@100mslive/types-prebuilt": "0.12.11",
"@emoji-mart/data": "^1.0.6",
"@emoji-mart/react": "^1.0.1",
"@radix-ui/react-accordion": "1.0.0",
diff --git a/packages/roomkit-react/src/Prebuilt/components/AppData/AppData.tsx b/packages/roomkit-react/src/Prebuilt/components/AppData/AppData.tsx
index cb8cda955c..0b6e006088 100644
--- a/packages/roomkit-react/src/Prebuilt/components/AppData/AppData.tsx
+++ b/packages/roomkit-react/src/Prebuilt/components/AppData/AppData.tsx
@@ -1,4 +1,5 @@
import React, { useEffect, useMemo, useRef } from 'react';
+import { useMedia } from 'react-use';
import {
HMSRoomState,
selectFullAppData,
@@ -10,6 +11,7 @@ import {
useHMSStore,
useRecordingStreaming,
} from '@100mslive/react-sdk';
+import { config as cssConfig } from '../../../Theme';
import { LayoutMode } from '../Settings/LayoutSettings';
import { useRoomLayoutConferencingScreen } from '../../provider/roomLayoutProvider/hooks/useRoomLayoutScreen';
//@ts-ignore
@@ -26,6 +28,7 @@ import {
UI_MODE_GRID,
UI_SETTINGS,
} from '../../common/constants';
+import { DEFAULT_TILES_IN_VIEW } from '../MoreSettings/constants';
const initialAppData = {
[APP_DATA.uiSettings]: {
@@ -80,6 +83,7 @@ export const AppData = React.memo(() => {
const { isLocalVideoEnabled } = useAVToggle();
const sidepaneOpenedRef = useRef(false);
const [, setNoiseCancellationEnabled] = useSetNoiseCancellation();
+ const isMobile = useMedia(cssConfig.media.md);
useEffect(() => {
if (elements?.noise_cancellation?.enabled_by_default) {
@@ -117,9 +121,12 @@ export const AppData = React.memo(() => {
...uiSettings,
[UI_SETTINGS.isAudioOnly]: undefined,
[UI_SETTINGS.uiViewMode]: uiSettings.uiViewMode || UI_MODE_GRID,
+ [UI_SETTINGS.maxTileCount]: isMobile
+ ? DEFAULT_TILES_IN_VIEW.MWEB
+ : Number(elements?.video_tile_layout?.grid?.tiles_in_view) || DEFAULT_TILES_IN_VIEW.DESKTOP,
};
hmsActions.setAppData(APP_DATA.uiSettings, updatedSettings, true);
- }, [preferences, hmsActions]);
+ }, [preferences, hmsActions, elements?.video_tile_layout, isMobile]);
useEffect(() => {
if (!preferences.subscribedNotifications) {
diff --git a/packages/roomkit-react/src/Prebuilt/components/AudioVideoToggle.tsx b/packages/roomkit-react/src/Prebuilt/components/AudioVideoToggle.tsx
index 1a1552dd50..dfcf4902c1 100644
--- a/packages/roomkit-react/src/Prebuilt/components/AudioVideoToggle.tsx
+++ b/packages/roomkit-react/src/Prebuilt/components/AudioVideoToggle.tsx
@@ -1,4 +1,4 @@
-import React, { Fragment, useEffect, useState } from 'react';
+import React, { Fragment, useCallback, useEffect, useState } from 'react';
import { HMSKrispPlugin } from '@100mslive/hms-noise-cancellation';
import {
DeviceType,
@@ -14,6 +14,7 @@ import {
useDevices,
useHMSActions,
useHMSStore,
+ useHMSVanillaStore,
} from '@100mslive/react-sdk';
import {
AudioLevelIcon,
@@ -105,22 +106,26 @@ const useNoiseCancellationWithPlugin = () => {
const actions = useHMSActions();
const [inProgress, setInProgress] = useState(false);
const [, setNoiseCancellationEnabled] = useSetNoiseCancellation();
- const setNoiseCancellationWithPlugin = async (enabled: boolean) => {
- if (inProgress) {
- return;
- }
- if (!krispPlugin.checkSupport().isSupported) {
- throw Error('Krisp plugin is not supported');
- }
- setInProgress(true);
- if (enabled) {
- await actions.addPluginToAudioTrack(krispPlugin);
- } else {
- await actions.removePluginFromAudioTrack(krispPlugin);
- }
- setNoiseCancellationEnabled(enabled);
- setInProgress(false);
- };
+ const isEnabledForRoom = useHMSStore(selectRoom)?.isNoiseCancellationEnabled;
+ const setNoiseCancellationWithPlugin = useCallback(
+ async (enabled: boolean) => {
+ if (!isEnabledForRoom || inProgress) {
+ return;
+ }
+ if (!krispPlugin.checkSupport().isSupported) {
+ throw Error('Krisp plugin is not supported');
+ }
+ setInProgress(true);
+ if (enabled) {
+ await actions.addPluginToAudioTrack(krispPlugin);
+ } else {
+ await actions.removePluginFromAudioTrack(krispPlugin);
+ }
+ setNoiseCancellationEnabled(enabled);
+ setInProgress(false);
+ },
+ [actions, inProgress, isEnabledForRoom, setNoiseCancellationEnabled],
+ );
return {
setNoiseCancellationWithPlugin,
inProgress,
@@ -274,6 +279,7 @@ export const AudioVideoToggle = ({ hideOptions = false }: { hideOptions?: boolea
const localPeer = useHMSStore(selectLocalPeer);
const { isLocalVideoEnabled, isLocalAudioEnabled, toggleAudio, toggleVideo } = useAVToggle();
const actions = useHMSActions();
+ const vanillaStore = useHMSVanillaStore();
const videoTrackId = useHMSStore(selectLocalVideoTrackID);
const localVideoTrack = useHMSStore(selectVideoTrackByID(videoTrackId));
const roomState = useHMSStore(selectRoomState);
@@ -289,7 +295,14 @@ export const AudioVideoToggle = ({ hideOptions = false }: { hideOptions?: boolea
useEffect(() => {
(async () => {
- if (isNoiseCancellationEnabled && !isKrispPluginAdded && !inProgress && localPeer?.audioTrack) {
+ const isEnabledForRoom = vanillaStore.getState(selectRoom)?.isNoiseCancellationEnabled;
+ if (
+ isEnabledForRoom &&
+ isNoiseCancellationEnabled &&
+ !isKrispPluginAdded &&
+ !inProgress &&
+ localPeer?.audioTrack
+ ) {
try {
await setNoiseCancellationWithPlugin(true);
ToastManager.addToast({
diff --git a/packages/roomkit-react/src/Prebuilt/components/MoreSettings/constants.ts b/packages/roomkit-react/src/Prebuilt/components/MoreSettings/constants.ts
index 41c17d405f..b655f264f3 100644
--- a/packages/roomkit-react/src/Prebuilt/components/MoreSettings/constants.ts
+++ b/packages/roomkit-react/src/Prebuilt/components/MoreSettings/constants.ts
@@ -10,3 +10,5 @@ export const trackTypeOptions = [
{ label: 'audio', value: 'audio' },
{ label: 'video', value: 'video' },
];
+
+export const DEFAULT_TILES_IN_VIEW = { MWEB: 4, DESKTOP: 9 };
diff --git a/packages/roomkit-react/src/Prebuilt/components/Notifications/ReconnectNotifications.tsx b/packages/roomkit-react/src/Prebuilt/components/Notifications/ReconnectNotifications.tsx
index 1350f85344..71511f55c9 100644
--- a/packages/roomkit-react/src/Prebuilt/components/Notifications/ReconnectNotifications.tsx
+++ b/packages/roomkit-react/src/Prebuilt/components/Notifications/ReconnectNotifications.tsx
@@ -1,6 +1,5 @@
-import React, { useEffect, useState } from 'react';
+import { useEffect, useRef } from 'react';
import { HMSNotificationTypes, useHMSNotifications } from '@100mslive/react-sdk';
-import { Dialog, Flex, Loading, Text } from '../../..';
// @ts-ignore: No implicit Any
import { ToastConfig } from '../Toast/ToastConfig';
// @ts-ignore: No implicit Any
@@ -15,50 +14,24 @@ let notificationId: string | null = null;
export const ReconnectNotifications = () => {
const notification = useHMSNotifications(notificationTypes);
- const [open, setOpen] = useState(false);
+ const prevErrorCode = useRef(0);
useEffect(() => {
if (!notification) {
return;
}
- if (notification.type === HMSNotificationTypes.ERROR && notification.data?.isTerminal) {
- setOpen(false);
- } else if (notification.type === HMSNotificationTypes.RECONNECTED) {
- notificationId = ToastManager.replaceToast(notificationId, ToastConfig.RECONNECTED.single());
- setOpen(false);
+ if (notification.type === HMSNotificationTypes.RECONNECTED) {
+ notificationId = ToastManager.replaceToast(
+ notificationId,
+ ToastConfig.RECONNECTED.single([4005, 4006].includes(prevErrorCode.current)),
+ );
} else if (notification.type === HMSNotificationTypes.RECONNECTING) {
+ prevErrorCode.current = notification.data?.code || 0;
notificationId = ToastManager.replaceToast(
notificationId,
ToastConfig.RECONNECTING.single(notification.data?.message),
);
}
}, [notification]);
- if (!open) return null;
- return (
-
-
-
-
-
-
-
-
-
- You lost your network connection. Trying to reconnect.
-
-
-
-
-
- );
+
+ return null;
};
diff --git a/packages/roomkit-react/src/Prebuilt/components/Preview/PreviewJoin.tsx b/packages/roomkit-react/src/Prebuilt/components/Preview/PreviewJoin.tsx
index a594978307..7dc9797ceb 100644
--- a/packages/roomkit-react/src/Prebuilt/components/Preview/PreviewJoin.tsx
+++ b/packages/roomkit-react/src/Prebuilt/components/Preview/PreviewJoin.tsx
@@ -4,6 +4,7 @@ import {
HMSRoomState,
selectAppData,
selectIsLocalVideoEnabled,
+ selectIsVBEnabled,
selectLocalPeer,
selectRoomState,
selectVideoTrackByID,
@@ -131,23 +132,20 @@ const PreviewJoin = ({
return roomState === HMSRoomState.Preview ? (
-
+
{toggleVideo ? null : }
-
+
-
+
{previewHeader.title}
{previewHeader.sub_title}
-
+
{isStreamingOn ? (
{toggleVideo ? : null}
-
+
{
const isMobile = useMedia(cssConfig.media.md);
+ const isVBEnabledForUser = useHMSStore(selectIsVBEnabled);
return (
- {vbEnabled ? : null}
+ {vbEnabled && isVBEnabledForUser ? : null}
{isMobile && }
diff --git a/packages/roomkit-react/src/Prebuilt/components/ScreenShareToggle.jsx b/packages/roomkit-react/src/Prebuilt/components/ScreenShareToggle.jsx
index 9d8d145bef..23c35876dc 100644
--- a/packages/roomkit-react/src/Prebuilt/components/ScreenShareToggle.jsx
+++ b/packages/roomkit-react/src/Prebuilt/components/ScreenShareToggle.jsx
@@ -2,6 +2,7 @@ import React, { Fragment } from 'react';
import { selectIsAllowedToPublish, useAwayNotifications, useHMSStore, useScreenShare } from '@100mslive/react-sdk';
import { ShareScreenIcon } from '@100mslive/react-icons';
import { ShareScreenOptions } from './pdfAnnotator/shareScreenOptions';
+import { ToastManager } from './Toast/ToastManager';
import { Box, Flex } from '../../Layout';
import { Tooltip } from '../../Tooltip';
import { ScreenShareButton } from './ShareMenuIcon';
@@ -13,7 +14,17 @@ export const ScreenshareToggle = ({ css = {} }) => {
const isAllowedToPublish = useHMSStore(selectIsAllowedToPublish);
const isAudioOnly = useUISettings(UI_SETTINGS.isAudioOnly);
- const { amIScreenSharing, screenShareVideoTrackId: video, toggleScreenShare } = useScreenShare();
+ const {
+ amIScreenSharing,
+ screenShareVideoTrackId: video,
+ toggleScreenShare,
+ } = useScreenShare(error => {
+ ToastManager.addToast({
+ title: error.message,
+ variant: 'error',
+ duration: 2000,
+ });
+ });
const { requestPermission } = useAwayNotifications();
const isVideoScreenshare = amIScreenSharing && !!video;
if (!isAllowedToPublish.screen || !isScreenshareSupported()) {
diff --git a/packages/roomkit-react/src/Prebuilt/components/Toast/ToastConfig.jsx b/packages/roomkit-react/src/Prebuilt/components/Toast/ToastConfig.jsx
index ccc5fa1076..82ee7f9a65 100644
--- a/packages/roomkit-react/src/Prebuilt/components/Toast/ToastConfig.jsx
+++ b/packages/roomkit-react/src/Prebuilt/components/Toast/ToastConfig.jsx
@@ -153,9 +153,9 @@ export const ToastConfig = {
},
},
RECONNECTED: {
- single: () => {
+ single: online => {
return {
- title: `You are now connected`,
+ title: `You are now ${online ? 'online' : 'connected'}`,
icon: ,
variant: 'success',
duration: 3000,
diff --git a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBHandler.tsx b/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBHandler.tsx
index 2a7c44bb01..faae6af437 100644
--- a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBHandler.tsx
+++ b/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBHandler.tsx
@@ -1,8 +1,7 @@
import { HMSEffectsPlugin, HMSVBPlugin, HMSVirtualBackgroundTypes } from '@100mslive/hms-virtual-background';
-
export class VBPlugin {
private hmsPlugin?: HMSVBPlugin;
- private effectsPlugin?: HMSEffectsPlugin | undefined;
+ private effectsPlugin?: HMSEffectsPlugin;
initialisePlugin = (effectsSDKKey?: string, onInit?: () => void) => {
if (this.getVBObject()) {
diff --git a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBPicker.tsx b/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBPicker.tsx
index 7b9788816e..54745d2761 100644
--- a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBPicker.tsx
+++ b/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBPicker.tsx
@@ -30,7 +30,6 @@ import { useSidepaneResetOnLayoutUpdate } from '../AppData/useSidepaneResetOnLay
// @ts-ignore
import { useSetAppDataByKey, useUISettings } from '../AppData/useUISettings';
import { APP_DATA, SIDE_PANE_OPTIONS, UI_SETTINGS } from '../../common/constants';
-import { defaultMedia } from './constants';
const iconDims = { height: '40px', width: '40px' };
@@ -52,9 +51,7 @@ export const VBPicker = ({ backgroundMedia = [] }: { backgroundMedia: VirtualBac
const [loadingEffects, setLoadingEffects] = useSetAppDataByKey(APP_DATA.loadingEffects);
const isPluginAdded = useHMSStore(selectIsLocalVideoPluginPresent(VBHandler?.getName() || ''));
const background = useHMSStore(selectAppData(APP_DATA.background));
- const mediaList = backgroundMedia.length
- ? backgroundMedia.map((media: VirtualBackgroundMedia) => media.url || '')
- : defaultMedia;
+ const mediaList = backgroundMedia.map((media: VirtualBackgroundMedia) => media.url || '');
const inPreview = roomState === HMSRoomState.Preview;
// Hidden in preview as the effect will be visible in the preview tile
@@ -64,7 +61,8 @@ export const VBPicker = ({ backgroundMedia = [] }: { backgroundMedia: VirtualBac
if (!track?.id) {
return;
}
- if (!isPluginAdded) {
+ const vbObject = VBHandler.getVBObject();
+ if (!isPluginAdded && !vbObject) {
setLoadingEffects(true);
let vbObject = VBHandler.getVBObject();
if (!vbObject) {
diff --git a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBToggle.tsx b/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBToggle.tsx
index 5511763330..470debf780 100644
--- a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBToggle.tsx
+++ b/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBToggle.tsx
@@ -1,5 +1,11 @@
import React from 'react';
-import { selectAppData, selectIsEffectsEnabled, selectIsLocalVideoEnabled, useHMSStore } from '@100mslive/react-sdk';
+import {
+ selectAppData,
+ selectIsEffectsEnabled,
+ selectIsLocalVideoEnabled,
+ selectIsVBEnabled,
+ useHMSStore,
+} from '@100mslive/react-sdk';
import { VirtualBackgroundIcon } from '@100mslive/react-icons';
import { Loading } from '../../../Loading';
import { Tooltip } from '../../../Tooltip';
@@ -12,10 +18,11 @@ export const VBToggle = () => {
const toggleVB = useSidepaneToggle(SIDE_PANE_OPTIONS.VB);
const isVBOpen = useIsSidepaneTypeOpen(SIDE_PANE_OPTIONS.VB);
const isVideoOn = useHMSStore(selectIsLocalVideoEnabled);
+ const isVBEnabled = useHMSStore(selectIsVBEnabled);
const isEffectsEnabled = useHMSStore(selectIsEffectsEnabled);
const loadingEffects = useHMSStore(selectAppData(APP_DATA.loadingEffects));
- if (!isVideoOn || (!isEffectsEnabled && isSafari)) {
+ if (!isVideoOn || (!isEffectsEnabled && isSafari) || !isVBEnabled) {
return null;
}
diff --git a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/constants.ts b/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/constants.ts
deleted file mode 100644
index 42b9857363..0000000000
--- a/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/constants.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-export const defaultMedia = [
- 'https://assets.100ms.live/webapp/vb-mini/vb-1.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-2.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-3.png',
- 'https://assets.100ms.live/webapp/vb-mini/vb-4.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-5.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-6.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-7.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-8.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-9.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-10.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-11.jpg',
- 'https://assets.100ms.live/webapp/vb-mini/vb-12.jpg',
-];
diff --git a/yarn.lock b/yarn.lock
index 44e590384d..64ab688117 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7,10 +7,10 @@
resolved "https://registry.yarnpkg.com/@100mslive/hms-noise-cancellation/-/hms-noise-cancellation-0.0.1.tgz#037c8bdfb6b2d7bf12f9d257422150fe6ca43acb"
integrity sha512-DGnzcXRDJREWypIjGX70er+f2k/XLLRF41lrXPs1+PtB1imdEkECPPS0Fg4BA0BCWKDNAGTZBHZPrBDgUmr9Lw==
-"@100mslive/types-prebuilt@0.12.9":
- version "0.12.9"
- resolved "https://registry.yarnpkg.com/@100mslive/types-prebuilt/-/types-prebuilt-0.12.9.tgz#ecfec2710c281acc8e53a684f3f96b0e7ec874a1"
- integrity sha512-3lia/GwpTTPJB9eu/mgWuGFrqYIiT9S7X7NNI+efuBoPExEjIX0uBSNwaSvD/ChEWD4dXOSQt0OOxP0RAeiq/A==
+"@100mslive/types-prebuilt@0.12.11":
+ version "0.12.11"
+ resolved "https://registry.yarnpkg.com/@100mslive/types-prebuilt/-/types-prebuilt-0.12.11.tgz#775416b89fb869163ed58c8dbd131adcb5f47336"
+ integrity sha512-jJUpizFfuDiK2PbQPyD545obyNQjTapE8fVp50lklTDZvcs8JZPOvXBMxUR0fwf2rfPPhcStctxaRcvUW1N8IA==
"@aashutoshrathi/word-wrap@^1.2.3":
version "1.2.6"
@@ -16534,16 +16534,7 @@ string-natural-compare@^3.0.1:
resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-3.0.1.tgz#7a42d58474454963759e8e8b7ae63d71c1e7fdf4"
integrity sha512-n3sPwynL1nwKi3WJ6AIsClwBMa0zTi54fn2oLU6ndfTSIO05xaznjSf15PcBZU6FNWbmN5Q6cxT4V5hGvB4taw==
-"string-width-cjs@npm:string-width@^4.2.0":
- version "4.2.3"
- resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
- integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
- dependencies:
- emoji-regex "^8.0.0"
- is-fullwidth-code-point "^3.0.0"
- strip-ansi "^6.0.1"
-
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -16648,14 +16639,7 @@ string_decoder@~1.1.1:
dependencies:
safe-buffer "~5.1.0"
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
- version "6.0.1"
- resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
- integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
- dependencies:
- ansi-regex "^5.0.1"
-
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -18008,7 +17992,7 @@ worker-timers@^7.0.40:
worker-timers-broker "^6.0.95"
worker-timers-worker "^7.0.59"
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -18026,15 +18010,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0:
string-width "^4.1.0"
strip-ansi "^6.0.0"
-wrap-ansi@^7.0.0:
- version "7.0.0"
- resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
- integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
- dependencies:
- ansi-styles "^4.0.0"
- string-width "^4.1.0"
- strip-ansi "^6.0.0"
-
wrap-ansi@^8.1.0:
version "8.1.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"