From fbdec620c871f2fbb22b11ba80fd9a1c3be90ee6 Mon Sep 17 00:00:00 2001 From: Yogesh Singh Date: Wed, 11 Dec 2024 16:21:18 +0530 Subject: [PATCH 1/7] fix: added error logs when failing to initialize audio sink --- .../src/audio-sink-manager/AudioSinkManager.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 6d1e667888..7453854e47 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -10,7 +10,6 @@ import { HMSRemoteAudioTrack } from '../media/tracks'; import { HMSRemotePeer } from '../sdk/models/peer'; import { Store } from '../sdk/store'; import HMSLogger from '../utils/logger'; -import { sleep } from '../utils/timer-utils'; /** * Following are the errors thrown when autoplay is blocked in different browsers @@ -82,6 +81,7 @@ export class AudioSinkManager { init(elementId?: string) { if (this.state.initialized || this.audioSink) { + HMSLogger.e(this.TAG, 'audio sink already initialized', this.audioSink); return; } this.state.initialized = true; @@ -144,12 +144,12 @@ export class AudioSinkManager { audioEl.onerror = async () => { HMSLogger.e(this.TAG, 'error on audio element', audioEl.error); const ex = ErrorFactory.TracksErrors.AudioPlaybackError( - `Audio playback error for track - ${track.trackId} code - ${audioEl?.error?.code}`, + `Audio playback error for track - ${track.trackId} code - ${audioEl?.error?.code} ${audioEl?.error} ${track}`, ); this.eventBus.analytics.publish(AnalyticsEventFactory.audioPlaybackError(ex)); if (audioEl?.error?.code === MediaError.MEDIA_ERR_DECODE) { this.removeAudioElement(audioEl, track); - await sleep(500); + // await sleep(500); await this.handleTrackAdd({ track, peer, callListener: false }); } }; @@ -157,7 +157,11 @@ export class AudioSinkManager { track.setVolume(this.volume); HMSLogger.d(this.TAG, 'Audio track added', `${track}`); this.init(); // call to create sink element if not already created - this.audioSink?.append(audioEl); + if (this.audioSink) { + this.audioSink.append(audioEl); + } else { + HMSLogger.e(this.TAG, 'audio sink not initialized', this.audioSink); + } this.outputDevice && (await track.setOutputDevice(this.outputDevice)); audioEl.srcObject = new MediaStream([track.nativeTrack]); callListener && this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, peer); @@ -262,6 +266,8 @@ export class AudioSinkManager { audioEl.srcObject = null; audioEl.remove(); track.setAudioElement(null); + } else { + HMSLogger.e(this.TAG, 'Audio element not found', `${track}`); } }; From 6dfdc9f4460e809e770f71fb78b83086583a787d Mon Sep 17 00:00:00 2001 From: Yogesh Singh Date: Wed, 11 Dec 2024 18:12:13 +0530 Subject: [PATCH 2/7] fix: added more error logs --- .../src/audio-sink-manager/AudioSinkManager.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 7453854e47..09c725b45e 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -149,8 +149,9 @@ export class AudioSinkManager { this.eventBus.analytics.publish(AnalyticsEventFactory.audioPlaybackError(ex)); if (audioEl?.error?.code === MediaError.MEDIA_ERR_DECODE) { this.removeAudioElement(audioEl, track); - // await sleep(500); await this.handleTrackAdd({ track, peer, callListener: false }); + } else { + HMSLogger.e(this.TAG, 'going into else case, the error on audio element is: ', audioEl.error); } }; track.setAudioElement(audioEl); @@ -162,9 +163,17 @@ export class AudioSinkManager { } else { HMSLogger.e(this.TAG, 'audio sink not initialized', this.audioSink); } - this.outputDevice && (await track.setOutputDevice(this.outputDevice)); + if (this.outputDevice) { + await track.setOutputDevice(this.outputDevice); + } else { + HMSLogger.e(this.TAG, 'output device not initialized', this.outputDevice); + } audioEl.srcObject = new MediaStream([track.nativeTrack]); - callListener && this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, peer); + if (callListener) { + this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, peer); + } else { + HMSLogger.e(this.TAG, 'listener set to false', this.listener); + } await this.handleAutoplayError(track); }; From 38517c815348e277589eaea4db86422155c33a58 Mon Sep 17 00:00:00 2001 From: Yogesh Singh Date: Wed, 11 Dec 2024 18:12:39 +0530 Subject: [PATCH 3/7] fix: awaiting setting of volume --- .../hms-video-store/src/audio-sink-manager/AudioSinkManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 09c725b45e..73cffa1ccb 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -155,7 +155,7 @@ export class AudioSinkManager { } }; track.setAudioElement(audioEl); - track.setVolume(this.volume); + await track.setVolume(this.volume); HMSLogger.d(this.TAG, 'Audio track added', `${track}`); this.init(); // call to create sink element if not already created if (this.audioSink) { From 1634d59699d1dcfa85a6019f19434e2998759c91 Mon Sep 17 00:00:00 2001 From: Yogesh Singh Date: Wed, 11 Dec 2024 19:01:41 +0530 Subject: [PATCH 4/7] fix: added more error logs --- .../audio-sink-manager/AudioSinkManager.ts | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 73cffa1ccb..456f6b5a18 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -75,16 +75,18 @@ export class AudioSinkManager { */ async unblockAutoplay() { if (this.autoPausedTracks.size > 0) { - this.unpauseAudioTracks(); + HMSLogger.e(this.TAG, 'unpausing audio tracks', this.autoPausedTracks); + await this.unpauseAudioTracks(); + } else { + HMSLogger.e(this.TAG, 'no audio tracks to unpause', this.autoPausedTracks); } } init(elementId?: string) { if (this.state.initialized || this.audioSink) { - HMSLogger.e(this.TAG, 'audio sink already initialized', this.audioSink); + HMSLogger.e(this.TAG, 'audio sink already initialized', this.audioSink, this.state); return; } - this.state.initialized = true; const audioSink = document.createElement('div'); audioSink.id = `HMS-SDK-audio-sink-${uuid()}`; const userElement = elementId && document.getElementById(elementId); @@ -92,7 +94,8 @@ export class AudioSinkManager { audioSinkParent.append(audioSink); this.audioSink = audioSink; - HMSLogger.d(this.TAG, 'audio sink created', this.audioSink); + this.state.initialized = true; + HMSLogger.e(this.TAG, 'audio sink created', this.audioSink, this.state); } cleanup() { @@ -181,27 +184,34 @@ export class AudioSinkManager { /** * if it's not known whether autoplay will succeed, wait for it to be known */ - if (this.state.autoplayFailed === undefined) { + if (!this.state || this.state.autoplayFailed === undefined) { if (!this.state.autoplayCheckPromise) { // it's the first track, try to play it, that'll tell us whether autoplay is allowed this.state.autoplayCheckPromise = new Promise(resolve => { this.playAudioFor(track).then(resolve); }); + } else { + HMSLogger.e(this.TAG, 'autoplayCheckPromise already exists', this.state.autoplayCheckPromise); } // and wait for the result to be known await this.state.autoplayCheckPromise; + } else { + HMSLogger.e(this.TAG, 'autoplayFailed already known', this.state.autoplayFailed); } /** * Don't play the track if autoplay failed, add to paused list */ if (this.state.autoplayFailed) { this.autoPausedTracks.add(track); + HMSLogger.e(this.TAG, 'autoplay failed, adding to paused list', this.autoPausedTracks); return; + } else { + HMSLogger.e(this.TAG, 'autoplay successful, playing track', track); } await this.playAudioFor(track); }; - private handleAudioDeviceChange = (event: HMSDeviceChangeEvent) => { + private handleAudioDeviceChange = async (event: HMSDeviceChangeEvent) => { // this means the initial load if (!event.selection) { HMSLogger.d(this.TAG, 'device change called'); @@ -211,7 +221,7 @@ export class AudioSinkManager { if (event.isUserSelection || event.error || !event.selection || event.type === 'video') { return; } - this.unpauseAudioTracks(); + await this.unpauseAudioTracks(); }; /** From 22cc7d55c451935e683bbd1e2bc7a2992fcbb530 Mon Sep 17 00:00:00 2001 From: Yogesh Singh Date: Wed, 11 Dec 2024 19:04:14 +0530 Subject: [PATCH 5/7] fix: removed unnecessary async declaration --- .../hms-video-store/src/audio-sink-manager/AudioSinkManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 456f6b5a18..2644c53163 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -117,7 +117,7 @@ export class AudioSinkManager { this.state = { ...INITIAL_STATE }; } - private handleAudioPaused = async (event: any) => { + private handleAudioPaused = (event: any) => { // this means the audio paused because of external factors(headset removal, incoming phone call) HMSLogger.d(this.TAG, 'Audio Paused', event.target.id); const audioTrack = this.store.getTrackById(event.target.id); From b88435ee2e28e36a62b40d9919ab8067b4aac26f Mon Sep 17 00:00:00 2001 From: Yogesh Singh Date: Wed, 11 Dec 2024 19:06:38 +0530 Subject: [PATCH 6/7] fix: added autoplay error log --- .../hms-video-store/src/audio-sink-manager/AudioSinkManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 2644c53163..07ede0f88b 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -250,6 +250,8 @@ export class AudioSinkManager { ex.addNativeError(error); this.eventBus.analytics.publish(AnalyticsEventFactory.autoplayError()); this.eventBus.autoplayError.publish(ex); + } else { + HMSLogger.e(this.TAG, 'autoplay failed, but not a NotAllowedError', error); } } } From 4c6a5ec253e2e2b9bd9827f2c7ce97c4a484f0f8 Mon Sep 17 00:00:00 2001 From: Yogesh Singh Date: Thu, 12 Dec 2024 16:09:48 +0530 Subject: [PATCH 7/7] fix: added error logs --- .../audio-sink-manager/AudioSinkManager.ts | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 9359b0b88e..adebd2a6a2 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -9,7 +9,6 @@ import { HMSRemoteAudioTrack } from '../media/tracks'; import { HMSRemotePeer } from '../sdk/models/peer'; import { Store } from '../sdk/store'; import HMSLogger from '../utils/logger'; -import { sleep } from '../utils/timer-utils'; /** * Following are the errors thrown when autoplay is blocked in different browsers @@ -72,15 +71,18 @@ export class AudioSinkManager { */ async unblockAutoplay() { if (this.autoPausedTracks.size > 0) { - this.unpauseAudioTracks(); + HMSLogger.e(this.TAG, 'Unpausing audio tracks'); + await this.unpauseAudioTracks(); + } else { + HMSLogger.e(this.TAG, 'No audio tracks to unpause'); } } init(elementId?: string) { if (this.state.initialized || this.audioSink) { + HMSLogger.e(this.TAG, 'Audio sink already initialized'); return; } - this.state.initialized = true; const audioSink = document.createElement('div'); audioSink.id = `HMS-SDK-audio-sink-${uuid()}`; const userElement = elementId && document.getElementById(elementId); @@ -88,7 +90,8 @@ export class AudioSinkManager { audioSinkParent.append(audioSink); this.audioSink = audioSink; - HMSLogger.d(this.TAG, 'audio sink created', this.audioSink); + this.state.initialized = true; + HMSLogger.e(this.TAG, 'audio sink created', this.audioSink); } cleanup() { @@ -104,7 +107,7 @@ export class AudioSinkManager { this.state = { ...INITIAL_STATE }; } - private handleAudioPaused = async (event: any) => { + private handleAudioPaused = (event: any) => { // this means the audio paused because of external factors(headset removal, incoming phone call) HMSLogger.d(this.TAG, 'Audio Paused', event.target.id); const audioTrack = this.store.getTrackById(event.target.id); @@ -134,19 +137,20 @@ export class AudioSinkManager { audioEl.onerror = async () => { HMSLogger.e(this.TAG, 'error on audio element', audioEl.error); const ex = ErrorFactory.TracksErrors.AudioPlaybackError( - `Audio playback error for track - ${track.trackId} code - ${audioEl?.error?.code}`, + `Audio playback error for track - ${track.trackId} code - ${audioEl?.error?.code} ${audioEl.error} ${track}`, ); this.eventBus.analytics.publish(AnalyticsEventFactory.audioPlaybackError(ex)); if (audioEl?.error?.code === MediaError.MEDIA_ERR_DECODE) { // try to wait for main execution to complete first this.removeAudioElement(audioEl, track); - await sleep(500); await this.handleTrackAdd({ track, peer, callListener: false }); if (!this.state.autoplayFailed) { this.eventBus.analytics.publish( AnalyticsEventFactory.audioRecovered('Audio recovered after media decode error'), ); } + } else { + HMSLogger.e(this.TAG, 'Audio playback error', audioEl.error); } }; track.setAudioElement(audioEl); @@ -154,9 +158,17 @@ export class AudioSinkManager { HMSLogger.d(this.TAG, 'Audio track added', `${track}`); this.init(); // call to create sink element if not already created this.audioSink?.append(audioEl); - this.outputDevice && (await track.setOutputDevice(this.outputDevice)); + if (this.outputDevice) { + await track.setOutputDevice(this.outputDevice); + } else { + HMSLogger.e(this.TAG, 'No output device found', `${track}`); + } audioEl.srcObject = new MediaStream([track.nativeTrack]); - callListener && this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, peer); + if (callListener) { + this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, peer); + } else { + HMSLogger.e(this.TAG, 'No listener found', `${track}`); + } await this.handleAutoplayError(track); }; @@ -164,7 +176,7 @@ export class AudioSinkManager { /** * if it's not known whether autoplay will succeed, wait for it to be known */ - if (this.state.autoplayFailed === undefined) { + if (!this.state.autoplayFailed) { if (!this.state.autoplayCheckPromise) { // it's the first track, try to play it, that'll tell us whether autoplay is allowed this.state.autoplayCheckPromise = new Promise(resolve => { @@ -173,23 +185,28 @@ export class AudioSinkManager { } // and wait for the result to be known await this.state.autoplayCheckPromise; + } else { + HMSLogger.e(this.TAG, 'Autoplay check promise already known', `${track}`); } /** * Don't play the track if autoplay failed, add to paused list */ if (this.state.autoplayFailed) { this.autoPausedTracks.add(track); + HMSLogger.e(this.TAG, 'Autoplay failed', `${track}`); return; + } else { + HMSLogger.e(this.TAG, 'Autoplay succeeded', `${track}`); } await this.playAudioFor(track); }; - private handleAudioDeviceChange = (event: HMSDeviceChangeEvent) => { + private handleAudioDeviceChange = async (event: HMSDeviceChangeEvent) => { // if there is no selection that means this is an init request. No need to do anything if (event.isUserSelection || event.error || !event.selection || event.type === 'video') { return; } - this.unpauseAudioTracks(); + await this.unpauseAudioTracks(); }; /** @@ -253,6 +270,8 @@ export class AudioSinkManager { audioEl.srcObject = null; audioEl.remove(); track.setAudioElement(null); + } else { + HMSLogger.e(this.TAG, 'Audio element not found', `${track}`); } }; }