Merge pull request #50 from shreyas-jadhav/main

speaker sync while streaming

index.d.ts

@@ -0,0 +1,5 @@
+declare global {
+  var _roundwareSpeakerPausedAt: Date | null;
+  var _roundwareSpeakerStartedAt: Date | null;
+}
+export {};

package.json

@@ -5,7 +5,7 @@
     "type": "git",
     "url": "git+https://github.com/roundware/roundware-web-framework.git"
   },
-  "version": "0.12.6",
+  "version": "0.12.7",
   "description": "Client framework for Roundware, a location-based contributory audio platform",
   "main": "dist/roundware.js",
   "files": [

src/players/SpeakerPrefetchPlayer.ts

@@ -48,7 +48,9 @@ export class SpeakerPrefetchPlayer implements ISpeakerPlayer {
         audioData,
         function (buffer) {
           speakerContext.buffer = buffer;
-
+          // @ts-ignore
+          global._roundwareTotalAudioBufferSize +=
+            buffer.length * buffer.numberOfChannels * 4;
           speakerContext.loaded = true;
           speakerContext.log(`loaded successfully`);
         },

src/players/SpeakerStreamer.ts

@@ -228,6 +228,6 @@ export class SpeakerStreamer implements ISpeakerPlayer {
     this.audio.onended = callback;
   }
   replay(): void {
-    this.audio.fastSeek(0);
+    this.audio.currentTime = 0;
   }
 }

src/players/SpeakerSyncStreamer.ts

@@ -3,12 +3,10 @@ import {
   IGainNode,
   IMediaElementAudioSourceNode,
 } from "standardized-audio-context";
-import { silenceAudioBase64 } from "../playlistAudioTrack";
 import { SpeakerConfig } from "../types/roundware";
 import { ISpeakerPlayer, SpeakerConstructor } from "../types/speaker";
-import { cleanAudioURL, makeAudioSafeToPlay, speakerLog } from "../utils";
+import { cleanAudioURL, speakerLog } from "../utils";
 
-export type SpeakerState = "playing" | "waiting";
 export class SpeakerSyncStreamer implements ISpeakerPlayer {
   isSafeToPlay: boolean = true;
   playing: boolean = false;
@@ -21,25 +19,27 @@ export class SpeakerSyncStreamer implements ISpeakerPlayer {
   gainNode: IGainNode<IAudioContext>;
   mediaSource: IMediaElementAudioSourceNode<IAudioContext>;
   context: IAudioContext;
-  state: SpeakerState;
+  private _fadingTimeout?: NodeJS.Timeout;
+
   constructor({ audioContext, config, uri, id }: SpeakerConstructor) {
     this.id = id;
     this.config = config;
-    this.uri = cleanAudioURL(uri, true);
+    this.uri = cleanAudioURL(uri);
     this.loaded = true;
     this.loadedPercentage = 100;
     this.context = audioContext;
     this.audio = new Audio(this.uri);
     this.audio.loop = config.loop || false;
     this.gainNode = audioContext.createGain();
     this.gainNode.gain.value = 0;
+    this.audio.crossOrigin = "anonymous";
+    this.audio.loop = false;
+
     this.mediaSource = audioContext.createMediaElementSource(this.audio);
     this.mediaSource.connect(this.gainNode).connect(audioContext.destination);
     this.audio.preload = "auto";
-    this.state = "waiting";
+
     const that = this;
-    this.audio.addEventListener("playing", () => that.setState("playing"));
-    this.audio.addEventListener("waiting", () => that.setState("waiting"));
 
     this.log(`sync streamer initiaed`);
   }
@@ -48,56 +48,88 @@ export class SpeakerSyncStreamer implements ISpeakerPlayer {
 
   async play(): Promise<boolean> {
     if (this.playing) return true;
-    if (this.started) {
-      // set gain to last paused
-      this.gainNode.connect(this.context.destination);
-      this.gainNode.gain.value = 1;
-      this.playing = true;
-      this.log(`Playing... ${this.audio.currentTime}`);
-      return true;
-    }
     try {
+      if (this.context.state !== "running") {
+        await this.context.resume();
+      }
+
       await this.audio.play();
-      this.log(`Started...`);
-      this.started = true;
-      return true;
+
+      this.playing = true;
+
+      if (!this.started) {
+        global._roundwareSpeakerStartedAt = new Date();
+        this.started = true;
+      } else if (
+        global._roundwareSpeakerPausedAt instanceof Date &&
+        global._roundwareSpeakerStartedAt instanceof Date
+      ) {
+        const pausedTime =
+          new Date().getTime() - global._roundwareSpeakerPausedAt.getTime();
+        global._roundwareSpeakerStartedAt = new Date(
+          global._roundwareSpeakerStartedAt.getTime() + pausedTime
+        );
+        global._roundwareSpeakerPausedAt = null;
+      }
+
+      this.log(`Playing...`);
     } catch (e) {
-      console.error(`Failed to play`, e);
-      return false;
+      console.error(e);
+      this.playing = false;
     }
+    return true;
   }
+
   pause(): void {
     this.playing = false;
-    this.gainNode.disconnect();
+    this.audio.pause();
+
+    global._roundwareSpeakerPausedAt = new Date();
   }
   replay(): void {
     this.audio.currentTime = 0;
   }
+
   timerStart(): void {
+    this.syncTracker = Number(
+      setInterval(() => {
+        this.trackSync();
+      }, this.config.syncCheckInterval || 2500)
+    );
+
     this.play();
   }
   timerStop(): void {
     this.pause();
+    clearInterval(this.syncTracker);
   }
   fadingDestination = 0;
   fading = false;
-  fade(destinationVolume?: number, duration: number = 3): void {
-    // if (this.fadingDestination == destinationVolume && this.fading) {
-    //   return;
-    // }
-    // if (typeof destinationVolume == "number") {
-    //   this.fadingDestination = destinationVolume;
-    // }
-
-    // this.gainNode.gain.linearRampToValueAtTime(
-    //   this.fadingDestination,
-    //   this.context.currentTime + duration
-    // );
-    // setTimeout(() => {
-    //   this.fading = false;
-    // }, 3000);
-    this.gainNode.gain.value = 1;
+
+  fade(toVolume: number = this.fadingDestination, duration: number = 3): void {
+    if (this.fadingDestination == toVolume && this.fading) return;
+    this.fadingDestination = toVolume;
+
+    // already at that volume
+    if (Math.abs(this.gainNode.gain.value - this.fadingDestination) < 0.05)
+      return;
+    this.log(
+      `startng fade ${this.gainNode.gain.value} -> ${this.fadingDestination}`
+    );
+    this.gainNode.gain.cancelScheduledValues(0);
+
+    this.gainNode.gain.linearRampToValueAtTime(
+      this.fadingDestination,
+      this.context.currentTime + duration
+    );
+    if (this._fadingTimeout) {
+      clearTimeout(this._fadingTimeout);
+    }
+    this._fadingTimeout = setTimeout(() => {
+      this.fading = false;
+    }, duration * 1000);
   }
+
   fadeOutAndPause(): void {
     this.fade(0);
   }
@@ -106,16 +138,40 @@ export class SpeakerSyncStreamer implements ISpeakerPlayer {
   }
   onLoadingProgress(callback: (newPercent: number) => void): void {}
   onEnd(callback: () => void): void {}
+
   updateTime(newTime: number): void {
     this.audio.currentTime = newTime;
   }
 
-  stateCallback: (newState: SpeakerState) => void = () => {};
-  onStateUpdate(callback: (newState: SpeakerState) => any) {
-    this.stateCallback = callback;
-  }
-  setState(newState: "playing" | "waiting") {
-    this.state = newState;
-    this.stateCallback(this.state);
+  syncTracker?: number;
+  trackSync() {
+    const startedAt: Date = global._roundwareSpeakerStartedAt || new Date();
+    if (!(startedAt instanceof Date)) return;
+    if (!this.playing) return;
+
+    const elapsedTime = new Date().getTime() - startedAt.getTime();
+    if (elapsedTime > this.audio.duration * 1000) return;
+
+    const audioTime = this.audio.currentTime * 1000;
+
+    const difference = elapsedTime - audioTime;
+
+    this.log(
+      `Difference: ${difference} ms; Volume: ${this.gainNode.gain.value}`
+    );
+    if (Math.abs(difference) < (this.config.acceptableDelayMs || 50)) {
+      // within acceptable range;
+      this.audio.playbackRate = 1;
+    } else if (Math.abs(difference) > (this.config.syncCheckInterval || 2500)) {
+      // difference is too much; try to seek instead; seek bit ahead to compensate buffering time
+      this.log(`Seeking to ${(elapsedTime + 500) / 1000}s`);
+      this.audio.currentTime = (elapsedTime + 500) / 1000;
+
+      this.audio.playbackRate = 1;
+    } else {
+      this.audio.playbackRate =
+        1 + difference / (this.config.syncCheckInterval || 2500);
+      this.log(`Changing Playback rate to ${this.audio.playbackRate}`);
+    }
   }
 }

src/speaker_track.ts

@@ -16,6 +16,7 @@ import { SpeakerPrefetchPlayer } from "./players/SpeakerPrefetchPlayer";
 import { ISpeakerData, ISpeakerPlayer } from "./types/speaker";
 import { speakerLog } from "./utils";
 import { SpeakerConfig } from "./types/roundware";
+import { SpeakerSyncStreamer } from "./players/SpeakerSyncStreamer";
 
 const convertLinesToPolygon = (shape: any): Polygon | MultiPolygon =>
   // @ts-ignore
@@ -201,7 +202,11 @@ export class SpeakerTrack {
   }
 
   initPlayer() {
-    const Player = this.config.sync ? SpeakerPrefetchPlayer : SpeakerStreamer;
+    const Player = this.config.sync
+      ? this.config.prefetch
+        ? SpeakerPrefetchPlayer
+        : SpeakerSyncStreamer
+      : SpeakerStreamer;
     this.player = new Player({
       audioContext: this.audioContext,
       id: this.speakerId,

src/types/roundware.d.ts

@@ -42,4 +42,6 @@ export type SpeakerConfig = {
   sync?: boolean;
   loop?: boolean;
   length?: number;
+  acceptableDelayMs?: number;
+  syncCheckInterval?: number;
 };

tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "include": ["./src/**/*"],
+  "include": ["./src/**/*", "./index.d.ts"],
   "compilerOptions": {
     "outDir": "./dist/",
     "target": "es5",