From 1229ed35f9ced09552c372c7d7e04334e0d65c6a Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Thu, 2 Nov 2023 16:12:37 -0400 Subject: [PATCH] initial draft of AudioPlayer class --- .../src/modules/plugin-api/AudioPlayer.ts | 75 ++++++++++ .../src/modules/plugin-api/MediaAPI.ts | 134 +++++------------- 2 files changed, 112 insertions(+), 97 deletions(-) create mode 100644 packages/jspsych/src/modules/plugin-api/AudioPlayer.ts diff --git a/packages/jspsych/src/modules/plugin-api/AudioPlayer.ts b/packages/jspsych/src/modules/plugin-api/AudioPlayer.ts new file mode 100644 index 0000000000..9daef348ab --- /dev/null +++ b/packages/jspsych/src/modules/plugin-api/AudioPlayer.ts @@ -0,0 +1,75 @@ +export interface AudioPlayerOptions { + useWebAudio: boolean; + audioContext?: AudioContext; +} + +export class AudioPlayer { + private audio: HTMLAudioElement | AudioBufferSourceNode; + private audioContext: AudioContext | null; + private useWebAudio: boolean; + private src: string; + + constructor(src: string, options: AudioPlayerOptions = { useWebAudio: false }) { + this.src = src; + this.useWebAudio = options.useWebAudio; + this.audioContext = options.audioContext || null; + } + + async load() { + if (this.useWebAudio) { + this.audio = await this.preloadWebAudio(this.src); + } else { + this.audio = await this.preloadHTMLAudio(this.src); + } + } + + play() { + if (this.audio instanceof HTMLAudioElement) { + this.audio.play(); + } else { + this.audio!.start(); + } + } + + stop() { + if (this.audio instanceof HTMLAudioElement) { + this.audio.pause(); + this.audio.currentTime = 0; + } else { + this.audio!.stop(); + } + } + + addEventListener(eventName: string, callback: EventListenerOrEventListenerObject) { + this.audio.addEventListener(eventName, callback); + } + + removeEventListener(eventName: string, callback: EventListenerOrEventListenerObject) { + this.audio.removeEventListener(eventName, callback); + } + + private async preloadWebAudio(src: string): Promise { + const buffer = await fetch(src); + const arrayBuffer = await buffer.arrayBuffer(); + const audioBuffer = await this.audioContext!.decodeAudioData(arrayBuffer); + const source = this.audioContext!.createBufferSource(); + source.buffer = audioBuffer; + source.connect(this.audioContext!.destination); + return source; + } + + private async preloadHTMLAudio(src: string): Promise { + return new Promise((resolve, reject) => { + const audio = new Audio(src); + audio.addEventListener("canplaythrough", () => { + resolve(audio); + }); + audio.addEventListener("error", (err) => { + reject(err); + }); + audio.addEventListener("abort", (err) => { + reject(err); + }); + }); + } +} diff --git a/packages/jspsych/src/modules/plugin-api/MediaAPI.ts b/packages/jspsych/src/modules/plugin-api/MediaAPI.ts index eb3588afb4..5e5ecb80e0 100644 --- a/packages/jspsych/src/modules/plugin-api/MediaAPI.ts +++ b/packages/jspsych/src/modules/plugin-api/MediaAPI.ts @@ -1,5 +1,6 @@ import { ParameterType } from "../../modules/plugins"; import { unique } from "../utils"; +import { AudioPlayer } from "./AudioPlayer"; const preloadParameterTypes = [ ParameterType.AUDIO, @@ -9,7 +10,7 @@ const preloadParameterTypes = [ type PreloadType = typeof preloadParameterTypes[number]; export class MediaAPI { - constructor(private useWebaudio: boolean) { + constructor(public useWebaudio: boolean) { if ( this.useWebaudio && typeof window !== "undefined" && @@ -32,36 +33,24 @@ export class MediaAPI { private context: AudioContext = null; private audio_buffers = []; - audioContext() { + audioContext(): AudioContext { if (this.context && this.context.state !== "running") { this.context.resume(); } return this.context; } - getAudioBuffer(audioID) { - return new Promise((resolve, reject) => { - // check whether audio file already preloaded - if ( - typeof this.audio_buffers[audioID] == "undefined" || - this.audio_buffers[audioID] == "tmp" - ) { - // if audio is not already loaded, try to load it - this.preloadAudio( - [audioID], - () => { - resolve(this.audio_buffers[audioID]); - }, - () => {}, - (e) => { - reject(e.error); - } - ); - } else { - // audio is already loaded - resolve(this.audio_buffers[audioID]); - } - }); + async getAudioPlayer(audioID: string): Promise { + if (this.audio_buffers[audioID] instanceof AudioPlayer) { + return this.audio_buffers[audioID]; + } else { + this.audio_buffers[audioID] = new AudioPlayer(audioID, { + useWebAudio: this.useWebaudio, + audioContext: this.context, + }); + await this.audio_buffers[audioID].load(); + return this.audio_buffers[audioID]; + } } // preloading stimuli // @@ -72,8 +61,8 @@ export class MediaAPI { preloadAudio( files, callback_complete = () => {}, - callback_load = (filepath) => {}, - callback_error = (error_msg) => {} + callback_load = (filepath: string) => {}, + callback_error = (error: string) => {} ) { files = unique(files.flat()); @@ -84,80 +73,31 @@ export class MediaAPI { return; } - const load_audio_file_webaudio = (source, count = 1) => { - const request = new XMLHttpRequest(); - request.open("GET", source, true); - request.responseType = "arraybuffer"; - request.onload = () => { - this.context.decodeAudioData( - request.response, - (buffer) => { - this.audio_buffers[source] = buffer; - n_loaded++; - callback_load(source); - if (n_loaded == files.length) { - callback_complete(); - } - }, - (e) => { - callback_error({ source: source, error: e }); - } - ); - }; - request.onerror = (e) => { - let err: ProgressEvent | string = e; - if (request.status == 404) { - err = "404"; - } - callback_error({ source: source, error: err }); - }; - request.onloadend = (e) => { - if (request.status == 404) { - callback_error({ source: source, error: "404" }); - } - }; - request.send(); - this.preload_requests.push(request); - }; - - const load_audio_file_html5audio = (source, count = 1) => { - const audio = new Audio(); - const handleCanPlayThrough = () => { - this.audio_buffers[source] = audio; - n_loaded++; - callback_load(source); - if (n_loaded == files.length) { - callback_complete(); - } - audio.removeEventListener("canplaythrough", handleCanPlayThrough); - }; - audio.addEventListener("canplaythrough", handleCanPlayThrough); - audio.addEventListener("error", function handleError(e) { - callback_error({ source: audio.src, error: e }); - audio.removeEventListener("error", handleError); - }); - audio.addEventListener("abort", function handleAbort(e) { - callback_error({ source: audio.src, error: e }); - audio.removeEventListener("abort", handleAbort); - }); - audio.src = source; - this.preload_requests.push(audio); - }; - for (const file of files) { - if (typeof this.audio_buffers[file] !== "undefined") { + // check if file was already loaded + if (this.audio_buffers[file] instanceof AudioPlayer) { n_loaded++; callback_load(file); if (n_loaded == files.length) { callback_complete(); } } else { - this.audio_buffers[file] = "tmp"; - if (this.audioContext() !== null) { - load_audio_file_webaudio(file); - } else { - load_audio_file_html5audio(file); - } + this.audio_buffers[file] = new AudioPlayer(file, { + useWebAudio: this.useWebaudio, + audioContext: this.context, + }); + this.audio_buffers[file] + .load() + .then(() => { + n_loaded++; + callback_load(file); + if (n_loaded == files.length) { + callback_complete(); + } + }) + .catch((e) => { + callback_error(e); + }); } } } @@ -223,7 +163,7 @@ export class MediaAPI { const request = new XMLHttpRequest(); request.open("GET", video, true); request.responseType = "blob"; - request.onload = () => { + request.onload = () => { if (request.status === 200 || request.status === 0) { const videoBlob = request.response; video_buffers[video] = URL.createObjectURL(videoBlob); // IE10+ @@ -234,14 +174,14 @@ export class MediaAPI { } } }; - request.onerror = (e) => { + request.onerror = (e) => { let err: ProgressEvent | string = e; if (request.status == 404) { err = "404"; } callback_error({ source: video, error: err }); }; - request.onloadend = (e) => { + request.onloadend = (e) => { if (request.status == 404) { callback_error({ source: video, error: "404" }); }