diff --git a/extensions/BeatBlox/README.md b/extensions/BeatBlox/README.md index eca8fcf..570e67f 100644 --- a/extensions/BeatBlox/README.md +++ b/extensions/BeatBlox/README.md @@ -1,3 +1,3 @@ # Updating `webAudioAPI.js` -Pull from [Will's library](https://github.com/hedgecrw/WebAudioAPI). +Pull from [Will's library](https://github.com/hedgecrw/WebAudioAPI) (single-file ECMAScript module). diff --git a/extensions/BeatBlox/index.js b/extensions/BeatBlox/index.js index 8a31d0d..b991f80 100644 --- a/extensions/BeatBlox/index.js +++ b/extensions/BeatBlox/index.js @@ -1,273 +1,124 @@ (function () { + const I32_MAX = 2147483647; + const SCHEDULING_WINDOW = 0.02; // seconds + + function clamp(x, a, b) { return x < a ? a : x > b ? b : x; } + function lerp(x, a, b) { return (1 - x) * a + x * b; } + + const EFFECT_INFO = { + 'Volume': { type: 'Volume' , identity: 1.0, params: x => ({ 'intensity': clamp(x, 0, 1) }) }, + 'Delay': { type: 'Delay', identity: 0.0, params: x => ({ 'delay': clamp(x, 0, 1), 'attenuation': 0.5 }) }, + 'Reverb': { type: 'Reverb', identity: 0.0, params: x => ({ 'decay': 0.3, 'roomSize': 0.1, 'intensity': clamp(x, 0, 1) }) }, + 'Echo': { type: 'Echo', identity: 0.0, params: x => ({ 'echoTime': 0.5, 'intensity': clamp(x, 0, 1) * 0.4 }) }, + 'Panning': { type: 'Panning', identity: 0.5, params: x => ({ 'leftToRightRatio': clamp(x, 0, 1) }) }, + 'Underwater': { type: 'LowPassFilter', identity: 0.0, params: x => ({ 'cutoffFrequency': lerp(Math.pow(clamp(x, 0, 1), 0.1666), 22050, 500), 'resonance': 12 }) }, + 'Telephone': { type: 'HighPassFilter', identity: 0.0, params: x => ({ 'cutoffFrequency': Math.pow(clamp(x, 0, 1), 1.4) * 1800, 'resonance': 10 }) }, + 'Fan': { type: 'Tremolo', identity: 0.0, params: x => ({ 'rate': 15, 'intensity': Math.pow(clamp(x, 0, 1), 0.7) }) }, + }; + + const ANALYSIS_INFO = { + 'samples': { type: 'TimeSeries', transform: x => Array.from(x).map(v => (v - 128) / 128) }, + 'spectrum': { type: 'PowerSpectrum', transform: x => Array.from(x).map(v => v / 255) } + }; + + const DRUM_TO_NOTE = { + 'kick': 'A1', 'kick #2': 'C2', 'snare': 'G1', 'side stick snare': 'C#2', + 'open snare': 'E2', 'closed hi-hat': 'F#2', 'clap': 'Eb2', 'tom': 'C3', + 'rack tom': 'B2', 'floor tom': 'F2', 'crash': 'Bb2', 'crash #2': 'E3', + 'ride': 'Eb3', 'ride #2': 'F3', 'tamborine': 'F#3', + }; + + function absoluteUrl(relative) { + const defaultSrc = 'https://extensions.netsblox.org/extensions/BeatBlox/webAudioAPI.js'; + const src = JSON.parse(new URLSearchParams(window.location.search).get('extensions') || '[]').find(x => x.includes('BeatBlox/index.js')) || defaultSrc; + const res = `${src.substring(0, src.lastIndexOf('/'))}/${relative}`; + console.log(`resolved '${relative}' to ${res}`); + return res; + } + const script = document.createElement('script'); - const devRootScript = 'http://localhost:9090/extensions/BeatBlox/webAudioAPI.js'; - const releaseRootScript = 'https://extensions.netsblox.org/extensions/BeatBlox/webAudioAPI.js'; script.type = 'module'; - script.src = window.origin.includes('localhost') ? devRootScript : releaseRootScript; script.async = false; + script.src = absoluteUrl('webAudioAPI.js'); script.onload = () => { - let lastRecordedClip = null; - let currentDeviceType = null; - let appliedEffects = []; - const audioAPI = new window.WebAudioAPI(); - const I32_MAX = 2147483647; - const SCHEDULING_WINDOW = 0.02; // seconds + const audio = new window.WebAudioAPI(); - const availableEffects = audioAPI.getAvailableEffects(); - const availableNoteDurations = audioAPI.getAvailableNoteDurations(); - const availableNotes = audioAPI.getAvailableNotes(); - const availableAnalysisTypes = audioAPI.getAvailableAnalysisTypes(); - const availableKeySignatures = audioAPI.getAvailableKeySignatures(); - const availableEncoders = audioAPI.getAvailableEncoders(); - const availableNoteModifiers = audioAPI.getAvailableNoteModifications() + const DURATIONS = audio.getAvailableNoteDurations(); + const MODIFIERS = audio.getAvailableNoteModifications() + const ANALYSES = audio.getAvailableAnalysisTypes(); + const ENCODERS = audio.getAvailableEncoders(); + const EFFECTS = audio.getAvailableEffects(); + const NOTES = audio.getAvailableNotes(); + const KEYS = audio.getAvailableKeySignatures(); - const devRoot = 'http://localhost:9090/extensions/BeatBlox/instruments/'; - const releaseRoot = 'https://extensions.netsblox.org/extensions/BeatBlox/instruments/'; - const instrumentLocation = window.origin.includes('localhost') ? devRoot : releaseRoot; + let INPUT_DEVICES = []; + let MIDI_DEVICES = []; + let INSTRUMENTS = []; + let connectedDevice = null; + let activeRecording = null; + audio.start(); - let midiDevices = []; - let audioDevices = []; - let midiInstruments = []; - - audioAPI.start(); - - const instrumentPrefetch = (async () => { + const PREFETCH = (async () => { try { - midiDevices = (await audioAPI.getAvailableMidiDevices()).map((x) => `${x}---(midi)`); + MIDI_DEVICES = (await audio.getAvailableMidiDevices()).map(x => `${x}---(midi)`); } catch (e) { console.error('failed to load midi devices', e); } try { - audioDevices = await audioAPI.getAvailableAudioInputDevices(); + INPUT_DEVICES = await audio.getAvailableAudioInputDevices(); } catch (e) { console.error('failed to load audio input devices', e); } try { - midiInstruments = await audioAPI.getAvailableInstruments(instrumentLocation); - const indexOfDrumKit = midiInstruments.indexOf('Drum Kit'); - midiInstruments.splice(indexOfDrumKit,1); + INSTRUMENTS = await audio.getAvailableInstruments(absoluteUrl('instruments')); + console.log('beginning instrument pre-fetch...'); const tempTrack = '<<>>'; - audioAPI.createTrack(tempTrack); - await Promise.all(midiInstruments.map((x) => audioAPI.updateInstrument(tempTrack, x))); - audioAPI.removeTrack(tempTrack); + audio.createTrack(tempTrack); + await Promise.all(INSTRUMENTS.map(x => audio.updateInstrument(tempTrack, x))); + audio.removeTrack(tempTrack); console.log('instrument pre-fetch completed'); + + const p = INSTRUMENTS.indexOf('Drum Kit'); + if (p >= 0) INSTRUMENTS.splice(p, 1); } catch (e) { console.error('failed to load instruments', e); } })(); - const EffectsPreset = { - 'Under Water': ['LowPassFilter', { - ['cutoffFrequency']: 500, - ['resonance']: 12, - }], - 'Telephone': ['HighPassFilter', { - ['cutoffFrequency']: 1800, - ['resonance']: 10, - }], - 'Cave': ['Echo', { - ['feedback']: 0.5, - ['intensity']: 0.4, - }], - 'Fan Blade': ['Tremolo', { - ['tremeloFrequency'] : 18, - }], - }; - - const CHORD_PATTERNS = { - 'Major': [0, 4, 7], - 'Minor': [0, 3, 7], - 'Diminished': [0, 3, 6], - 'Augmented': [0, 4, 8], - 'Major 7th': [0, 4, 7, 11], - 'Dominant 7th': [0, 4, 7, 10], - 'Minor 7th': [0, 3, 7, 10], - 'Diminished 7th': [0, 3, 6, 9], - }; - const SCALE_PATTERNS = { - 'Major': [0, 2, 4, 5, 7, 9, 11, 12], - 'Minor': [0, 2, 3, 5, 7, 8, 10, 12], - }; - - const NOTE_DURATION_LENGTHS = [ - 0.125, 0.1875, 0.21875, 0.25, 0.375, 0.4375, 0.5, - 0.75, 0.875, 1, 1.5, 1.75, 2, 3, 3.5, 4, 6, 7 - ]; - const NOTE_DURATION_NAMES = [ - 'ThirtySecond', 'DottedThirtySecond', 'DoubleDottedThirtySecond', 'Sixteenth', - 'DottedSixteenth', 'DoubleDottedSixteenth', 'Eighth', 'DottedEighth', 'DoubleDottedEighth', - 'Quarter', 'DottedQuarter', 'DoubleDottedQuarter', 'Half', 'DottedHalf', 'DoubleDottedHalf', - 'Whole', 'DottedWhole', 'DoubleDottedWhole' - ]; - - /** - * Connects a MIDI device to the WebAudioAPI - * @param {String} trackName - Name of the Track - * @param {String} device - Name of the MIDI device being connected. - */ - function midiConnect(trackName, device) { - if (device !== '') { - const mDevice = device.replace('---(midi)', ''); - audioAPI.connectMidiDeviceToTrack(trackName, mDevice).then(() => { - console.log('Connected to MIDI device!'); - }); - currentDeviceType = 'midi'; - } - } - - /** - * Connects and audio input device to NetsBlox - * @param {String} trackName - Name of the Track - * @param {String} device - Name of the audio device being connected. - */ - function audioConnect(trackName, device) { - if (device != '') { - audioAPI.connectAudioInputDeviceToTrack(trackName, device).then(() => { - console.log('Connected to audio device!'); - }); - currentDeviceType = 'audio'; - } - } - - /** - * Converts an AudioClip k to a Snap! Sound. - * @async - * @param {AudioClip} clip - The clip being rendered. - * @returns A Snap! Sound. - */ - async function clipToSnap(clip) { - const blob = await clip.getEncodedData(availableEncoders['WAV']); - const audio = new Audio(URL.createObjectURL(blob, { type: 'audio/wav' })); - return new Sound(audio, 'netsblox-sound'); - } - function snapify(value) { if (typeof (value.map) === 'function') { - return new List(value.map((x) => snapify(x))); + return new List((Array.isArray(value) ? value : Array.from(value)).map(x => snapify(x))); } else if (typeof (value) === 'object') { const res = []; - for (const key in value) res.push(new List([key, snapify(value[key])])); + for (const key in value) { + res.push(new List([key, snapify(value[key])])); + } return new List(res); - } else return value; - } - - function drumToMidiNote(drumName){ - switch (drumName.toLowerCase()) { - case 'kick': - return "A1"; - case 'kick #2': - return "C2"; - case 'snare': - return "G1"; - case 'side stick snare': - return "C#2"; - case 'open snare': - return "E2"; - case 'closed hi-hat': - return "F#2"; - case 'clap': - return "Eb2"; - case 'tom': - return "C3"; - case 'rack tom': - return "B2"; - case 'floor tom': - return "F2"; - case 'crash': - return "Bb2"; - case 'crash #2': - return "E3"; - case 'ride': - return "Eb3"; - case 'ride #2': - return "F3"; - case 'tamborine': - return "F#3"; - case 'rest': - return "Rest"; - default: - return drumName; + } else { + return value; } } - /** - * Disconnects all audio and midi devices from NetsBlox - * @param {String} trackName - name of the Track - * @async - */ - async function disconnectDevices(trackName) { - console.log('device disconnected'); - if (audioDevices.length > 0) - await audioAPI.disconnectAudioInputDeviceFromTrack(trackName); - if (midiDevices.length > 0) - await audioAPI.disconnectMidiDeviceFromTrack(trackName); - } - - /** - * Converts base64 encoding to ArrayBuffer - * @param {String} base64 - base64 encoded audio file - * @returns An Array Buffer - */ - function base64toArrayBuffer(base64) { - const binaryString = window.atob(base64.split(',')[1]); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); + function decodeBase64(base64) { + const bin = atob(base64); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) { + bytes[i] = bin.charCodeAt(i); } return bytes.buffer; } - async function playClip(trackName, clip, startTime, duration = undefined) { - const buffer = clip.audioBuffer || base64toArrayBuffer(clip.audio.src); - return audioAPI.playClip(trackName, buffer, startTime, duration); - } - - /** - * Plays an audio clip - * @param {String} trackName - name of CurrentTrack - * @param {List} notes - notes to be played - * @param {Number} startTime - API time at which to start playback - * @param {Number} noteDuration - duration of note to be played - * @returns An Array Buffer - */ - async function playChord(trackName, notes, startTime, noteDuration, mod = undefined) { - if (notes.length === 0) return 0; - let ret = Infinity; - for (const note of notes) { - ret = Math.min(ret, await audioAPI.playNote(trackName, note, startTime, noteDuration, mod)); - } - return ret; - } - - /** - * Plays an audio clip - * @param {String} trackName - name of CurrentTrack - * @param {List} notes - notes to be played - * @param {Number} startTime - API time at which to start playback - * @param {Number} beats - duration of note to be played - * @returns An Array Buffer - */ - async function playChordBeats(trackName, notes, startTime, beats, mod = undefined, isDrumNote = false) { - if (notes.length === 0) return 0; - let ret = Infinity; - let beatMultiplier = 60 / getBPM(); - for (const note of notes) { - ret = Math.min(ret, await audioAPI.playNote(trackName, note, startTime, -[beatMultiplier * beats], mod, isDrumNote)); - } - return ret; - } - function parseNote(note) { - if (Array.isArray(note)) return note.map((x) => parseNote(x)); - if (note.contents !== undefined) return note.contents.map((x) => parseNote(x)); + if (Array.isArray(note)) return note.map(x => parseNote(x)); + if (note.contents !== undefined) return note.contents.map(x => parseNote(x)); if (typeof (note) === 'number' && Number.isInteger(note)) return note; if (typeof (note) !== 'string' || note === '') throw Error(`expected a note, got '${note}'`); - if (note === 'Rest') return availableNotes[note]; + if (note.toLowerCase() === 'rest') return NOTES['Rest']; const v = Number(note); if (Number.isInteger(v) && note == v) return v; @@ -275,229 +126,105 @@ const letter = note[0]; let octave = null; let delta = 0; - let natural = false; + let accidental = false; for (let i = 1; i < note.length; ++i) { let v = note[i]; if (v === '+' || v === '-' || (v >= '0' && v <= '9')) { if (octave != null) throw Error(`expected a note, got '${note}' (multiple octaves listed)`); ++i; - for (; i < note.length && note[i] >= '0' && note[i] <= '9'; ++i) v += note[i]; + for (; i < note.length && note[i] >= '0' && note[i] <= '9'; ++i) { + v += note[i]; + } --i; octave = parseInt(v); } else if (v === 's' || v === '#' || v === '♯') { delta += 1; + accidental = true; } else if (v === 'b' || v === '♭') { delta -= 1; + accidental = true; } else if (v === 'n') { - natural = true; + accidental = true; } else { throw Error(`expected a note, got '${note}' (unknown character '${v}')`); } } if (octave == null) throw Error(`expected a note, got '${note}' (missing octave number)`); - if (natural && delta != 0) throw Error(`naturals cannot be sharp/flat, got '${note}'`); - const base = availableNotes[`${letter}${octave}`.toUpperCase()]; + const base = NOTES[`${letter}${octave}`.toUpperCase()]; if (base === undefined) throw Error(`expected a note, got '${note}'`); const off = base + delta; if (off <= 0 || off >= 128) throw Error(`Note outside of valid range, got ${note}`); - return natural ? -off : off; + return accidental ? -off : off; } + function parseDrumNote(note) { + if (Array.isArray(note)) return note.map(x => parseDrumNote(x)); + if (note.contents !== undefined) return note.contents.map(x => parseDrumNote(x)); + if (typeof (note) === 'number' && Number.isInteger(note)) return note; + if (typeof (note) !== 'string' || note === '') throw Error(`expected a drum note, got '${note}'`); + if (note.toLowerCase() === 'rest') return NOTES['Rest']; - function durationToBeats(duration, durationSpecial = ''){ - let playDuration = availableNoteDurations[duration]; - - // Regular durations - if(!durationSpecial){ - return 4 / playDuration; - } - else{ - //Special durations - if(durationSpecial === 'Dotted') return (4 / playDuration) * 1.5; - if(durationSpecial === 'DottedDotted') return (4 / playDuration) *1.75; - } + const res = DRUM_TO_NOTE[note.toLowerCase()]; + if (res === undefined) throw Error(`unknown drum sound: "${note}"`); + return parseNote(res); } - - - - async function setTrackEffect(trackName, effectName, level) { - const effectType = availableEffects[effectName]; - if (!appliedEffects.includes(trackName + effectName)) { - await audioAPI.applyTrackEffect(trackName, effectName, effectType); - appliedEffects.push(trackName + effectName); - } - const parameters = audioAPI.getAvailableEffectParameters(effectType); - const effectOptions = {}; - for (let i = 0; i < parameters.length; i++) { - effectOptions[parameters[i].name] = level; - } - await audioAPI.updateTrackEffect(trackName, effectName, effectOptions); - } + async function setupEntity(entity) { + if (!(entity instanceof SpriteMorph) && !(entity instanceof StageMorph)) throw Error('internal error'); - function getBPM() { - var tempoObject = audioAPI.getTempo(); - var bpm = tempoObject.beatsPerMinute - return bpm; - } + if (entity.musicInfo === undefined) { + entity.musicInfo = { + effects: {}, + }; - /** - * Convert seconds to beats given a BPM value. - * - * @param {number} seconds - The number of seconds to convert. - * @param {number} bpm - The beats per minute (BPM). - * @returns {number} - The equivalent number of beats. - */ - function secondsToBeats(seconds, bpm) { - if (bpm <= 0 || seconds < 0) { - throw new Error("BPM must be greater than 0 and seconds must be non-negative."); - } - return (seconds * bpm) / 60; - } + audio.createTrack(entity.id); + audio.createTrack(entity.id + 'Drum'); - async function arrayBufferToTimeSeries(arrayBuffer) { - // Wrap decodeAudioData in a Promise - const audioBuffer = await audioAPI.decodeAudioClip(arrayBuffer); - console.log(audioBuffer); - if (audioBuffer.numberOfChannels > 1) { - const result = []; - for (let i = 0; i < audioBuffer.numberOfChannels; i += 1) { - result.push(Array.from(audioBuffer.getChannelData(i))); - } - return result; + await PREFETCH; + await audio.updateInstrument(entity.id, 'Grand Piano'); + await audio.updateInstrument(entity.id + 'Drum', 'Drum Kit'); } - - const result = Array.from(audioBuffer.getChannelData(0)); - return result; - } - - function getAudioObjectDuration(audioElement) { - return new Promise((resolve, reject) => { - if (audioElement.readyState >= 1) { - // Metadata is already loaded - resolve(audioElement.duration); - } else { - // Wait for the metadata to load - audioElement.addEventListener('loadedmetadata', () => { - resolve(audioElement.duration); - }, { once: true }); - - // Handle error case - audioElement.addEventListener('error', (err) => { - reject('Error loading audio metadata'); - }, { once: true }); - } - }); } + function setupProcess(proc) { + if (!(proc instanceof Process)) throw Error('internal error'); - function getEffectValues(trackName, appliedEffects) { - const res = []; - for (let i = 0; i < appliedEffects.length; i++) { - const objectOfParameters = audioAPI.getCurrentTrackEffectParameters(trackName, appliedEffects[i].replace(trackName, '')); - const valueOfFirstElement = objectOfParameters[Object.keys(objectOfParameters)[0]] - res.push([appliedEffects[i].replace(trackName, ''), valueOfFirstElement * 100]); - + if (proc.musicInfo === undefined) { + proc.musicInfo = { + t: audio.getCurrentTime() + SCHEDULING_WINDOW, + mods: [], + }; } - return snapify(res); - } - - function createTrack(trackName) { - audioAPI.createTrack(trackName); - } - - function stopAudio() { - audioAPI.stop(); - audioAPI.clearAllTracks(); - audioAPI.start(); - } - async function addFxPreset(track, effect) { - const effectName = EffectsPreset[effect][0]; - await audioAPI.applyTrackEffect(track, effectName, availableEffects[effectName]); - appliedEffects.push(effectName); - const effectOptions = EffectsPreset[effect][1]; - await audioAPI.updateTrackEffect(track, effectName, effectOptions); - } - function setupTrack(name) { - const drumTrackName = name+"Drum"; - instrumentPrefetch.then(() => { - appliedEffects = []; - createTrack(name); - createTrack(drumTrackName); - audioAPI.updateInstrument(name, 'Synthesizer'); - audioAPI.updateInstrument(drumTrackName,'Drum Kit'); - }); - } - function setupProcess(proc) { - if (proc.musicInfo) return; - proc.musicInfo = { - t: audioAPI.getCurrentTime() + 0.001, - }; } async function wait(duration) { - return new Promise(resolve => { - setTimeout(resolve, duration * 1000); - }) + return duration <= 0 ? undefined : new Promise(resolve => setTimeout(resolve, duration * 1000)); } async function waitUntil(t) { - await wait(t - audioAPI.getCurrentTime()); - } - - function durationFromInt(beatLength) { - let index = NOTE_DURATION_LENGTHS.indexOf(beatLength); - if (index !== -1) return NOTE_DURATION_NAMES[index]; - - let duration = ''; - while (beatLength >= NOTE_DURATION_LENGTHS[0]) { - for (let i = 0; i < NOTE_DURATION_LENGTHS.length; i++) { - if (i === NOTE_DURATION_LENGTHS.length - 1 && beatLength >= NOTE_DURATION_LENGTHS[i]) index = i; - else if (beatLength < NOTE_DURATION_LENGTHS[i]) { - index = i - 1; - break; - } - } - duration += duration === '' ? `${NOTE_DURATION_NAMES[index]}` : `+${NOTE_DURATION_NAMES[index]}`; - beatLength -= NOTE_DURATION_LENGTHS[index]; - } - return duration === '' ? NOTE_DURATION_NAMES[0] : duration; + await wait(t - audio.getCurrentTime()); } // ---------------------------------------------------------------------- + class MusicApp extends Extension { constructor(ide) { super('MusicApp'); this.ide = ide; + ide.hideCategory('sound'); - appliedEffects = []; - - const oldStopAllActiveSounds = StageMorph.prototype.runStopScripts; + const _runStopScripts = StageMorph.prototype.runStopScripts; StageMorph.prototype.runStopScripts = function () { - oldStopAllActiveSounds.call(this); - stopAudio(); + _runStopScripts.call(this); + audio.clearAllTracks(); }; - - ide.hideCategory('sound'); } onOpenRole() { - //Create Tracks for all sprites - for (const sprite of this.ide.sprites.contents) { - setupTrack(sprite.id); - } - - //Create Track for the stage - setupTrack(this.ide.stage.id); + audio.updateTempo(4, this.ide.stage.tempo); } - onNewSprite(sprite) { - setupTrack(sprite.id); - } - - getMenu() { return {}; } - getCategories() { return [ new Extension.Category('music', new Color(148, 0, 211)), @@ -507,43 +234,29 @@ getPalette() { const blocks = [ new Extension.Palette.Block('setInstrument'), - new Extension.Palette.Block('setKeySignature'), - new Extension.Palette.Block('makeTempo'), + new Extension.Palette.Block('setKey'), + new Extension.Palette.Block('setBPM'), + new Extension.Palette.Block('getBPM'), '-', - new Extension.Palette.Block('playNote'), + new Extension.Palette.Block('playNotes'), + new Extension.Palette.Block('playDrums'), new Extension.Palette.Block('rest'), + new Extension.Palette.Block('noteMod'), + new Extension.Palette.Block('noteNumber'), '-', - new Extension.Palette.Block('hitDrumsOverDuration'), - '-', - new Extension.Palette.Block('playAudioClip'), - new Extension.Palette.Block('playAudioClipForDuration'), - new Extension.Palette.Block('playSampleForDuration'), - new Extension.Palette.Block('stopAudio'), - '-', - new Extension.Palette.Block('soundMetaData'), - '-', - new Extension.Palette.Block('noteModifierC'), - '-', - new Extension.Palette.Block('presetEffect'), - new Extension.Palette.Block('setTrackEffect'), - new Extension.Palette.Block('clearTrackEffects'), - '-', + new Extension.Palette.Block('playClip'), + new Extension.Palette.Block('queryClip'), new Extension.Palette.Block('audioAnalysis'), + new Extension.Palette.Block('createClip'), '-', - new Extension.Palette.Block('appliedEffects').withWatcherToggle(), - new Extension.Palette.Block('tempo').withWatcherToggle(), + new Extension.Palette.Block('setAudioEffect'), + new Extension.Palette.Block('getAudioEffect'), + new Extension.Palette.Block('clearAudioEffects'), '-', - new Extension.Palette.Block('setInputDevice'), - new Extension.Palette.Block('startRecordingInput'), - new Extension.Palette.Block('startRecordingOutput'), - new Extension.Palette.Block('recordOutputForDuration'), - new Extension.Palette.Block('stopRecording'), - new Extension.Palette.Block('lastRecordedClip'), - '-', - new Extension.Palette.Block('noteNew'), - new Extension.Palette.Block('chords'), - new Extension.Palette.Block('scales'), - new Extension.Palette.Block('duration'), + new Extension.Palette.Block('setAudioInput'), + new Extension.Palette.Block('startRecording'), + new Extension.Palette.Block('finishRecording'), + new Extension.Palette.Block('isRecording'), ]; return [ new Extension.PaletteCategory('music', blocks, SpriteMorph), @@ -552,387 +265,299 @@ } getBlocks() { - function playNoteCommon(duration, notes, mods = undefined, isDrumNote=false) { - setupProcess(this); - this.runAsyncFn(async () => { - if (duration === '') throw Error('Please select a valid note duration'); - const durations = duration.split('+'); - - notes = parseNote(notes); - if (!Array.isArray(notes)) notes = [notes]; - if (notes.length === 0) return; - - if (this.mods === undefined) mods = []; - else { - mods = []; - for (let i = 0; i < this.mods.length; i++) { - mods.push(audioAPI.getModification(availableNoteModifiers[this.mods[i]])); - } - } - - await instrumentPrefetch; // wait for all instruments to be loaded - const trackName = this.receiver.id; - let sequence = []; - for (let i = 0; i < durations.length - 1; i++) sequence.push([notes, availableNoteDurations[durations[i]], mods.concat(audioAPI.getModification(availableNoteModifiers["Tie"]))]); - sequence.push([notes, availableNoteDurations[durations[durations.length - 1]], mods]); - for (let i = 0; i < sequence.length; i++) { - const t = await playChord(trackName, sequence[i][0], this.musicInfo.t, sequence[i][1], sequence[i][2], isDrumNote); - this.musicInfo.t += t; - } - await waitUntil(this.musicInfo.t - SCHEDULING_WINDOW); - }, { args: [], timeout: I32_MAX }); - } - async function playNoteCommonBeats(trackName,beats, notes, mod = undefined, isDrumNote=false) { - if (beats === '') throw Error('Please select a valid beat duration'); - if(isDrumNote && notes.length > 1){ - var drumNotes = [] - for(const k of notes){ - drumNotes.push(drumToMidiNote(k)); - } - notes = drumNotes; - } - notes = parseNote(notes); - if (!Array.isArray(notes)) notes = [notes]; - if (notes.length === 0) return; - - setupProcess(this); - await instrumentPrefetch; // wait for all instruments to be loaded - const t = await playChordBeats(trackName, notes, this.musicInfo.t, beats, mod, isDrumNote); - this.musicInfo.t += t; - await waitUntil(this.musicInfo.t - SCHEDULING_WINDOW); - } - async function playClipForDuration(trackName,clip, duration){ - const clipDuration = await getAudioObjectDuration(clip.audio); - if(duration > clipDuration) { - let remainingDuration = duration; - while (remainingDuration > 0){ - const playingDuration = Math.min(remainingDuration, clipDuration); - const t = await playClip(trackName, clip, this.musicInfo.t, playingDuration); - this.musicInfo.t += t; - await waitUntil(this.musicInfo.t - SCHEDULING_WINDOW); - remainingDuration -= playingDuration; - } - } - else{ - const t = await playClip(trackName, clip, this.musicInfo.t, duration); - this.musicInfo.t += t; - await waitUntil(this.musicInfo.t - SCHEDULING_WINDOW); - } - } return [ - new Extension.Block('setInstrument', 'command', 'music', 'set instrument %webMidiInstrument', ['Synthesizer'], function (instrument) { - if (instrument === '') throw Error(`instrument cannot be empty`); - const trackName = this.receiver.id; - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - await audioAPI.updateInstrument(trackName, instrument); + new Extension.Block('setInstrument', 'command', 'music', 'set instrument %instrument', ['Grand Piano'], function (instrument) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + if (INSTRUMENTS.indexOf(instrument) < 0) throw Error(`unknown instrument: "${instrument}"`); + + await audio.updateInstrument(this.receiver.id, instrument); }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('playNote', 'command', 'music', 'play note(s) %s %noteDurations', ['C3', 'Quarter'], function (notes, duration) { - if (notes instanceof List) { - playNoteCommon.apply(this, [duration, notes]); - } else if(typeof notes === 'object') { - var noteName = notes.noteName; - var modifier = notes.modifier; - playNoteCommon.apply(this, [duration, noteName, audioAPI.getModification(modifier, 1)]); - } else { - playNoteCommon.apply(this, [duration, notes]); - } + new Extension.Block('setKey', 'command', 'music', 'set key %keySig', ['CMajor'], function (key) { + if (KEYS[key] === undefined) throw Error(`unknown key: '${key}'`); + audio.updateKeySignature(KEYS[key]); }), - new Extension.Block('rest', 'command', 'music', 'rest %noteDurations', ['Quarter'], function (duration, durationSpecial) { - playNoteCommon.apply(this, [duration, 'Rest']); + new Extension.Block('setBPM', 'command', 'music', 'set tempo %n bpm', [60], function (tempo) { + tempo = Math.max(tempo, 1); + audio.updateTempo(4, tempo, 4, 4); + world.children[0].stage.tempo = tempo; }), - new Extension.Block('playAudioClip', 'command', 'music', 'play clip %snd', [null], function (clip) { - setupProcess(this); - if (clip === '') throw Error(`sound cannot be empty`); - if (this.receiver.sounds.contents.length) { - for (let i = 0; i < this.receiver.sounds.contents.length; i++) { - if (clip === this.receiver.sounds.contents[i].name) { - clip = this.receiver.sounds.contents[i]; - break; - } + new Extension.Block('getBPM', 'reporter', 'music', 'tempo', [], function () { + return audio.getTempo().beatsPerMinute; + }), + new Extension.Block('playNotes', 'command', 'music', 'play %noteDuration note %mult%s', ['Quarter', ['C4']], function (duration, notes) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + notes = parseNote(notes); + if (!Array.isArray(notes)) notes = [notes]; + if (notes.length === 0) notes = [parseNote('Rest')]; + + if (duration.contents !== undefined) duration = duration.contents; + if (!Array.isArray(duration)) duration = notes.map(() => duration); + if (duration.some(x => DURATIONS[x] === undefined)) throw Error('unknown note duration'); + if (duration.length !== notes.length) throw Error('number of durations and notes must match'); + + const mods = this.musicInfo.mods.map(x => audio.getModification(MODIFIERS[x])); + + let t = Infinity; + for (let i = 0; i < notes.length; ++i) { + t = Math.min(t, await audio.playNote(this.receiver.id, notes[i], this.musicInfo.t, DURATIONS[duration[i]], mods)); + } + this.musicInfo.t += t; + await waitUntil(this.musicInfo.t - SCHEDULING_WINDOW); + }, { args: [], timeout: I32_MAX }); + }), + new Extension.Block('playDrums', 'command', 'music', 'hit %noteDuration note drums %mult%drum', ['Quarter', ['Kick']], function (duration, notes) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + notes = parseDrumNote(notes); + if (!Array.isArray(notes)) notes = [notes]; + if (notes.length === 0) notes = [parseDrumNote('Rest')]; + + if (DURATIONS[duration] === undefined) throw Error(`unknown note duration: "${duration}"`); + + const mods = this.musicInfo.mods.map(x => audio.getModification(MODIFIERS[x])); + + const t = await audio.playNote(this.receiver.id + 'Drum', parseDrumNote('Rest'), this.musicInfo.t, DURATIONS[duration], mods); + for (let i = 0; i < notes.length; ++i) { + await audio.playNote(this.receiver.id + 'Drum', notes[i], this.musicInfo.t + t * (i / notes.length), DURATIONS[duration], mods, true); } - } - if(clip instanceof List){ - const newClip = {} - const listArray = clip.contents; - const bytes = new Float32Array(listArray); - const buffer = ((new AudioContext()).createBuffer(1, listArray.length, 44100)); - buffer.copyToChannel(bytes,0); - newClip.audioBuffer = buffer; - clip = newClip; - } - this.runAsyncFn(async () => { - - await instrumentPrefetch; // wait for all instruments to be loaded - const trackName = this.receiver.id; - const t = await playClip(trackName, clip, this.musicInfo.t); this.musicInfo.t += t; await waitUntil(this.musicInfo.t - SCHEDULING_WINDOW); }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('playAudioClipForDuration', 'command', 'music', 'play clip %snd for duration %n secs', [null, 0], function (clip, duration) { + new Extension.Block('rest', 'command', 'music', 'rest %noteDuration', ['Quarter'], function (duration) { + this.playNotes(duration, 'Rest'); + }), + new Extension.Block('noteMod', 'command', 'music', 'note modifiers %mult%noteModifier %c', [['TurnUpper']], function (mods, code) { setupProcess(this); - if (clip === '') throw Error(`sound cannot be empty`); - if (duration <= 0) throw Error(`duration must be greater than 0`); - if (this.receiver.sounds.contents.length) { - for (let i = 0; i < this.receiver.sounds.contents.length; i++) { - if (clip === this.receiver.sounds.contents[i].name) { - clip = this.receiver.sounds.contents[i]; - break; - } + + if (mods.contents !== undefined) mods = mods.contents; + if (!Array.isArray(mods)) mods = [mods]; + if (mods.some(x => MODIFIERS[x] === undefined)) throw Error('unknown note modifier'); + + if (!this.context.modInfo) { + this.context.modInfo = { count: mods.length }; + + for (const mod of mods) { + this.musicInfo.mods.push(mod); } + this.pushContext(code?.blockSequence() || []); + this.pushContext(); + } else { + for (let i = 0; i < this.context.modInfo.count; ++i) { + this.musicInfo.mods.pop(); + } } - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - const trackName = this.receiver.id; - await playClipForDuration.apply(this,[trackName,clip,duration]); - + }), + new Extension.Block('noteNumber', 'reporter', 'music', 'note# %s', ['C4'], parseNote), + new Extension.Block('playClip', 'command', 'music', 'play sound %snd', [], function (rawSound) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + const sound = typeof(rawSound) === 'string' ? this.receiver.sounds.contents.find(x => x.name === rawSound) : rawSound; + if (!sound) throw Error(typeof(rawSound) === 'string' ? `unknown sound: "${rawSound}"` : 'input must be a sound'); + + const buffer = sound.audioBuffer || decodeBase64(sound.audio.src.split(',')[1]); + const t = await audio.playClip(this.receiver.id, buffer, this.musicInfo.t); + this.musicInfo.t += t; + await waitUntil(this.musicInfo.t - SCHEDULING_WINDOW); }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('playSampleForDuration', 'command', 'music', 'play clip %snd for duration %noteDurations', [null, 'Quarter'], function (clip, duration) { - setupProcess(this); - let playDuration = availableNoteDurations[duration]; - if (clip === '') throw Error(`sound cannot be empty`); - if (this.receiver.sounds.contents.length) { - for (let i = 0; i < this.receiver.sounds.contents.length; i++) { - if (clip === this.receiver.sounds.contents[i].name) { - clip = this.receiver.sounds.contents[i]; - break; + new Extension.Block('queryClip', 'reporter', 'music', '%audioQuery of sound %snd', ['samples'], function (query, rawSound) { + return this.runAsyncFn(async () => { + const sound = typeof(rawSound) === 'string' ? this.receiver.sounds.contents.find(x => x.name === rawSound) : rawSound; + if (!sound) throw Error(typeof(rawSound) === 'string' ? `unknown sound: "${rawSound}"` : 'input must be a sound'); + + if (query === 'name') { + return sound.name || 'untitled'; + } else if (query === 'duration') { + return await new Promise((resolve, reject) => { + sound.audio.addEventListener('loadedmetadata', () => resolve(sound.audio.duration), { once: true }); + sound.audio.addEventListener('error', () => reject('Error loading audio metadata'), { once: true }); + if (sound.audio.readyState >= 1) return resolve(sound.audio.duration); + }); + } else if (query === 'samples') { + const buffer = sound.audioBuffer || decodeBase64(sound.audio.src.split(',')[1]); + const decoded = await audio.decodeAudioClip(buffer); + + const res = []; + for (let i = 0; i < decoded.numberOfChannels; ++i) { + res.push(Array.from(decoded.getChannelData(i))); } + return snapify(res); + } else if (query === 'sample rate') { + const buffer = sound.audioBuffer || decodeBase64(sound.audio.src.split(',')[1]); + const decoded = await audio.decodeAudioClip(buffer); + return decoded.sampleRate; + } else { + throw Error(`unknown sound property: "${query}"`); } - } - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - const trackName = this.receiver.id; - const durationInSecs = audioAPI.convertNoteDurationToSeconds(playDuration); - await playClipForDuration.apply(this,[trackName,clip,durationInSecs]); - }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('stopAudio', 'command', 'music', 'stop all audio', [], function () { - stopAudio(); - this.doStopAll(); + new Extension.Block('audioAnalysis', 'reporter', 'music', 'output %audioAnalysis', ['samples'], function (analysis) { + const { type, transform } = ANALYSIS_INFO[analysis] || {}; + if (type === undefined || transform === undefined) throw Error(`unknown audio analysis type: "${analysis}"`); + + return snapify(transform(audio.analyzeAudio(ANALYSES[type]))); }), - new Extension.Block('hitDrumsOverDuration','command','music','for %noteDurations drum sequence %mult%drums',['Quarter', ['Kick']], function(duration,drum){ - setupProcess(this); - if (drum.contents.length === 0) throw Error(`drum cannot be empty`); - if(duration == '') throw Error(`duration cannot be empty`); - const durationInBeats = durationToBeats(duration); - const numberOfNotes = drum.contents.length; - this.runAsyncFn(async () => { - const trackName = this.receiver.id; - const drumTrackName = trackName+"Drum"; - for(const k of drum.contents){ - if(k instanceof List){ - await playNoteCommonBeats.apply(this, [drumTrackName,(durationInBeats/numberOfNotes), k.contents,undefined,true]); - } - else{ - const noteToPlay = drumToMidiNote(k); - const receivedNote = parseNote(noteToPlay); - await playNoteCommonBeats.apply(this, [drumTrackName,(durationInBeats/numberOfNotes), receivedNote,undefined,true]); + new Extension.Block('createClip', 'reporter', 'music', 'samples %l at %n Hz', [null, 44100], function (samples, sampleRate) { + return this.runAsyncFn(async () => { + sampleRate = +sampleRate; + if (isNaN(sampleRate) || !isFinite(sampleRate) || sampleRate < 1) throw Error('invalid sample rate'); + + if (samples.contents !== undefined) samples = samples.contents; + if (!Array.isArray(samples)) throw Error(`expected a list, got ${typeof(samples)}`); + if (samples.every(x => x.contents === undefined && !Array.isArray(x))) samples = [samples]; + samples = [...samples]; + + let maxLen = 0; + for (let i = 0; i < samples.length; ++i) { + if (samples[i].contents !== undefined) samples[i] = samples[i].contents; + if (!Array.isArray(samples[i])) throw Error(`expected a list, got ${typeof(samples[i])}`); + samples[i] = samples[i].map(x => +x); + if (samples[i].some(x => isNaN(x) || !isFinite(x))) throw Error('samples must be numbers'); + maxLen = Math.max(maxLen, samples[i].length); + } + for (const channel of samples) { + for (let i = channel.length; i < maxLen; ++i) { + channel.push(0); } } + + const buffer = audio.createAudioBufferFromSamples(sampleRate, samples); + const blob = await audio.encodeAudioAs(ENCODERS['WAV'], buffer); + const res = new Sound(new Audio(URL.createObjectURL(blob, { type: 'audio/wav' })), 'netsblox-sound'); + res.audioBuffer = blob; + return res; }, { args: [], timeout: I32_MAX }); - - }), - new Extension.Block('soundMetaData', 'reporter', 'music', '%soundMetaData of sound %snd', ['duration', ''], function (option, sound) { - if (sound === '') throw Error(`sound cannot be empty`); - - for (const element of this.receiver.sounds.contents) { - if (sound === element.name) { - sound = element; - break; - } - } - - switch (option) { - case 'name': - return sound.name; - case 'duration': - return this.runAsyncFn(async () => { - const duration = await getAudioObjectDuration(sound.audio); - return duration; - }, { args: [], timeout: I32_MAX }); - case 'beats': - return this.runAsyncFn(async () => { - var currentBPM = getBPM(); - var clipDuration = await getAudioObjectDuration(sound.audio); - return secondsToBeats(clipDuration,currentBPM); - }, { args: [], timeout: I32_MAX }); - case 'samples': - return this.runAsyncFn(async () => { - var buffer = base64toArrayBuffer(sound.audio.src); - var timeSeries = await arrayBufferToTimeSeries(buffer); - return snapify(timeSeries); - }, { args: [], timeout: I32_MAX }); - } - return "OK"; - }), - new Extension.Block('noteModifierC', 'command', 'music', 'modifier %noteModifiers %c', ['Piano'], function (mod, raw_block) { - if (this.mods === undefined) - this.mods = []; - if (!this.context.modFlag) { - this.context.modFlag = true; - this.mods.push(mod); - this.pushContext(raw_block.blockSequence()); - this.pushContext(); - } else { - this.mods.pop(); - } }), - new Extension.Block('noteNew', 'reporter', 'music', 'note %note', [60], parseNote), - new Extension.Block('scales', 'reporter', 'music', 'scale %midiNote type %scaleTypes', ['C3', 'Major'], function (rootNote, type) { - rootNote = parseNote(rootNote); + new Extension.Block('setAudioEffect', 'command', 'music', 'set %audioEffect effect to %n %', ['Volume', 100], function (effect, rawValue) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + const { type, params } = EFFECT_INFO[effect] || {}; + if (type === undefined || params === undefined) throw Error(`unknown effect: "${effect}"`); - const pattern = SCALE_PATTERNS[type]; - if (!pattern) throw Error(`invalid chord type: '${type}'`); + const value = rawValue / 100; + if (isNaN(value)) throw Error(`expected a number, got "${rawValue}"`); - return snapify(pattern.map((x) => rootNote + x)); + const targets = [this.receiver.id, this.receiver.id + 'Drum']; + if (this.receiver.musicInfo.effects[effect] === undefined) { + for (const target of targets) { + await audio.applyTrackEffect(target, effect, EFFECTS[type]); + } + } + this.receiver.musicInfo.effects[effect] = value; + for (const target of targets) { + await audio.updateTrackEffect(target, effect, params(value)); + } + }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('chords', 'reporter', 'music', 'chord %midiNote type %chordTypes', ['C3', 'Major'], function (rootNote, type) { - rootNote = parseNote(rootNote); + new Extension.Block('getAudioEffect', 'reporter', 'music', 'get %audioEffectAug effect', ['Volume'], function (effect) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); - const pattern = CHORD_PATTERNS[type]; - if (!pattern) throw Error(`invalid chord type: '${type}'`); + const getSingle = effect => { + const { identity } = EFFECT_INFO[effect] || {}; + if (identity === undefined) throw Error(`unknown effect: "${effect}"`); - return snapify(pattern.map((x) => rootNote + x)); - }), - new Extension.Block('setTrackEffect', 'command', 'music', 'track %supportedEffects effect to %n %', ['Delay', '50'], function (effectName, level) { - if (parseInt(level) > 100) level = 100 - if (parseInt(level) < 0) level = 0 - if (effectName == 'Echo' && level > 95) level = 95 - if (effectName == 'Reverb' && level < 10) level = 10 - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - const trackName = this.receiver.id; - const drumTrackName = trackName+"Drum"; - await setTrackEffect(trackName, effectName, parseInt(level) / 100); - await setTrackEffect(drumTrackName, effectName, parseInt(level) / 100); + return { value: this.receiver.musicInfo.effects[effect] ?? identity, identity }; + }; + + if (effect === 'every') { + return snapify(Object.keys(EFFECT_INFO).map(x => [x, 100 * getSingle(x).value])); + } else if (effect === 'every active') { + return snapify(Object.keys(EFFECT_INFO).map(x => [x, getSingle(x)]).filter(x => x[1].value !== x[1].identity).map(x => [x[0], 100 * x[1].value])); + } else { + return 100 * getSingle(effect).value; + } }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('clearTrackEffects', 'command', 'music', 'clear track effects', [], function () { - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - const trackName = this.receiver.id; - const drumTrackName = trackName+"Drum"; - for (const effectName in availableEffects) { - await audioAPI.removeTrackEffect(trackName, effectName); - await audioAPI.removeTrackEffect(drumTrackName, effectName); + new Extension.Block('clearAudioEffects', 'command', 'music', 'clear audio effects', [], function () { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + for (const effect in EFFECT_INFO) { + for (const target of [this.receiver.id, this.receiver.id + 'Drum']) { + audio.removeTrackEffect(target, effect); + } } - appliedEffects = []; + this.receiver.musicInfo.effects = {}; }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('makeTempo', 'command', 'music', 'set tempo %n bpm', [120], function (tempo) { - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - audioAPI.updateTempo(4, tempo, 4, 4); + new Extension.Block('setAudioInput', 'command', 'music', 'use input %audioInput', [], function (device) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + if (device === '') { + await audio.disconnectAudioInputDeviceFromTrack(this.receiver.id); + await audio.disconnectMidiDeviceFromTrack(this.receiver.id); + connectedDevice = null; + } else if (MIDI_DEVICES.indexOf(device) !== -1) { + await audio.connectMidiDeviceToTrack(this.receiver.id, device.replace('---(midi)', '')); + connectedDevice = 'midi'; + } else if (INPUT_DEVICES.indexOf(device !== -1)) { + await audio.connectAudioInputDeviceToTrack(this.receiver.id, device); + connectedDevice = 'audio'; + } else { + throw Error(`device not found: ${device}`); + } }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('setKeySignature', 'command', 'music', 'set key signature %keySignatures', ['DMajor'], function (key) { - if (availableKeySignatures[key] === undefined) throw Error(`unknown key signature: '${key}'`); - const currentKeySignature = audioAPI.getKeySignature(); - if (currentKeySignature != key) { - audioAPI.updateKeySignature(availableKeySignatures[key]); - } - console.log(availableKeySignatures[key]); - }), - new Extension.Block('appliedEffects', 'reporter', 'music', 'applied effects', [], function () { - if (appliedEffects.length === 0) { - return new List(); - } - - const trackName = this.id; - return getEffectValues(trackName, appliedEffects); - }).for(SpriteMorph, StageMorph), - new Extension.Block('tempo', 'reporter', 'music', 'tempo', [], function () { - return getBPM(); - }).for(SpriteMorph, StageMorph), - new Extension.Block('presetEffect', 'command', 'music', 'preset effects %fxPreset %onOff', ['', 'on'], function (effect, status) { - const trackName = this.receiver.id; - const drumTrackName = trackName+"Drum"; - if (effect != '') { - if (status == 'on') { - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - await addFxPreset(trackName, effect); - await addFxPreset(drumTrackName, effect); - }); + new Extension.Block('startRecording', 'command', 'music', 'start recording %io', ['output'], function (io) { + return this.runAsyncFn(async () => { + await setupEntity(this.receiver); + setupProcess(this); + + if (activeRecording) throw Error('recording already in progress'); + + if (io === 'input') { + if (!connectedDevice) { + throw Error('no connected input device'); + } else if (connectedDevice === 'midi') { + activeRecording = audio.recordMidiClip(this.receiver.id, audio.getCurrentTime()); + } else if (connectedDevice === 'audio') { + activeRecording = audio.recordAudioClip(this.receiver.id, audio.getCurrentTime()); + } else { + throw Error(`unknown connected device type: "${connectedDevice}"`); + } + } else if (io === 'output') { + activeRecording = audio.recordOutput(); } else { - const effectName = EffectsPreset[effect][0]; - this.runAsyncFn(async () => { - await instrumentPrefetch; // wait for all instruments to be loaded - await audioAPI.removeTrackEffect(trackName, effectName); - await audioAPI.removeTrackEffect(drumTrackName, effectName); - }); + throw Error(`unknown audio direction: ${io}`); } - } else { - throw Error('must select an effect'); - } - }), - new Extension.Block('setInputDevice', 'command', 'music', 'set input device: %inputDevice', [''], function (device) { - const trackName = this.receiver.id; - - if (device === '') - this.runAsyncFn(async () => { - disconnectDevices(trackName); - }, { args: [], timeout: I32_MAX }); - else if (midiDevices.indexOf(device) != -1) - midiConnect(trackName, device); - else if (audioDevices.indexOf(device != -1)) - audioConnect(trackName, device); - else - throw Error('device not found'); - - if (midiInstruments.length > 0) - audioAPI.updateInstrument(trackName, midiInstruments[0]).then(() => { - console.log('default instrument set'); - }); - else - console.log('no default instruments'); - }), - new Extension.Block('startRecordingInput', 'command', 'music', 'start recording input', [], function () { - const trackName = this.receiver.id; - switch (currentDeviceType) { - case 'midi': - lastRecordedClip = audioAPI.recordMidiClip( - trackName, audioAPI.getCurrentTime() - ); - break; - case 'audio': - lastRecordedClip = audioAPI.recordAudioClip( - trackName, audioAPI.getCurrentTime() - ); - break; - } - }), - new Extension.Block('stopRecording', 'command', 'music', 'stop recording', [], function () { - this.runAsyncFn(async () => { - await lastRecordedClip.finalize(); }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('startRecordingOutput', 'command', 'music', 'start recording output', [], function () { - lastRecordedClip = audioAPI.recordOutput(); - }), - new Extension.Block('recordOutputForDuration', 'command', 'music', 'record output for %n seconds', [0], function (time) { - lastRecordedClip = audioAPI.recordOutput(null, null, time); - }), - new Extension.Block('lastRecordedClip', 'reporter', 'music', 'last recorded clip', [], function () { - if (lastRecordedClip == null) throw Error('no recording found'); - + new Extension.Block('finishRecording', 'reporter', 'music', 'finish recording', [], function () { return this.runAsyncFn(async () => { - let temp = await clipToSnap(lastRecordedClip); - temp.audioBuffer = await lastRecordedClip.getEncodedData(availableEncoders['WAV']); - return temp; + const recording = activeRecording; + activeRecording = null; + if (!recording) throw Error('no recording in progress'); + + await recording.finalize(); + + const blob = await recording.getEncodedData(ENCODERS['WAV']); + const res = new Sound(new Audio(URL.createObjectURL(blob, { type: 'audio/wav' })), 'netsblox-sound'); + res.audioBuffer = blob; + return res; }, { args: [], timeout: I32_MAX }); }), - new Extension.Block('audioAnalysis', 'reporter', 'music', 'get output %analysisType', ['TimeSeries'], function (ty) { - if (!availableAnalysisTypes[ty]) throw Error(`unknown audio analysis type: '${ty}'`); - return snapify(audioAPI.analyzeAudio(availableAnalysisTypes[ty])); - }), - new Extension.Block('duration', 'reporter', 'music', 'beat length %n', [], function (beatLength) { - return durationFromInt(beatLength); + new Extension.Block('isRecording', 'predicate', 'music', 'recording?', [], function () { + return !!activeRecording; }), ]; } @@ -940,232 +565,50 @@ getLabelParts() { function identityMap(s) { const res = {}; - for (const x of s) res[x] = x; + for (const x of s) { + if (Array.isArray(x)) { + res[x[0]] = x[1]; + } else { + res[x] = x; + } + } + return res; + } + function unionMaps(maps) { + const res = {}; + for (const map of maps) { + for (const key in map) { + res[key] = map[key]; + } + } return res; } + const basicEnum = (name, values) => new Extension.LabelPart(name, () => new InputSlotMorph(null, false, values, true)); return [ - new Extension.LabelPart('enabled', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(['Enabled', 'Disabled']), - true, - )), - new Extension.LabelPart('effects', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(Object.keys(availableEffects)), - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('supportedEffects', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(['Volume','Delay', 'Reverb', 'Echo', 'Panning']), - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('noteNames', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(['C', 'D', 'E', 'F', 'G', 'A', 'B']), - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('octaves', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']), - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('accidentals', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(['\u266F', '\u266D']), - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('midiNote', () => new InputSlotMorph( - null, //text - false, //numeric - { - 'Rest': 'Rest', - 'C': { - '0': identityMap(['C0bb', 'C0b', 'C0', 'C0s', 'C0ss']), - '1': identityMap(['C1bb', 'C1b', 'C1', 'C1s', 'C1ss']), - '2': identityMap(['C2bb', 'C2b', 'C2', 'C2s', 'C2ss']), - '3': identityMap(['C3bb', 'C3b', 'C3', 'C3s', 'C3ss']), - '4': identityMap(['C4bb', 'C4b', 'C4', 'C4s', 'C4ss']), - '5': identityMap(['C5bb', 'C5b', 'C5', 'C5s', 'C5ss']), - '6': identityMap(['C6bb', 'C6b', 'C6', 'C6s', 'C6ss']), - '7': identityMap(['C7bb', 'C7b', 'C7', 'C7s', 'C7ss']), - '8': identityMap(['C8bb', 'C8b', 'C8', 'C8s', 'C8ss']), - '9': identityMap(['C9bb', 'C9b', 'C9', 'C9s', 'C9ss']), - }, 'D': { - '0': identityMap(['D0bb', 'D0b', 'D0', 'D0s', 'D0ss']), - '1': identityMap(['D1bb', 'D1b', 'D1', 'D1s', 'D1ss']), - '2': identityMap(['D2bb', 'D2b', 'D2', 'D2s', 'D2ss']), - '3': identityMap(['D3bb', 'D3b', 'D3', 'D3s', 'D3ss']), - '4': identityMap(['D4bb', 'D4b', 'D4', 'D4s', 'D4ss']), - '5': identityMap(['D5bb', 'D5b', 'D5', 'D5s', 'D5ss']), - '6': identityMap(['D6bb', 'D6b', 'D6', 'D6s', 'D6ss']), - '7': identityMap(['D7bb', 'D7b', 'D7', 'D7s', 'D7ss']), - '8': identityMap(['D8bb', 'D8b', 'D8', 'D8s', 'D8ss']), - '9': identityMap(['D9bb', 'D9b', 'D9', 'D9s', 'D9ss']), - }, 'E': { - '0': identityMap(['E0bb', 'E0b', 'E0', 'E0s', 'E0ss']), - '1': identityMap(['E1bb', 'E1b', 'E1', 'E1s', 'E1ss']), - '2': identityMap(['E2bb', 'E2b', 'E2', 'E2s', 'E2ss']), - '3': identityMap(['E3bb', 'E3b', 'E3', 'E3s', 'E3ss']), - '4': identityMap(['E4bb', 'E4b', 'E4', 'E4s', 'E4ss']), - '5': identityMap(['E5bb', 'E5b', 'E5', 'E5s', 'E5ss']), - '6': identityMap(['E6bb', 'E6b', 'E6', 'E6s', 'E6ss']), - '7': identityMap(['E7bb', 'E7b', 'E7', 'E7s', 'E7ss']), - '8': identityMap(['E8bb', 'E8b', 'E8', 'E8s', 'E8ss']), - '9': identityMap(['E9bb', 'E9b', 'E9', 'E9s', 'E9ss']), - }, 'F': { - '0': identityMap(['F0bb', 'F0b', 'F0', 'F0s', 'F0ss']), - '1': identityMap(['F1bb', 'F1b', 'F1', 'F1s', 'F1ss']), - '2': identityMap(['F2bb', 'F2b', 'F2', 'F2s', 'F2ss']), - '3': identityMap(['F3bb', 'F3b', 'F3', 'F3s', 'F3ss']), - '4': identityMap(['F4bb', 'F4b', 'F4', 'F4s', 'F4ss']), - '5': identityMap(['F5bb', 'F5b', 'F5', 'F5s', 'F5ss']), - '6': identityMap(['F6bb', 'F6b', 'F6', 'F6s', 'F6ss']), - '7': identityMap(['F7bb', 'F7b', 'F7', 'F7s', 'F7ss']), - '8': identityMap(['F8bb', 'F8b', 'F8', 'F8s', 'F8ss']), - '9': identityMap(['F9bb', 'F9b', 'F9', 'F9s', 'F9ss']), - }, 'G': { - '0': identityMap(['G0bb', 'G0b', 'G0', 'G0s', 'G0ss']), - '1': identityMap(['G1bb', 'G1b', 'G1', 'G1s', 'G1ss']), - '2': identityMap(['G2bb', 'G2b', 'G2', 'G2s', 'G2ss']), - '3': identityMap(['G3bb', 'G3b', 'G3', 'G3s', 'G3ss']), - '4': identityMap(['G4bb', 'G4b', 'G4', 'G4s', 'G4ss']), - '5': identityMap(['G5bb', 'G5b', 'G5', 'G5s', 'G5ss']), - '6': identityMap(['G6bb', 'G6b', 'G6', 'G6s', 'G6ss']), - '7': identityMap(['G7bb', 'G7b', 'G7', 'G7s', 'G7ss']), - '8': identityMap(['G8bb', 'G8b', 'G8', 'G8s', 'G8ss']), - '9': identityMap(['G9bb', 'G9b', 'G9', 'G9s', 'G9ss']), - }, 'A': { - '0': identityMap(['A0bb', 'A0b', 'A0', 'A0s', 'A0ss']), - '1': identityMap(['A1bb', 'A1b', 'A1', 'A1s', 'A1ss']), - '2': identityMap(['A2bb', 'A2b', 'A2', 'A2s', 'A2ss']), - '3': identityMap(['A3bb', 'A3b', 'A3', 'A3s', 'A3ss']), - '4': identityMap(['A4bb', 'A4b', 'A4', 'A4s', 'A4ss']), - '5': identityMap(['A5bb', 'A5b', 'A5', 'A5s', 'A5ss']), - '6': identityMap(['A6bb', 'A6b', 'A6', 'A6s', 'A6ss']), - '7': identityMap(['A7bb', 'A7b', 'A7', 'A7s', 'A7ss']), - '8': identityMap(['A8bb', 'A8b', 'A8', 'A8s', 'A8ss']), - '9': identityMap(['A9bb', 'A9b', 'A9', 'A9s', 'A9ss']), - }, 'B': { - '0': identityMap(['B0bb', 'B0b', 'B0', 'B0s', 'B0ss']), - '1': identityMap(['B1bb', 'B1b', 'B1', 'B1s', 'B1ss']), - '2': identityMap(['B2bb', 'B2b', 'B2', 'B2s', 'B2ss']), - '3': identityMap(['B3bb', 'B3b', 'B3', 'B3s', 'B3ss']), - '4': identityMap(['B4bb', 'B4b', 'B4', 'B4s', 'B4ss']), - '5': identityMap(['B5bb', 'B5b', 'B5', 'B5s', 'B5ss']), - '6': identityMap(['B6bb', 'B6b', 'B6', 'B6s', 'B6ss']), - '7': identityMap(['B7bb', 'B7b', 'B7', 'B7s', 'B7ss']), - '8': identityMap(['B8bb', 'B8b', 'B8', 'B8s', 'B8ss']), - '9': identityMap(['B9bb', 'B9b', 'B9', 'B9s', 'B9ss']), - }, - }, - false, //readonly (no arbitrary text) - )), - new Extension.LabelPart('noteDurations', () => new InputSlotMorph( - null, //text - false, //numeric + basicEnum('instrument', identityMap(INSTRUMENTS)), + basicEnum('keySig', { + 'Major': identityMap(Object.keys(KEYS).filter(x => x.endsWith('Major')).map(x => [x.substring(0, x.length - 5), x])), + 'Minor': identityMap(Object.keys(KEYS).filter(x => x.endsWith('Minor')).map(x => [x.substring(0, x.length - 5), x])), + }), + basicEnum('noteDuration', (base => unionMaps([ + identityMap(base), { - 'Whole': 'Whole', - 'Half': 'Half', - 'Quarter': 'Quarter', - 'Eighth': 'Eighth', - 'Sixteenth': 'Sixteenth', - 'ThirtySecond': 'ThirtySecond', - 'SixtyFourth': 'SixtyFourth', - 'Dotted': identityMap([ - 'DottedWhole', 'DottedHalf', 'DottedQuarter', 'DottedEight', - 'DottedSixteenth', 'DottedThirtySecond', 'DottedSixtyFourth', - ]), - 'DoubleDotted': identityMap([ - 'DoubleDottedWhole', 'DoubleDottedHalf', 'DoubleDottedQuarter', 'DoubleDottedEight', - 'DoubleDottedSixteenth', 'DoubleDottedThirtySecond', 'DoubleDottedSixtyFourth', - ]), + 'Dotted': identityMap(base.map(x => [x, 'Dotted' + x])), + 'DottedDotted': identityMap(base.map(x => [x, 'DottedDotted' + x])), }, - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('chordTypes', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(['Major', 'Minor', 'Diminished', 'Augmented', 'Major 7th', 'Dominant 7th', 'Minor 7th', 'Diminished 7th']), - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('scaleTypes', () => new InputSlotMorph( - null, //text - false, //numeric - identityMap(['Major', 'Minor']), - true, //readonly (no arbitrary text) - )), - new Extension.LabelPart('fxPreset', () => new InputSlotMorph( - null, // text - false, //numeric - identityMap(['Under Water', 'Telephone', 'Cave', 'Fan Blade']), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('onOff', () => new InputSlotMorph( - null, // text - false, //numeric - identityMap(['on', 'off']), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('soundMetaData', () => new InputSlotMorph( - null, // text - false, //numeric - identityMap(['name', 'duration', 'beats', 'samples']), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('webMidiInstrument', () => new InputSlotMorph( - null, // text - false, //numeric - identityMap(midiInstruments), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('fileFormats', () => new InputSlotMorph( - null, // text - false, //numeric - identityMap(['WAV']), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('inputDevice', () => new InputSlotMorph( - null, // text - false, //numeric - identityMap([...midiDevices, ...audioDevices]), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('analysisType', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(Object.keys(availableAnalysisTypes)), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('keySignatures', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(Object.keys(availableKeySignatures)), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('noteModifiers', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(['Piano', 'Forte', 'Accent', 'Staccato', 'Triplet', 'TurnUpper', 'TurnLower']), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('drums', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(['Kick', 'Kick #2','Snare','Open Snare', 'Side Stick Snare','Closed Hi-Hat','Clap','Tom','Floor Tom','Rack Tom', 'Crash','Crash #2','Ride','Ride #2', 'Tamborine','Rest']), - true, // readonly (no arbitrary text) - )), + ]))(['Whole', 'Half', 'Quarter', 'Eighth', 'Sixteenth', 'ThirtySecond', 'SixtyFourth'])), + basicEnum('drum', identityMap(Object.keys(DRUM_TO_NOTE))), + basicEnum('noteModifier', identityMap(['Piano', 'Forte', 'Accent', 'Staccato', 'Triplet', 'TurnUpper', 'TurnLower'])), + basicEnum('audioQuery', identityMap(['name', 'duration', 'samples', 'sample rate'])), + basicEnum('audioEffect', identityMap(Object.keys(EFFECT_INFO))), + basicEnum('audioEffectAug', identityMap([...Object.keys(EFFECT_INFO), 'every', 'every active'])), + basicEnum('audioInput', identityMap([...MIDI_DEVICES, ...INPUT_DEVICES])), + basicEnum('io', identityMap(['input', 'output'])), + basicEnum('audioAnalysis', identityMap(Object.keys(ANALYSIS_INFO))), ]; } - } NetsBloxExtensions.register(MusicApp); }; document.body.appendChild(script); - })(); diff --git a/extensions/BeatBlox/webAudioAPI.js b/extensions/BeatBlox/webAudioAPI.js index e06fa09..2a06662 100644 --- a/extensions/BeatBlox/webAudioAPI.js +++ b/extensions/BeatBlox/webAudioAPI.js @@ -11,34 +11,34 @@ */ const Note = { Rest: 0, C0: 12, C0n: -12, D0bb: 12, C0s: 13, D0b: 13, D0: 14, D0n: -14, C0ss: 14, E0bb: 14, - D0s: 15, E0b: 15, F0bb: 15, E0: 16, E0n: -16, D0ss: 16, F0b: 16, F0: 17, F0n: -17, E0s: 17, G0bb: 17, + D0s: 15, E0b: 15, F0bb: 15, E0: 16, E0n: -16, D0ss: 16, F0b: 16, F0: 17, F0n: -17, E0s: 17, G0bb: 17, F0s: 18, E0ss: 18, G0b: 18, G0: 19, G0n: -19, F0ss: 19, A0bb: 19, G0s: 20, A0b: 20, - A0: 21, A0n: -21, G0ss: 21, B0bb: 21, A0s: 22, B0b: 22, C1bb: 22, B0: 23, B0n: -23, A0ss: 23, C1b: 23, - C1: 24, C1n: -24, B0s: 24, D1bb: 24, C1s: 25, B0ss: 25, D1b: 25, D1: 26, D1n: -26, C1ss: 26, E1bb: 26, - D1s: 27, E1b: 27, F1bb: 27, E1: 28, E1n: -28, D1ss: 28, F1b: 28, F1: 29, F1n: -29, E1s: 29, G1bb: 29, + A0: 21, A0n: -21, G0ss: 21, B0bb: 21, A0s: 22, B0b: 22, C1bb: 22, B0: 23, B0n: -23, A0ss: 23, C1b: 23, + C1: 24, C1n: -24, B0s: 24, D1bb: 24, C1s: 25, B0ss: 25, D1b: 25, D1: 26, D1n: -26, C1ss: 26, E1bb: 26, + D1s: 27, E1b: 27, F1bb: 27, E1: 28, E1n: -28, D1ss: 28, F1b: 28, F1: 29, F1n: -29, E1s: 29, G1bb: 29, F1s: 30, E1ss: 30, G1b: 30, G1: 31, G1n: -31, F1ss: 31, A1bb: 31, G1s: 32, A1b: 32, - A1: 33, A1n: -33, G1ss: 33, B1bb: 33, A1s: 34, B1b: 34, C2bb: 34, B1: 35, B1n: -35, A1ss: 35, C2b: 35, - C2: 36, C2n: -36, B1s: 36, D2bb: 36, C2s: 37, B1ss: 37, D2b: 37, D2: 38, D2n: -38, C2ss: 38, E2bb: 38, - D2s: 39, E2b: 39, F2bb: 39, E2: 40, E2n: -40, D2ss: 40, F2b: 40, F2: 41, F2n: -41, E2s: 41, G2bb: 41, + A1: 33, A1n: -33, G1ss: 33, B1bb: 33, A1s: 34, B1b: 34, C2bb: 34, B1: 35, B1n: -35, A1ss: 35, C2b: 35, + C2: 36, C2n: -36, B1s: 36, D2bb: 36, C2s: 37, B1ss: 37, D2b: 37, D2: 38, D2n: -38, C2ss: 38, E2bb: 38, + D2s: 39, E2b: 39, F2bb: 39, E2: 40, E2n: -40, D2ss: 40, F2b: 40, F2: 41, F2n: -41, E2s: 41, G2bb: 41, F2s: 42, E2ss: 42, G2b: 42, G2: 43, G2n: -43, F2ss: 43, A2bb: 43, G2s: 44, A2b: 44, - A2: 45, A2n: -45, G2ss: 45, B2bb: 45, A2s: 46, B2b: 46, C3bb: 46, B2: 47, B2n: -47, A2ss: 47, C3b: 47, - C3: 48, C3n: -48, B2s: 48, D3bb: 48, C3s: 49, B2ss: 49, D3b: 49, D3: 50, D3n: -50, C3ss: 50, E3bb: 50, - D3s: 51, E3b: 51, F3bb: 51, E3: 52, E3n: -52, D3ss: 52, F3b: 52, F3: 53, F3n: -53, E3s: 53, G3bb: 53, + A2: 45, A2n: -45, G2ss: 45, B2bb: 45, A2s: 46, B2b: 46, C3bb: 46, B2: 47, B2n: -47, A2ss: 47, C3b: 47, + C3: 48, C3n: -48, B2s: 48, D3bb: 48, C3s: 49, B2ss: 49, D3b: 49, D3: 50, D3n: -50, C3ss: 50, E3bb: 50, + D3s: 51, E3b: 51, F3bb: 51, E3: 52, E3n: -52, D3ss: 52, F3b: 52, F3: 53, F3n: -53, E3s: 53, G3bb: 53, F3s: 54, E3ss: 54, G3b: 54, G3: 55, G3n: -55, F3ss: 55, A3bb: 55, G3s: 56, A3b: 56, - A3: 57, A3n: -57, G3ss: 57, B3bb: 57, A3s: 58, B3b: 58, C4bb: 58, B3: 59, B3n: -59, A3ss: 59, C4b: 59, - C4: 60, C4n: -60, B3s: 60, D4bb: 60, C4s: 61, B3ss: 61, D4b: 61, D4: 62, D4n: -62, C4ss: 62, E4bb: 62, - D4s: 63, E4b: 63, F4bb: 63, E4: 64, E4n: -64, D4ss: 64, F4b: 64, F4: 65, F4n: -65, E4s: 65, G4bb: 65, + A3: 57, A3n: -57, G3ss: 57, B3bb: 57, A3s: 58, B3b: 58, C4bb: 58, B3: 59, B3n: -59, A3ss: 59, C4b: 59, + C4: 60, C4n: -60, B3s: 60, D4bb: 60, C4s: 61, B3ss: 61, D4b: 61, D4: 62, D4n: -62, C4ss: 62, E4bb: 62, + D4s: 63, E4b: 63, F4bb: 63, E4: 64, E4n: -64, D4ss: 64, F4b: 64, F4: 65, F4n: -65, E4s: 65, G4bb: 65, F4s: 66, E4ss: 66, G4b: 66, G4: 67, G4n: -67, F4ss: 67, A4bb: 67, G4s: 68, A4b: 68, - A4: 69, A4n: -69, G4ss: 69, B4bb: 69, A4s: 70, B4b: 70, C5bb: 70, B4: 71, B4n: -71, A4ss: 71, C5b: 71, - C5: 72, C5n: -72, B4s: 72, D5bb: 72, C5s: 73, B4ss: 73, D5b: 73, D5: 74, D5n: -74, C5ss: 74, E5bb: 74, - D5s: 75, E5b: 75, F5bb: 75, E5: 76, E5n: -76, D5ss: 76, F5b: 76, F5: 77, F5n: -77, E5s: 77, G5bb: 77, + A4: 69, A4n: -69, G4ss: 69, B4bb: 69, A4s: 70, B4b: 70, C5bb: 70, B4: 71, B4n: -71, A4ss: 71, C5b: 71, + C5: 72, C5n: -72, B4s: 72, D5bb: 72, C5s: 73, B4ss: 73, D5b: 73, D5: 74, D5n: -74, C5ss: 74, E5bb: 74, + D5s: 75, E5b: 75, F5bb: 75, E5: 76, E5n: -76, D5ss: 76, F5b: 76, F5: 77, F5n: -77, E5s: 77, G5bb: 77, F5s: 78, E5ss: 78, G5b: 78, G5: 79, G5n: -79, F5ss: 79, A5bb: 79, G5s: 80, A5b: 80, - A5: 81, A5n: -81, G5ss: 81, B5bb: 81, A5s: 82, B5b: 82, C6bb: 82, B5: 83, B5n: -83, A5ss: 83, C6b: 83, - C6: 84, C6n: -84, B5s: 84, D6bb: 84, C6s: 85, B5ss: 85, D6b: 85, D6: 86, D6n: -86, C6ss: 86, E6bb: 86, - D6s: 87, E6b: 87, F6bb: 87, E6: 88, E6n: -88, D6ss: 88, F6b: 88, F6: 89, F6n: -89, E6s: 89, G6bb: 89, + A5: 81, A5n: -81, G5ss: 81, B5bb: 81, A5s: 82, B5b: 82, C6bb: 82, B5: 83, B5n: -83, A5ss: 83, C6b: 83, + C6: 84, C6n: -84, B5s: 84, D6bb: 84, C6s: 85, B5ss: 85, D6b: 85, D6: 86, D6n: -86, C6ss: 86, E6bb: 86, + D6s: 87, E6b: 87, F6bb: 87, E6: 88, E6n: -88, D6ss: 88, F6b: 88, F6: 89, F6n: -89, E6s: 89, G6bb: 89, F6s: 90, E6ss: 90, G6b: 90, G6: 91, G6n: -91, F6ss: 91, A6bb: 91, G6s: 92, A6b: 92, - A6: 93, A6n: -93, G6ss: 93, B6bb: 93, A6s: 94, B6b: 94, C7bb: 94, B6: 95, B6n: -95, A6ss: 95, C7b: 95, - C7: 96, C7n: -96, B6s: 96, D7bb: 96, C7s: 97, B6ss: 97, D7b: 97, D7: 98, D7n: -98, C7ss: 98, E7bb: 98, + A6: 93, A6n: -93, G6ss: 93, B6bb: 93, A6s: 94, B6b: 94, C7bb: 94, B6: 95, B6n: -95, A6ss: 95, C7b: 95, + C7: 96, C7n: -96, B6s: 96, D7bb: 96, C7s: 97, B6ss: 97, D7b: 97, D7: 98, D7n: -98, C7ss: 98, E7bb: 98, D7s: 99, E7b: 99, F7bb: 99, E7: 100, E7n: -100, D7ss: 100, F7b: 100, F7: 101, F7n: -101, E7s: 101, G7bb: 101, F7s: 102, E7ss: 102, G7b: 102, G7: 103, G7n: -103, F7ss: 103, A7bb: 103, G7s: 104, A7b: 104, A7: 105, A7n: -105, G7ss: 105, B7bb: 105, A7s: 106, B7b: 106, C8bb: 106, B7: 107, B7n: -107, A7ss: 107, C8b: 107, @@ -126,6 +126,64 @@ const ModificationType = { TurnUpper: 124, TurnLower: 125, Glissando: 126, Portamento: 127 }; +/** + * Object representing modification types that conflict with one another. + * @constant {Object} + */ +const ModificationIncompatibilities = { + [ModificationType.Velocity]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.Piano]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.Forte]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.MezzoPiano]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.MezzoForte]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.Pianissimo]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.Fortissimo]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.Pianississimo]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.Fortississimo]: [ModificationType.Velocity, ModificationType.Piano, ModificationType.Forte, ModificationType.MezzoPiano, ModificationType.MezzoForte, + ModificationType.Pianissimo, ModificationType.Fortissimo, ModificationType.Pianississimo, ModificationType.Fortissimo], + [ModificationType.Crescendo]: [ModificationType.Crescendo, ModificationType.Decrescendo, ModificationType.Diminuendo], + [ModificationType.Decrescendo]: [ModificationType.Crescendo, ModificationType.Decrescendo, ModificationType.Diminuendo], + [ModificationType.Diminuendo]: [ModificationType.Crescendo, ModificationType.Decrescendo, ModificationType.Diminuendo], + [ModificationType.Accent]: [ModificationType.Accent, ModificationType.Marcato], + [ModificationType.Marcato]: [ModificationType.Accent, ModificationType.Marcato], + [ModificationType.Staccato]: [ModificationType.Staccato, ModificationType.Staccatissimo, ModificationType.Tenuto], + [ModificationType.Staccatissimo]: [ModificationType.Staccato, ModificationType.Staccatissimo, ModificationType.Tenuto], + [ModificationType.Tenuto]: [ModificationType.Staccato, ModificationType.Staccatissimo, ModificationType.Tenuto], + [ModificationType.OctaveShiftUp]: [ModificationType.OctaveShiftUp, ModificationType.OctaveShiftDown], + [ModificationType.OctaveShiftDown]: [ModificationType.OctaveShiftUp, ModificationType.OctaveShiftDown], + [ModificationType.GraceAcciaccatura]: [ModificationType.GraceAcciaccatura, ModificationType.GraceAppoggiatura], + [ModificationType.GraceAppoggiatura]: [ModificationType.GraceAcciaccatura, ModificationType.GraceAppoggiatura], + [ModificationType.Tuplet]: [ModificationType.Tuplet, ModificationType.Triplet, ModificationType.Quintuplet, ModificationType.Sextuplet, ModificationType.Septuplet], + [ModificationType.Triplet]: [ModificationType.Tuplet, ModificationType.Triplet, ModificationType.Quintuplet, ModificationType.Sextuplet, ModificationType.Septuplet], + [ModificationType.Quintuplet]: [ModificationType.Tuplet, ModificationType.Triplet, ModificationType.Quintuplet, ModificationType.Sextuplet, ModificationType.Septuplet], + [ModificationType.Sextuplet]: [ModificationType.Tuplet, ModificationType.Triplet, ModificationType.Quintuplet, ModificationType.Sextuplet, ModificationType.Septuplet], + [ModificationType.Septuplet]: [ModificationType.Tuplet, ModificationType.Triplet, ModificationType.Quintuplet, ModificationType.Sextuplet, ModificationType.Septuplet], + [ModificationType.TrillUpper]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento], + [ModificationType.TrillLower]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento], + [ModificationType.MordentUpper]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento], + [ModificationType.MordentLower]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento], + [ModificationType.TurnUpper]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento], + [ModificationType.TurnLower]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento], + [ModificationType.Glissando]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento], + [ModificationType.Portamento]: [ModificationType.TrillUpper, ModificationType.TrillLower, ModificationType.MordentUpper, ModificationType.MordentLower, + ModificationType.TurnUpper, ModificationType.TurnLower, ModificationType.Glissando, ModificationType.Portamento] +}; + /** * Object representing a mapping between an acoustic analysis type and its unique internal code. * @constant {Object} @@ -139,7 +197,7 @@ const AnalysisType = { * @constant {Object} */ const EncodingType = { - WAV: 1 + WAV: 1, WEBM: 2 }; /** @@ -916,7 +974,9 @@ class Mordent extends ModificationBase { } if (!Number.isInteger(mordentNote) || (Number(mordentNote) < 1)) throw new WebAudioValueError(`The offset value (${mordentNote}) must be a positive integer > 0`); - const mordentNoteDuration = 60.0 / ((32.0 / this.tempo.beatBase) * this.tempo.beatsPerMinute); + const mordentNoteDuration = (this.unmodifiedDetails.duration >= 16) ? + (60.0 / ((3.0 * this.unmodifiedDetails.duration / this.tempo.beatBase) * this.tempo.beatsPerMinute)) : + (60.0 / ((32.0 / this.tempo.beatBase) * this.tempo.beatsPerMinute)); const primaryNoteDuration = ((this.unmodifiedDetails.duration < 0) ? -this.unmodifiedDetails.duration : (60.0 / ((this.unmodifiedDetails.duration / this.tempo.beatBase) * this.tempo.beatsPerMinute))) - (2 * mordentNoteDuration); @@ -1608,7 +1668,9 @@ class Trill extends ModificationBase { const trill = []; const fullNoteDuration = (this.unmodifiedDetails.duration < 0) ? -this.unmodifiedDetails.duration : (60.0 / ((this.unmodifiedDetails.duration / this.tempo.beatBase) * this.tempo.beatsPerMinute)); - const trillNoteDuration = 60.0 / ((32.0 / this.tempo.beatBase) * this.tempo.beatsPerMinute); + const trillNoteDuration = (this.unmodifiedDetails.duration >= 16) ? + (60.0 / ((3.0 * this.unmodifiedDetails.duration / this.tempo.beatBase) * this.tempo.beatsPerMinute)) : + (60.0 / ((32.0 / this.tempo.beatBase) * this.tempo.beatsPerMinute)); const numNotes = Math.floor(fullNoteDuration / trillNoteDuration); for (let i = 0; i < numNotes; ++i) trill.push(new NoteDetails( @@ -1865,7 +1927,9 @@ class Turn extends ModificationBase { lowerNote -= Turn.lowerOffsetsMajor[lowerNote % 12]; lowerNote += this.key.offsets[lowerNote % 12]; } - const turnNoteDuration = 60.0 / ((32.0 / this.tempo.beatBase) * this.tempo.beatsPerMinute); + const turnNoteDuration = (this.unmodifiedDetails.duration >= 8) ? + (60.0 / ((5.0 * this.unmodifiedDetails.duration / this.tempo.beatBase) * this.tempo.beatsPerMinute)) : + (60.0 / ((32.0 / this.tempo.beatBase) * this.tempo.beatsPerMinute)); const primaryNoteDuration = ((this.unmodifiedDetails.duration < 0) ? -this.unmodifiedDetails.duration : (60.0 / ((this.unmodifiedDetails.duration / this.tempo.beatBase) * this.tempo.beatsPerMinute))) - (4 * turnNoteDuration); @@ -4884,6 +4948,52 @@ class WavFileEncoder extends EncoderBase { } } +var di=Object.defineProperty;var Pe=Object.getOwnPropertySymbols;var li=Object.prototype.hasOwnProperty,ui=Object.prototype.propertyIsEnumerable;var p=Math.pow,ve=(l,e,i)=>e in l?di(l,e,{enumerable:!0,configurable:!0,writable:!0,value:i}):l[e]=i,Me=(l,e)=>{for(var i in e||={})li.call(e,i)&&ve(l,i,e[i]);if(Pe)for(var i of Pe(e))ui.call(e,i)&&ve(l,i,e[i]);return l};var ge=(l,e,i)=>{if(!e.has(l))throw TypeError("Cannot "+i)};var t=(l,e,i)=>(ge(l,e,"read from private field"),i?i.call(l):e.get(l)),a=(l,e,i)=>{if(e.has(l))throw TypeError("Cannot add the same private member more than once");e instanceof WeakSet?e.add(l):e.set(l,i);},u=(l,e,i,s)=>(ge(l,e,"write to private field"),s?s.call(l,i):e.set(l,i),i);var h=(l,e,i)=>(ge(l,e,"access private method"),i);var J=class{constructor(e){this.value=e;}},O=class{constructor(e){this.value=e;}};var we=l=>l<1<<8?1:l<1<<16?2:l<1<<24?3:l{if(l<(1<<7)-1)return 1;if(l<(1<<14)-1)return 2;if(l<(1<<21)-1)return 3;if(l<(1<<28)-1)return 4;if(l{let s=0;for(let r=e;r>c;s<<=1,s|=b;}return s},Fe=(l,e,i,s)=>{for(let r=e;r>i-r-1<>8),t(this,m).setUint8(s++,e);break;case 3:t(this,m).setUint8(s++,1<<5|e>>16),t(this,m).setUint8(s++,e>>8),t(this,m).setUint8(s++,e);break;case 4:t(this,m).setUint8(s++,1<<4|e>>24),t(this,m).setUint8(s++,e>>16),t(this,m).setUint8(s++,e>>8),t(this,m).setUint8(s++,e);break;case 5:t(this,m).setUint8(s++,1<<3|e/p(2,32)&7),t(this,m).setUint8(s++,e>>24),t(this,m).setUint8(s++,e>>16),t(this,m).setUint8(s++,e>>8),t(this,m).setUint8(s++,e);break;case 6:t(this,m).setUint8(s++,1<<2|e/p(2,40)&3),t(this,m).setUint8(s++,e/p(2,32)|0),t(this,m).setUint8(s++,e>>24),t(this,m).setUint8(s++,e>>16),t(this,m).setUint8(s++,e>>8),t(this,m).setUint8(s++,e);break;default:throw new Error("Bad EBML VINT size "+i)}this.write(t(this,A).subarray(0,s));}writeEBML(e){var i,s;if(e!==null)if(e instanceof Uint8Array)this.write(e);else if(Array.isArray(e))for(let r of e)this.writeEBML(r);else if(this.offsets.set(e,this.pos),h(this,yt,ye).call(this,e.id),Array.isArray(e.data)){let r=this.pos,n=e.size===-1?1:(i=e.size)!=null?i:4;e.size===-1?h(this,Lt,_e).call(this,255):this.seek(this.pos+n);let o=this.pos;if(this.dataOffsets.set(e,o),this.writeEBML(e.data),e.size!==-1){let c=this.pos-o,b=this.pos;this.seek(r),this.writeEBMLVarInt(c,n),this.seek(b);}}else if(typeof e.data=="number"){let r=(s=e.size)!=null?s:we(e.data);this.writeEBMLVarInt(r),h(this,yt,ye).call(this,e.data,r);}else typeof e.data=="string"?(this.writeEBMLVarInt(e.data.length),h(this,qt,We).call(this,e.data)):e.data instanceof Uint8Array?(this.writeEBMLVarInt(e.data.byteLength,e.size),this.write(e.data)):e.data instanceof J?(this.writeEBMLVarInt(4),h(this,Yt,Ne).call(this,e.data.value)):e.data instanceof O&&(this.writeEBMLVarInt(8),h(this,Zt,Oe).call(this,e.data.value));}};A=new WeakMap,m=new WeakMap,Lt=new WeakSet,_e=function(e){t(this,m).setUint8(0,e),this.write(t(this,A).subarray(0,1));},Yt=new WeakSet,Ne=function(e){t(this,m).setFloat32(0,e,!1),this.write(t(this,A).subarray(0,4));},Zt=new WeakSet,Oe=function(e){t(this,m).setFloat64(0,e,!1),this.write(t(this,A));},yt=new WeakSet,ye=function(e,i=we(e)){let s=0;switch(i){case 6:t(this,m).setUint8(s++,e/p(2,40)|0);case 5:t(this,m).setUint8(s++,e/p(2,32)|0);case 4:t(this,m).setUint8(s++,e>>24);case 3:t(this,m).setUint8(s++,e>>16);case 2:t(this,m).setUint8(s++,e>>8);case 1:t(this,m).setUint8(s++,e);break;default:throw new Error("Bad UINT size "+i)}this.write(t(this,A).subarray(0,s));},qt=new WeakSet,We=function(e){this.write(new Uint8Array(e.split("").map(i=>i.charCodeAt(0))));};var kt,P,tt,Ct,ke,Kt=class extends $t{constructor(i){super();a(this,Ct);a(this,kt,void 0);a(this,P,new ArrayBuffer(p(2,16)));a(this,tt,new Uint8Array(t(this,P)));u(this,kt,i);}write(i){h(this,Ct,ke).call(this,this.pos+i.byteLength),t(this,tt).set(i,this.pos),this.pos+=i.byteLength;}finalize(){h(this,Ct,ke).call(this,this.pos),t(this,kt).buffer=t(this,P).slice(0,this.pos);}};kt=new WeakMap,P=new WeakMap,tt=new WeakMap,Ct=new WeakSet,ke=function(i){let s=t(this,P).byteLength;for(;so.start-c.start);i.push({start:s[0].start,size:s[0].data.byteLength});for(let o=1;og.start<=s&&sci){for(let g=0;g=i.written[o+1].start;)i.written[o].end=Math.max(i.written[o].end,i.written[o+1].end),i.written.splice(o+1,1);},Xt=new WeakSet,Be=function(i){let r={start:Math.floor(i/t(this,y))*t(this,y),data:new Uint8Array(t(this,y)),written:[],shouldFlush:!1};return t(this,w).push(r),t(this,w).sort((n,o)=>n.start-o.start),t(this,w).indexOf(r)},et=new WeakSet,Bt=function(i=!1){var s,r;for(let n=0;ne.stream.write({type:"write",data:r,position:n}),chunked:!0,chunkSize:(s=e.options)==null?void 0:s.chunkSize}),i);}};var it=1,Et=2,jt=3,mi=1,bi=2,pi=17,Te=p(2,15),zt=p(2,12),$e="https://github.com/Vanilagy/webm-muxer",Ke=6,Ge=5,gi=["strict","offset","permissive"],f,d,at,nt,x,B,$,R,K,z,G,L,S,Y,Z,V,D,F,ot,ht,q,Q,Dt,dt,lt,ie,Le,se,Ye,re,Ze,ae,qe,ne,Qe,oe,Xe,he,je,Pt,Se,vt,Ae,de,Je,_,st,N,rt,le,Ie,ue,ti,ut,Jt,ft,It,fe,ei,T,E,X,Vt,ct,te,ce,ii,Mt,Ue,mt,ee,xe=class{constructor(e){a(this,ie);a(this,se);a(this,re);a(this,ae);a(this,ne);a(this,oe);a(this,he);a(this,Pt);a(this,vt);a(this,de);a(this,_);a(this,N);a(this,le);a(this,ue);a(this,ut);a(this,ft);a(this,fe);a(this,T);a(this,X);a(this,ct);a(this,ce);a(this,Mt);a(this,mt);a(this,f,void 0);a(this,d,void 0);a(this,at,void 0);a(this,nt,void 0);a(this,x,void 0);a(this,B,void 0);a(this,$,void 0);a(this,R,void 0);a(this,K,void 0);a(this,z,void 0);a(this,G,void 0);a(this,L,void 0);a(this,S,void 0);a(this,Y,void 0);a(this,Z,0);a(this,V,[]);a(this,D,[]);a(this,F,[]);a(this,ot,void 0);a(this,ht,void 0);a(this,q,-1);a(this,Q,-1);a(this,Dt,-1);a(this,dt,void 0);a(this,lt,!1);var s;h(this,ie,Le).call(this,e),u(this,f,Me({type:"webm",firstTimestampBehavior:"strict"},e)),this.target=e.target;let i=!!t(this,f).streaming;if(e.target instanceof Wt)u(this,d,new Kt(e.target));else if(e.target instanceof I)u(this,d,(s=e.target.options)!=null&&s.chunked?new wt(e.target,i):new gt(e.target,i));else if(e.target instanceof Ht)u(this,d,new Gt(e.target,i));else throw new Error(`Invalid target: ${e.target}`);h(this,se,Ye).call(this);}addVideoChunk(e,i,s){let r=new Uint8Array(e.byteLength);e.copyTo(r),this.addVideoChunkRaw(r,e.type,s!=null?s:e.timestamp,i);}addVideoChunkRaw(e,i,s,r){if(h(this,mt,ee).call(this),!t(this,f).video)throw new Error("No video track declared.");t(this,ot)===void 0&&u(this,ot,s),r&&h(this,le,Ie).call(this,r);let n=h(this,ft,It).call(this,e,i,s,it);for(t(this,f).video.codec==="V_VP9"&&h(this,ue,ti).call(this,n),u(this,q,n.timestamp);t(this,D).length>0&&t(this,D)[0].timestamp<=n.timestamp;){let o=t(this,D).shift();h(this,T,E).call(this,o,!1);}!t(this,f).audio||n.timestamp<=t(this,Q)?h(this,T,E).call(this,n,!0):t(this,V).push(n),h(this,ut,Jt).call(this),h(this,_,st).call(this);}addAudioChunk(e,i,s){let r=new Uint8Array(e.byteLength);e.copyTo(r),this.addAudioChunkRaw(r,e.type,s!=null?s:e.timestamp,i);}addAudioChunkRaw(e,i,s,r){if(h(this,mt,ee).call(this),!t(this,f).audio)throw new Error("No audio track declared.");t(this,ht)===void 0&&u(this,ht,s),r!=null&&r.decoderConfig&&(t(this,f).streaming?u(this,z,h(this,X,Vt).call(this,r.decoderConfig.description)):h(this,ct,te).call(this,t(this,z),r.decoderConfig.description));let n=h(this,ft,It).call(this,e,i,s,Et);for(u(this,Q,n.timestamp);t(this,V).length>0&&t(this,V)[0].timestamp<=n.timestamp;){let o=t(this,V).shift();h(this,T,E).call(this,o,!0);}!t(this,f).video||n.timestamp<=t(this,q)?h(this,T,E).call(this,n,!t(this,f).video):t(this,D).push(n),h(this,ut,Jt).call(this),h(this,_,st).call(this);}addSubtitleChunk(e,i,s){if(h(this,mt,ee).call(this),!t(this,f).subtitles)throw new Error("No subtitle track declared.");i!=null&&i.decoderConfig&&(t(this,f).streaming?u(this,G,h(this,X,Vt).call(this,i.decoderConfig.description)):h(this,ct,te).call(this,t(this,G),i.decoderConfig.description));let r=h(this,ft,It).call(this,e.body,"key",s!=null?s:e.timestamp,jt,e.duration,e.additions);u(this,Dt,r.timestamp),t(this,F).push(r),h(this,ut,Jt).call(this),h(this,_,st).call(this);}finalize(){if(t(this,lt))throw new Error("Cannot finalize a muxer more than once.");for(;t(this,V).length>0;)h(this,T,E).call(this,t(this,V).shift(),!0);for(;t(this,D).length>0;)h(this,T,E).call(this,t(this,D).shift(),!0);for(;t(this,F).length>0&&t(this,F)[0].timestamp<=t(this,Z);)h(this,T,E).call(this,t(this,F).shift(),!1);if(t(this,f).streaming||h(this,Mt,Ue).call(this),t(this,d).writeEBML(t(this,L)),!t(this,f).streaming){let e=t(this,d).pos,i=t(this,d).pos-t(this,N,rt);t(this,d).seek(t(this,d).offsets.get(t(this,at))+4),t(this,d).writeEBMLVarInt(i,Ke),t(this,$).data=new O(t(this,Z)),t(this,d).seek(t(this,d).offsets.get(t(this,$))),t(this,d).writeEBML(t(this,$)),t(this,x).data[0].data[1].data=t(this,d).offsets.get(t(this,L))-t(this,N,rt),t(this,x).data[1].data[1].data=t(this,d).offsets.get(t(this,nt))-t(this,N,rt),t(this,x).data[2].data[1].data=t(this,d).offsets.get(t(this,B))-t(this,N,rt),t(this,d).seek(t(this,d).offsets.get(t(this,x))),t(this,d).writeEBML(t(this,x)),t(this,d).seek(e);}h(this,_,st).call(this),t(this,d).finalize(),u(this,lt,!0);}};f=new WeakMap,d=new WeakMap,at=new WeakMap,nt=new WeakMap,x=new WeakMap,B=new WeakMap,$=new WeakMap,R=new WeakMap,K=new WeakMap,z=new WeakMap,G=new WeakMap,L=new WeakMap,S=new WeakMap,Y=new WeakMap,Z=new WeakMap,V=new WeakMap,D=new WeakMap,F=new WeakMap,ot=new WeakMap,ht=new WeakMap,q=new WeakMap,Q=new WeakMap,Dt=new WeakMap,dt=new WeakMap,lt=new WeakMap,ie=new WeakSet,Le=function(e){if(e.type&&e.type!=="webm"&&e.type!=="matroska")throw new Error(`Invalid type: ${e.type}`);if(e.firstTimestampBehavior&&!gi.includes(e.firstTimestampBehavior))throw new Error(`Invalid first timestamp behavior: ${e.firstTimestampBehavior}`)},se=new WeakSet,Ye=function(){t(this,d)instanceof U&&t(this,d).target.options.onHeader&&t(this,d).startTrackingWrites(),h(this,re,Ze).call(this),t(this,f).streaming||h(this,oe,Xe).call(this),h(this,he,je).call(this),h(this,ae,qe).call(this),h(this,ne,Qe).call(this),t(this,f).streaming||(h(this,Pt,Se).call(this),h(this,vt,Ae).call(this)),h(this,de,Je).call(this),h(this,_,st).call(this);},re=new WeakSet,Ze=function(){var i;let e={id:440786851,data:[{id:17030,data:1},{id:17143,data:1},{id:17138,data:4},{id:17139,data:8},{id:17026,data:(i=t(this,f).type)!=null?i:"webm"},{id:17031,data:2},{id:17029,data:2}]};t(this,d).writeEBML(e);},ae=new WeakSet,qe=function(){u(this,K,{id:236,size:4,data:new Uint8Array(zt)}),u(this,z,{id:236,size:4,data:new Uint8Array(zt)}),u(this,G,{id:236,size:4,data:new Uint8Array(zt)});},ne=new WeakSet,Qe=function(){u(this,R,{id:21936,data:[{id:21937,data:2},{id:21946,data:2},{id:21947,data:2},{id:21945,data:0}]});},oe=new WeakSet,Xe=function(){let e=new Uint8Array([28,83,187,107]),i=new Uint8Array([21,73,169,102]),s=new Uint8Array([22,84,174,107]),r={id:290298740,data:[{id:19899,data:[{id:21419,data:e},{id:21420,size:5,data:0}]},{id:19899,data:[{id:21419,data:i},{id:21420,size:5,data:0}]},{id:19899,data:[{id:21419,data:s},{id:21420,size:5,data:0}]}]};u(this,x,r);},he=new WeakSet,je=function(){let e={id:17545,data:new O(0)};u(this,$,e);let i={id:357149030,data:[{id:2807729,data:1e6},{id:19840,data:$e},{id:22337,data:$e},t(this,f).streaming?null:e]};u(this,nt,i);},Pt=new WeakSet,Se=function(){let e={id:374648427,data:[]};u(this,B,e),t(this,f).video&&e.data.push({id:174,data:[{id:215,data:it},{id:29637,data:it},{id:131,data:mi},{id:134,data:t(this,f).video.codec},t(this,K),t(this,f).video.frameRate?{id:2352003,data:1e9/t(this,f).video.frameRate}:null,{id:224,data:[{id:176,data:t(this,f).video.width},{id:186,data:t(this,f).video.height},t(this,f).video.alpha?{id:21440,data:1}:null,t(this,R)]}]}),t(this,f).audio&&(u(this,z,t(this,f).streaming?t(this,z)||null:{id:236,size:4,data:new Uint8Array(zt)}),e.data.push({id:174,data:[{id:215,data:Et},{id:29637,data:Et},{id:131,data:bi},{id:134,data:t(this,f).audio.codec},t(this,z),{id:225,data:[{id:181,data:new J(t(this,f).audio.sampleRate)},{id:159,data:t(this,f).audio.numberOfChannels},t(this,f).audio.bitDepth?{id:25188,data:t(this,f).audio.bitDepth}:null]}]})),t(this,f).subtitles&&e.data.push({id:174,data:[{id:215,data:jt},{id:29637,data:jt},{id:131,data:pi},{id:134,data:t(this,f).subtitles.codec},t(this,G)]});},vt=new WeakSet,Ae=function(){let e={id:408125543,size:t(this,f).streaming?-1:Ke,data:[t(this,f).streaming?null:t(this,x),t(this,nt),t(this,B)]};if(u(this,at,e),t(this,d).writeEBML(e),t(this,d)instanceof U&&t(this,d).target.options.onHeader){let{data:i,start:s}=t(this,d).getTrackedWrites();t(this,d).target.options.onHeader(i,s);}},de=new WeakSet,Je=function(){u(this,L,{id:475249515,data:[]});},_=new WeakSet,st=function(){t(this,d)instanceof gt&&t(this,d).flush();},N=new WeakSet,rt=function(){return t(this,d).dataOffsets.get(t(this,at))},le=new WeakSet,Ie=function(e){if(!!e.decoderConfig){if(e.decoderConfig.colorSpace){let i=e.decoderConfig.colorSpace;if(u(this,dt,i),t(this,R).data=[{id:21937,data:{rgb:1,bt709:1,bt470bg:5,smpte170m:6}[i.matrix]},{id:21946,data:{bt709:1,smpte170m:6,"iec61966-2-1":13}[i.transfer]},{id:21947,data:{bt709:1,bt470bg:5,smpte170m:6}[i.primaries]},{id:21945,data:[1,2][Number(i.fullRange)]}],!t(this,f).streaming){let s=t(this,d).pos;t(this,d).seek(t(this,d).offsets.get(t(this,R))),t(this,d).writeEBML(t(this,R)),t(this,d).seek(s);}}e.decoderConfig.description&&(t(this,f).streaming?u(this,K,h(this,X,Vt).call(this,e.decoderConfig.description)):h(this,ct,te).call(this,t(this,K),e.decoderConfig.description));}},ue=new WeakSet,ti=function(e){if(e.type!=="key"||!t(this,dt))return;let i=0;if(W(e.data,0,2)!==2)return;i+=2;let s=(W(e.data,i+1,i+2)<<1)+W(e.data,i+0,i+1);i+=2,s===3&&i++;let r=W(e.data,i+0,i+1);if(i++,r)return;let n=W(e.data,i+0,i+1);if(i++,n!==0)return;i+=2;let o=W(e.data,i+0,i+24);if(i+=24,o!==4817730)return;s>=2&&i++;let c={rgb:7,bt709:2,bt470bg:1,smpte170m:3}[t(this,dt).matrix];Fe(e.data,i+0,i+3,c);},ut=new WeakSet,Jt=function(){let e=Math.min(t(this,f).video?t(this,q):1/0,t(this,f).audio?t(this,Q):1/0),i=t(this,F);for(;i.length>0&&i[0].timestamp<=e;)h(this,T,E).call(this,i.shift(),!t(this,f).video&&!t(this,f).audio);},ft=new WeakSet,It=function(e,i,s,r,n,o){let c=h(this,fe,ei).call(this,s,r);return {data:e,additions:o,type:i,timestamp:c,duration:n,trackNumber:r}},fe=new WeakSet,ei=function(e,i){let s=i===it?t(this,q):i===Et?t(this,Q):t(this,Dt);if(i!==jt){let r=i===it?t(this,ot):t(this,ht);if(t(this,f).firstTimestampBehavior==="strict"&&s===-1&&e!==0)throw new Error(`The first chunk for your media track must have a timestamp of 0 (received ${e}). Non-zero first timestamps are often caused by directly piping frames or audio data from a MediaStreamTrack into the encoder. Their timestamps are typically relative to the age of the document, which is probably what you want. + +If you want to offset all timestamps of a track such that the first one is zero, set firstTimestampBehavior: 'offset' in the options. +If you want to allow non-zero first timestamps, set firstTimestampBehavior: 'permissive'. +`);t(this,f).firstTimestampBehavior==="offset"&&(e-=r);}if(e=1e3;(!t(this,S)||r)&&h(this,ce,ii).call(this,s);let n=s-t(this,Y);if(n<0)return;if(n>=Te)throw new Error(`Current Matroska cluster exceeded its maximum allowed length of ${Te} milliseconds. In order to produce a correct WebM file, you must pass in a key frame at least every ${Te} milliseconds.`);let c=new Uint8Array(4),b=new DataView(c.buffer);if(b.setUint8(0,128|e.trackNumber),b.setInt16(1,n,!1),e.duration===void 0&&!e.additions){b.setUint8(3,Number(e.type==="key")<<7);let g={id:163,data:[c,e.data]};t(this,d).writeEBML(g);}else {let g=Math.floor(e.duration/1e3),pt={id:160,data:[{id:161,data:[c,e.data]},e.duration!==void 0?{id:155,data:g}:null,e.additions?{id:30113,data:e.additions}:null]};t(this,d).writeEBML(pt);}u(this,Z,Math.max(t(this,Z),s));},X=new WeakSet,Vt=function(e){return {id:25506,size:4,data:new Uint8Array(e)}},ct=new WeakSet,te=function(e,i){let s=t(this,d).pos;t(this,d).seek(t(this,d).offsets.get(e));let r=2+4+i.byteLength,n=zt-r;if(n<0){let o=i.byteLength+n;i instanceof ArrayBuffer?i=i.slice(0,o):i=i.buffer.slice(0,o),n=0;}e=[h(this,X,Vt).call(this,i),{id:236,size:4,data:new Uint8Array(n)}],t(this,d).writeEBML(e),t(this,d).seek(s);},ce=new WeakSet,ii=function(e){t(this,S)&&!t(this,f).streaming&&h(this,Mt,Ue).call(this),t(this,d)instanceof U&&t(this,d).target.options.onCluster&&t(this,d).startTrackingWrites(),u(this,S,{id:524531317,size:t(this,f).streaming?-1:Ge,data:[{id:231,data:e}]}),t(this,d).writeEBML(t(this,S)),u(this,Y,e);let i=t(this,d).offsets.get(t(this,S))-t(this,N,rt);t(this,L).data.push({id:187,data:[{id:179,data:e},t(this,f).video?{id:183,data:[{id:247,data:it},{id:241,data:i}]}:null,t(this,f).audio?{id:183,data:[{id:247,data:Et},{id:241,data:i}]}:null]});},Mt=new WeakSet,Ue=function(){let e=t(this,d).pos-t(this,d).dataOffsets.get(t(this,S)),i=t(this,d).pos;if(t(this,d).seek(t(this,d).offsets.get(t(this,S))+4),t(this,d).writeEBMLVarInt(e,Ge),t(this,d).seek(i),t(this,d)instanceof U&&t(this,d).target.options.onCluster){let{data:s,start:r}=t(this,d).getTrackedWrites();t(this,d).target.options.onCluster(s,r,t(this,Y));}},mt=new WeakSet,ee=function(){if(t(this,lt))throw new Error("Cannot add new video or audio chunks after the file has been finalized.")};new TextEncoder; + +/** + * Class containing all Webm/Opus encoding functionality. + * @extends EncoderBase + */ +class WebmOpusEncoder extends EncoderBase { + + /** + * Constructs a new {@link WebmOpusEncoder} object. + */ + constructor() { + super(); + } + + async encode(audioData, encodingOptions) { + const webmMuxer = new xe({ + target: new Wt(), + audio: { codec: 'A_OPUS', numberOfChannels: 1, sampleRate: audioData.sampleRate } + }); + const audioInputData = new ArrayBuffer(4 * audioData.numberOfChannels * audioData.length); + const audioInputDataFloats = new Float32Array(audioInputData); + for (let ch = 0; ch < audioData.numberOfChannels; ++ch) + audioInputDataFloats.set(audioData.getChannelData(ch), ch * audioData.length); + const audioInput = new AudioData({ format: 'f32-planar', + sampleRate: audioData.sampleRate, + numberOfFrames: audioData.length, + numberOfChannels: audioData.numberOfChannels, + timestamp: 0, + data: audioInputData, + transfer: [audioInputData] }); + const bitRate = (encodingOptions && ('bitRate' in encodingOptions)) ? encodingOptions.bitRate : 96000; + const audioEncoder = new AudioEncoder({ output: (chunk, meta) => webmMuxer.addAudioChunk(chunk, meta), error: e => console.error(e) }); + audioEncoder.configure({ codec: 'opus', sampleRate: audioData.sampleRate, numberOfChannels: 1, bitrate: bitRate }); + audioEncoder.encode(audioInput); + await audioEncoder.flush(); + audioEncoder.close(); + webmMuxer.finalize(); + return new Blob([webmMuxer.target.buffer], { type: 'audio/webm;codecs=opus' }); + } +} + /** * Module containing functionality to create and utilize {@link WebAudioAPI} data encoders. * @module Encoder @@ -4892,6 +5002,7 @@ class WavFileEncoder extends EncoderBase { const EncoderClasses = { [EncodingType.WAV]: WavFileEncoder, + [EncodingType.WEBM]: WebmOpusEncoder, }; /** @@ -5199,7 +5310,7 @@ function createTrack(name, audioContext, tempo, keySignature, trackAudioSink) { // Remove any duplicate modifications, keeping only the last one const exists = []; for (let i = modifications.length - 1; i >= 0; --i) - if (exists.includes(modifications[i].type)) + if (ModificationIncompatibilities[modifications[i].type].some(incompatibility => exists.includes(incompatibility))) modifications.splice(i, 1); else exists.push(modifications[i].type); @@ -6773,13 +6884,15 @@ class WebAudioAPI { /** * Decodes an {@link ArrayBuffer} containing an audio clip into an {@link AudioBuffer} object. * - * @param {ArrayBuffer} audioClip - Array buffer containing the audio clip to decode + * @param {ArrayBuffer|Blob} audioClip - Array buffer or blob containing the audio clip to decode * @returns {AudioBuffer} Decoded audio buffer for the specified audio clip + * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer ArrayBuffer} + * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/Blob Blob} */ async decodeAudioClip(audioClip) { - if (!(audioClip instanceof ArrayBuffer)) - throw new WebAudioValueError('The specified audio clip must be of type ArrayBuffer for decoding'); - return await this.#audioContext.decodeAudioData(audioClip); + if (!(audioClip instanceof ArrayBuffer || audioClip instanceof Blob)) + throw new WebAudioValueError('The specified audio clip must be of type ArrayBuffer or Blob for decoding'); + return await this.#audioContext.decodeAudioData(audioClip instanceof ArrayBuffer ? audioClip : await audioClip.arrayBuffer()); } /** @@ -7727,6 +7840,46 @@ class WebAudioAPI { return { getRawData, getDuration, finalize, getEncodedData, notifyWhenComplete }; } + /** + * Encodes a 2D array of floating point `samples` into a {@link https://developer.mozilla.org/en-US/docs/Web/API/Blob Blob} + * containing raw audio data according to the specified `sampleRate` and {@link module:Constants.EncodingType EncodingType} + * specified in the `encodingType` parameter. + * + * @param {number} encodingType - Numeric value corresponding to the desired {@link module:Constants.EncodingType EncodingType} + * @param {number} sampleRate - Sample rate at which the audio data was recorded + * @param {Array>} samples - 2D array of floating point audio samples to encode + * @returns {Blob} Data {@link https://developer.mozilla.org/en-US/docs/Web/API/Blob Blob} containing the newly encoded audio data + * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/Blob Blob} + * @see {@link module:Constants.EncodingType EncodingType} + */ + async encodeAudioAs(encodingType, audioBuffer) { + if (!Object.values(EncodingType).includes(Number(encodingType))) + throw new WebAudioTargetError(`An encoder for the target type identifier (${encodingType}) does not exist`); + if (!(audioBuffer instanceof AudioBuffer)) + throw new WebAudioValueError('The passed-in audio buffer is not a valid AudioBuffer object'); + return await getEncoderFor(Number(encodingType)).encode(audioBuffer); + } + + /** + * Converts a 2D array of floating point `samples` into an {@link AudioBuffer} object with the specified `sampleRate`. + * + * @param {number} sampleRate - Sample rate at which the audio data was recorded + * @param {Array>} samples - 2D array of floating point audio samples to encode + * @returns {AudioBuffer} Newly created {@link AudioBuffer} containing the audio data + * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/AudioBuffer AudioBuffer} + */ + createAudioBufferFromSamples(sampleRate, samples) { + if (!samples || !samples.length || !Array.isArray(samples) || !(Array.isArray(samples[0])) || !samples[0].length) + throw new WebAudioValueError('Cannot encode audio samples as they are not a 2D array of floats'); + const audioBuffer = this.#audioContext.createBuffer(samples.length, samples[0].length, sampleRate); + for (let ch = 0; ch < samples.length; ++ch) { + const channel_data = audioBuffer.getChannelData(ch); + for (let i = 0; i < samples[ch].length; ++i) + channel_data[i] = Math.min(Math.max(samples[ch][i], -1), 1); + } + return audioBuffer; + } + /** * Starts the {@link WebAudioAPI} library and allows audio playback to resume. */