From bdad8eba4f7e1c76fab5db0532602c735408da00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20S=C4=99k?= Date: Thu, 5 Dec 2024 16:25:59 +0100 Subject: [PATCH 1/5] feat: piano based on samples (#221) --- apps/common-app/src/examples/Piano/Piano.tsx | 14 ++- .../src/examples/Piano/PianoNote.tsx | 50 +++++--- apps/common-app/src/examples/Piano/utils.ts | 117 ++++++++++++++++-- 3 files changed, 146 insertions(+), 35 deletions(-) diff --git a/apps/common-app/src/examples/Piano/Piano.tsx b/apps/common-app/src/examples/Piano/Piano.tsx index 0313baca..71604c62 100644 --- a/apps/common-app/src/examples/Piano/Piano.tsx +++ b/apps/common-app/src/examples/Piano/Piano.tsx @@ -1,28 +1,36 @@ import { FC, useEffect, useRef } from 'react'; -import { AudioContext } from 'react-native-audio-api'; +import { AudioContext, AudioBuffer } from 'react-native-audio-api'; import { Container } from '../../components'; -import { KeyName, keyMap } from './utils'; +import { KeyName, sources, keyMap } from './utils'; import PianoNote from './PianoNote'; import Keyboard from './Keyboard'; const Piano: FC = () => { const audioContextRef = useRef(null); const notesRef = useRef>(null); + const bufferListRef = useRef>>({}); const onPressIn = (key: KeyName) => { - notesRef.current?.[key].start(); + notesRef.current?.[key].start(bufferListRef.current); }; const onPressOut = (key: KeyName) => { notesRef.current?.[key].stop(); }; + useEffect(() => {}, []); + useEffect(() => { if (!audioContextRef.current) { audioContextRef.current = new AudioContext(); } + Object.entries(sources).forEach(async ([key, url]) => { + bufferListRef.current[key as KeyName] = + await audioContextRef.current!.decodeAudioDataSource(url); + }); + const newNotes: Partial> = {}; Object.values(keyMap).forEach((key) => { diff --git a/apps/common-app/src/examples/Piano/PianoNote.tsx b/apps/common-app/src/examples/Piano/PianoNote.tsx index 702bbac1..4611c3d8 100644 --- a/apps/common-app/src/examples/Piano/PianoNote.tsx +++ b/apps/common-app/src/examples/Piano/PianoNote.tsx @@ -1,51 +1,61 @@ -import { GainNode, AudioContext, OscillatorNode } from 'react-native-audio-api'; +import { + GainNode, + AudioBuffer, + AudioContext, + AudioBufferSourceNode, +} from 'react-native-audio-api'; -import { Key } from './utils'; +import { Key, getSource } from './utils'; class PianoNote { public audioContext: AudioContext; public key: Key; - private oscillator: OscillatorNode | null = null; private gain: GainNode | null = null; + private bufferSource: AudioBufferSourceNode | null = null; constructor(audioContext: AudioContext, key: Key) { this.audioContext = audioContext; this.key = key; } - start() { - const tNow = this.audioContext.currentTime; + start(bufferList: Record) { + const { buffer /*, playbackRate */ } = getSource(bufferList, this.key); - this.oscillator = this.audioContext.createOscillator(); - this.gain = this.audioContext.createGain(); - this.oscillator.type = 'triangle'; + if (!buffer) { + return; + } - this.oscillator.connect(this.gain); - this.gain.connect(this.audioContext.destination); + const tNow = this.audioContext.currentTime; - this.oscillator.frequency.value = this.key.frequency; + this.bufferSource = this.audioContext.createBufferSource(); + this.bufferSource.buffer = buffer; + this.gain = this.audioContext.createGain(); this.gain.gain.setValueAtTime(0.001, this.audioContext.currentTime); - this.gain.gain.exponentialRampToValueAtTime(1, tNow + 0.1); + this.gain.gain.exponentialRampToValueAtTime( + 1, + this.audioContext.currentTime + 0.01 + ); - this.oscillator.start(tNow); + this.bufferSource.connect(this.gain); + this.gain.connect(this.audioContext.destination); + + this.bufferSource.start(tNow); } stop() { - if (!this.oscillator || !this.gain) { + if (!this.bufferSource || !this.gain) { return; } const tNow = this.audioContext.currentTime; - this.gain.gain.setValueAtTime(1, tNow); - this.gain.gain.exponentialRampToValueAtTime(0.001, tNow + 0.05); - this.gain.gain.setValueAtTime(0, tNow + 0.1); - - this.oscillator.stop(tNow + 0.1); + this.gain.gain.exponentialRampToValueAtTime(0.0001, tNow); + this.gain.gain.setValueAtTime(0, tNow + 0.01); + this.bufferSource.stop(tNow + 0.1); - this.oscillator = null; + this.bufferSource = null; this.gain = null; } } diff --git a/apps/common-app/src/examples/Piano/utils.ts b/apps/common-app/src/examples/Piano/utils.ts index 916430c8..5458b8c8 100644 --- a/apps/common-app/src/examples/Piano/utils.ts +++ b/apps/common-app/src/examples/Piano/utils.ts @@ -1,3 +1,5 @@ +import { AudioBuffer } from 'react-native-audio-api'; + export type KeyName = | 'C4' | 'C#4' @@ -19,17 +21,108 @@ export interface Key { export type KeyMap = Record; +export const sourcesTone: Partial> = { + 'C4': 'https://tonejs.github.io/audio/salamander/C4.mp3', + 'D#4': 'https://tonejs.github.io/audio/salamander/Ds4.mp3', + 'F#4': 'https://tonejs.github.io/audio/salamander/Fs4.mp3', + 'A4': 'https://tonejs.github.io/audio/salamander/A4.mp3', +}; + +export const sourcesTeda: Partial> = { + 'C4': 'https://michalsek.github.io/audio-samples/piano/1_C4.mp3', + 'C#4': 'https://michalsek.github.io/audio-samples/piano/2_Ch4.mp3', + 'D4': 'https://michalsek.github.io/audio-samples/piano/3_D4.mp3', + 'D#4': 'https://michalsek.github.io/audio-samples/piano/4_Dh4.mp3', + 'E4': 'https://michalsek.github.io/audio-samples/piano/5_E4.mp3', + 'F4': 'https://michalsek.github.io/audio-samples/piano/6_F4.mp3', + 'F#4': 'https://michalsek.github.io/audio-samples/piano/7_Fh4.mp3', + 'G4': 'https://michalsek.github.io/audio-samples/piano/8_G4.mp3', + 'G#4': 'https://michalsek.github.io/audio-samples/piano/9_Gh4.mp3', + 'A4': 'https://michalsek.github.io/audio-samples/piano/10_A4.mp3', + 'A#4': 'https://michalsek.github.io/audio-samples/piano/11_Ah4.mp3', + 'B4': 'https://michalsek.github.io/audio-samples/piano/12_B4.mp3', +}; + +export const sources = sourcesTeda; + export const keyMap: KeyMap = { - 'C4': { name: 'C4', frequency: 261.626 }, - 'C#4': { name: 'C#4', frequency: 277.183 }, - 'D4': { name: 'D4', frequency: 293.665 }, - 'D#4': { name: 'D#4', frequency: 311.127 }, - 'E4': { name: 'E4', frequency: 329.628 }, - 'F4': { name: 'F4', frequency: 349.228 }, - 'F#4': { name: 'F#4', frequency: 369.994 }, - 'G4': { name: 'G4', frequency: 391.995 }, - 'G#4': { name: 'G#4', frequency: 415.305 }, - 'A4': { name: 'A4', frequency: 440.0 }, - 'A#4': { name: 'A#4', frequency: 466.164 }, - 'B4': { name: 'B4', frequency: 493.883 }, + 'C4': { + name: 'C4', + frequency: 261.626, + }, + 'C#4': { + name: 'C#4', + frequency: 277.183, + }, + 'D4': { + name: 'D4', + frequency: 293.665, + }, + 'D#4': { + name: 'D#4', + frequency: 311.127, + }, + 'E4': { + name: 'E4', + frequency: 329.628, + }, + 'F4': { + name: 'F4', + frequency: 349.228, + }, + 'F#4': { + name: 'F#4', + frequency: 369.994, + }, + 'G4': { + name: 'G4', + frequency: 391.995, + }, + 'G#4': { + name: 'G#4', + frequency: 415.305, + }, + 'A4': { + name: 'A4', + frequency: 440.0, + }, + 'A#4': { + name: 'A#4', + frequency: 466.164, + }, + 'B4': { + name: 'B4', + frequency: 493.883, + }, } as const; + +export function getClosest(key: Key) { + let closestKey = keyMap.C4; + let minDiff = closestKey.frequency - key.frequency; + + for (const sourcedKeys of Object.keys(sources)) { + const currentKey = keyMap[sourcedKeys as KeyName]; + + const diff = currentKey.frequency - key.frequency; + + if (Math.abs(diff) < Math.abs(minDiff)) { + minDiff = diff; + closestKey = currentKey; + } + } + + return closestKey; +} + +export function getSource(bufferList: Record, key: Key) { + if (key.name in bufferList) { + return { buffer: bufferList[key.name], playbackRate: 1 }; + } + + const closestKey = getClosest(key); + + return { + buffer: bufferList[closestKey.name], + playbackRate: key.frequency / closestKey.frequency, + }; +} From d1038f3221af622c49ab5a17d176057df9c3a0f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20S=C4=99k?= Date: Thu, 5 Dec 2024 16:26:16 +0100 Subject: [PATCH 2/5] Text to speech example (#216) * feat: working on open ai example * feat: cleanup and renaming --- .gitignore | 2 + .../examples/TextToSpeech/TextToSpeech.tsx | 125 ++++++++++++++++++ .../src/examples/TextToSpeech/index.ts | 1 + apps/common-app/src/examples/index.ts | 8 ++ apps/common-app/src/utils/env.ts | 3 + apps/fabric-example/babel.config.js | 2 +- apps/fabric-example/package.json | 1 + yarn.lock | 19 +++ 8 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 apps/common-app/src/examples/TextToSpeech/TextToSpeech.tsx create mode 100644 apps/common-app/src/examples/TextToSpeech/index.ts create mode 100644 apps/common-app/src/utils/env.ts diff --git a/.gitignore b/.gitignore index 94a48503..4bf942bb 100644 --- a/.gitignore +++ b/.gitignore @@ -83,4 +83,6 @@ react-native-audio-api*.tgz # Android .kotlin + +# Envs .env diff --git a/apps/common-app/src/examples/TextToSpeech/TextToSpeech.tsx b/apps/common-app/src/examples/TextToSpeech/TextToSpeech.tsx new file mode 100644 index 00000000..7dd96d1a --- /dev/null +++ b/apps/common-app/src/examples/TextToSpeech/TextToSpeech.tsx @@ -0,0 +1,125 @@ +import React, { useState, FC } from 'react'; +import { AudioBuffer, AudioContext } from 'react-native-audio-api'; +import { ActivityIndicator, TextInput, StyleSheet } from 'react-native'; + +import { Container, Button, Spacer } from '../../components'; +import Env from '../../utils/env'; +import { colors } from '../../styles'; + +async function getOpenAIResponse(input: string, voice: string = 'alloy') { + return await fetch('https://api.openai.com/v1/audio/speech', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${Env.openAiToken}`, + }, + body: JSON.stringify({ + model: 'tts-1-hd', + voice: voice, + input: input, + response_format: 'pcm', + }), + }).then((response) => response.arrayBuffer()); +} + +const openAISampleRate = 24000; +const maxInputValue = 32768.0; + +// TODO: this should ideally be done using native code through .decodeAudioData +function goofyResample( + audioContext: AudioContext, + input: Int16Array +): AudioBuffer { + const scale = audioContext.sampleRate / openAISampleRate; + + const outputBuffer = audioContext.createBuffer( + 2, + input.length * scale, + audioContext.sampleRate + ); + + const processingChannel: Array = []; + const upSampleChannel: Array = []; + + for (let i = 0; i < input.length; i += 1) { + processingChannel[i] = input[i] / maxInputValue; + } + + for (let i = 0; i < input.length; i += 1) { + const isLast = i === input.length - 1; + const currentSample = processingChannel[i]; + const nextSample = isLast ? currentSample : processingChannel[i + 1]; + + upSampleChannel[2 * i] = currentSample; + upSampleChannel[2 * i + 1] = (currentSample + nextSample) / 2; + } + + outputBuffer.copyToChannel(upSampleChannel, 0); + outputBuffer.copyToChannel(upSampleChannel, 1); + + return outputBuffer; +} + +const TextToSpeech: FC = () => { + const [isLoading, setIsLoading] = useState(false); + const [textToRead, setTextToRead] = useState(''); + + const onReadText = async () => { + if (isLoading) { + return; + } + + const aCtx = new AudioContext(); + + setIsLoading(true); + const results = await getOpenAIResponse(textToRead, 'alloy'); + setIsLoading(false); + + const audioBuffer = goofyResample(aCtx, new Int16Array(results)); + const sourceNode = aCtx.createBufferSource(); + const duration = audioBuffer.duration; + const now = aCtx.currentTime; + + sourceNode.buffer = audioBuffer; + + sourceNode.connect(aCtx.destination); + + sourceNode.start(now); + sourceNode.stop(now + duration); + }; + + return ( + + + + +