From 0a18c7e93d7f3e6c55202d5a643f815a5143aaf6 Mon Sep 17 00:00:00 2001 From: Daniel Rochetti Date: Thu, 9 Nov 2023 18:46:19 -0800 Subject: [PATCH] feat: add audio example with whisper (#25) * feat: whisper example * feat: whisper working example * feat: whisper demo impl * fix: fal config proxy * chore: add app router info to readme * fix: proxy url example --- README.md | 2 +- .../app/whisper/page.tsx | 216 ++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 apps/demo-nextjs-app-router/app/whisper/page.tsx diff --git a/README.md b/README.md index c3043e8..89ec8b2 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ You can find a minimal Next.js + fal application examples in [apps/demo-nextjs-p 1. Run `npm install` on the repository root. 2. Create a `.env.local` file and add your API Key as `FAL_KEY` environment variable (or export it any other way your prefer). -3. Run `npx nx serve demo-nextjs-page-router` to start the Next.js app. +3. Run `npx nx serve demo-nextjs-page-router` to start the Next.js app (`demo-nextjs-app-router` is also available if you're interested in the app router version). Check our [Next.js integration docs](https://fal.ai/docs/integrations/nextjs) for more details. diff --git a/apps/demo-nextjs-app-router/app/whisper/page.tsx b/apps/demo-nextjs-app-router/app/whisper/page.tsx new file mode 100644 index 0000000..3d15c1b --- /dev/null +++ b/apps/demo-nextjs-app-router/app/whisper/page.tsx @@ -0,0 +1,216 @@ +'use client'; + +import * as fal from '@fal-ai/serverless-client'; +import { useCallback, useMemo, useState } from 'react'; + +fal.config({ + // credentials: 'FAL_KEY_ID:FAL_KEY_SECRET', + requestMiddleware: fal.withProxy({ + targetUrl: '/api/fal/proxy', + }), +}); + +type ErrorProps = { + error: any; +}; + +function Error(props: ErrorProps) { + if (!props.error) { + return null; + } + return ( +
+ Error {props.error.message} +
+ ); +} + +type RecorderOptions = { + maxDuration?: number; +}; + +function useMediaRecorder({ maxDuration = 10000 }: RecorderOptions = {}) { + const [isRecording, setIsRecording] = useState(false); + const [mediaRecorder, setMediaRecorder] = useState( + null + ); + + const record = useCallback(async () => { + setIsRecording(true); + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const audioChunks: BlobPart[] = []; + const recorder = new MediaRecorder(stream); + setMediaRecorder(recorder); + return new Promise((resolve, reject) => { + try { + recorder.addEventListener('dataavailable', (event) => { + audioChunks.push(event.data); + }); + recorder.addEventListener('stop', async () => { + const fileOptions = { type: 'audio/wav' }; + const audioBlob = new Blob(audioChunks, fileOptions); + const audioFile = new File( + [audioBlob], + `recording_${Date.now()}.wav`, + fileOptions + ); + setIsRecording(false); + resolve(audioFile); + }); + setTimeout(() => { + recorder.stop(); + recorder.stream.getTracks().forEach((track) => track.stop()); + }, maxDuration); + recorder.start(); + } catch (error) { + reject(error); + } + }); + }, [maxDuration]); + + const stopRecording = useCallback(() => { + setIsRecording(false); + mediaRecorder?.stop(); + mediaRecorder?.stream.getTracks().forEach((track) => track.stop()); + }, [mediaRecorder]); + + return { record, stopRecording, isRecording }; +} + +export default function WhisperDemo() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [logs, setLogs] = useState([]); + const [audioFile, setAudioFile] = useState(null); + const [result, setResult] = useState(null); // eslint-disable-line @typescript-eslint/no-explicit-any + const [elapsedTime, setElapsedTime] = useState(0); + + const { record, stopRecording, isRecording } = useMediaRecorder(); + + const reset = () => { + setLoading(false); + setError(null); + setLogs([]); + setElapsedTime(0); + setResult(null); + }; + + const audioFileLocalUrl = useMemo(() => { + if (!audioFile) { + return null; + } + return URL.createObjectURL(audioFile); + }, [audioFile]); + + const transcribeAudio = async (audioFile: File) => { + reset(); + setLoading(true); + const start = Date.now(); + try { + const result = await fal.subscribe('110602490-whisper', { + input: { + file_name: 'recording.wav', + url: audioFile, + }, + pollInterval: 1000, + logs: true, + onQueueUpdate(update) { + setElapsedTime(Date.now() - start); + if ( + update.status === 'IN_PROGRESS' || + update.status === 'COMPLETED' + ) { + setLogs((update.logs || []).map((log) => log.message)); + } + }, + }); + setResult(result); + console.log(result); + } catch (error: any) { + setError(error); + } finally { + setLoading(false); + setElapsedTime(Date.now() - start); + } + }; + return ( +
+
+

+ Hello fal and{' '} + whisper +

+ +
+ + +
+ + {audioFileLocalUrl && ( +
+
+ )} + + + +
+
+

JSON Result

+

+ {`Elapsed Time (seconds): ${(elapsedTime / 1000).toFixed(2)}`} +

+ 
+              {result
+                ? JSON.stringify(result, null, 2)
+                : '// result pending...'}
+
+
+ +
+

Logs

+ 
+              {logs.filter(Boolean).join('\n')}
+
+
+
+
+
+ ); +}