From 3f0def4ea856cbe23f1f6004bd2446bfd0ba03ed Mon Sep 17 00:00:00 2001
From: Christopher Trimboli <cjft@trimboli.io>
Date: Sun, 10 Dec 2023 10:07:25 -0700
Subject: [PATCH] add whisper voice to interact

---
 package-lock.json               |  33 ++++++++++-
 package.json                    |   7 ++-
 src/components/VRMCompanion.tsx |  10 +++-
 src/overlay/Overlay.tsx         | 100 +++++++++++++++++++++++++++++++-
 4 files changed, 143 insertions(+), 7 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index e7bd356..cb91b37 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "lala-companion",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "lala-companion",
-      "version": "0.0.4",
+      "version": "0.0.5",
       "license": "AGPL-3.0",
       "dependencies": {
         "@pixiv/three-vrm": "^2.0.7",
@@ -15,10 +15,12 @@
         "ai": "^2.2.28",
         "dotenv": "^16.3.1",
         "electron-squirrel-startup": "^1.0.0",
+        "hark": "^1.2.3",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "three": "^0.159.0",
-        "update-electron-app": "^3.0.0"
+        "update-electron-app": "^3.0.0",
+        "wavesurfer.js": "^7.5.1"
       },
       "devDependencies": {
         "@electron-forge/cli": "^7.2.0",
@@ -29,6 +31,7 @@
         "@electron-forge/plugin-auto-unpack-natives": "^7.2.0",
         "@electron-forge/plugin-vite": "^7.2.0",
         "@electron-forge/publisher-github": "^7.2.0",
+        "@types/hark": "^1.2.5",
         "@types/react-dom": "^18.2.17",
         "@typescript-eslint/eslint-plugin": "^5.62.0",
         "@typescript-eslint/parser": "^5.62.0",
@@ -1586,6 +1589,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/hark": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/@types/hark/-/hark-1.2.5.tgz",
+      "integrity": "sha512-PyVF1PIrbKHeC1+6W/HZoMT40DLoQ0d1FAxuv7Vcg+XBfmpio8qr+y29chxYutJpra/K2MWXiXm/lScEv2E9pg==",
+      "dev": true
+    },
     "node_modules/@types/http-cache-semantics": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
@@ -5257,6 +5266,14 @@
       "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
       "dev": true
     },
+    "node_modules/hark": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/hark/-/hark-1.2.3.tgz",
+      "integrity": "sha512-u68vz9SCa38ESiFJSDjqK8XbXqWzyot7Cj6Y2b6jk2NJ+II3MY2dIrLMg/kjtIAun4Y1DHF/20hfx4rq1G5GMg==",
+      "dependencies": {
+        "wildemitter": "^1.2.0"
+      }
+    },
     "node_modules/has-bigints": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz",
@@ -9310,6 +9327,11 @@
         }
       }
     },
+    "node_modules/wavesurfer.js": {
+      "version": "7.5.1",
+      "resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.5.1.tgz",
+      "integrity": "sha512-lFDzf9+4TLCiDoGTrwrhO7zzOG3FvDSA2uo2aE19TpTGL1tSaB0tDuRM9VyeQXJyKujHNgibWhn+pCbj6oa8UA=="
+    },
     "node_modules/wcwidth": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
@@ -9403,6 +9425,11 @@
         "string-width": "^1.0.2 || 2 || 3 || 4"
       }
     },
+    "node_modules/wildemitter": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/wildemitter/-/wildemitter-1.2.1.tgz",
+      "integrity": "sha512-UMmSUoIQSir+XbBpTxOTS53uJ8s/lVhADCkEbhfRjUGFDPme/XGOb0sBWLx5sTz7Wx/2+TlAw1eK9O5lw5PiEw=="
+    },
     "node_modules/word-wrap": {
       "version": "1.2.5",
       "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
diff --git a/package.json b/package.json
index 1f7c9aa..8ba992a 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "lala-companion",
   "productName": "lala-companion",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "3D personified desktop assistants, tuned for you, powered by AI vision and voice.",
   "main": ".vite/build/main.js",
   "scripts": {
@@ -30,6 +30,7 @@
     "@electron-forge/plugin-auto-unpack-natives": "^7.2.0",
     "@electron-forge/plugin-vite": "^7.2.0",
     "@electron-forge/publisher-github": "^7.2.0",
+    "@types/hark": "^1.2.5",
     "@types/react-dom": "^18.2.17",
     "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "^5.62.0",
@@ -46,9 +47,11 @@
     "ai": "^2.2.28",
     "dotenv": "^16.3.1",
     "electron-squirrel-startup": "^1.0.0",
+    "hark": "^1.2.3",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "three": "^0.159.0",
-    "update-electron-app": "^3.0.0"
+    "update-electron-app": "^3.0.0",
+    "wavesurfer.js": "^7.5.1"
   }
 }
diff --git a/src/components/VRMCompanion.tsx b/src/components/VRMCompanion.tsx
index 140f455..fa75ddd 100644
--- a/src/components/VRMCompanion.tsx
+++ b/src/components/VRMCompanion.tsx
@@ -48,12 +48,16 @@ interface VrmCompanionProps {
   virtualText: string;
   voiceUrl: string;
   audioRef?: MutableRefObject<HTMLAudioElement>;
+  onSpeakStart?: () => void;
+  onSpeakEnd?: () => void;
 }
 
 const VrmCompanion = ({
   virtualText,
   voiceUrl,
   audioRef,
+  onSpeakStart,
+  onSpeakEnd,
 }: VrmCompanionProps) => {
   const [gltf, setGltf] = useState(null);
   const [animationMixer, setAnimationMixer] = useState<AnimationMixer>(null);
@@ -199,6 +203,7 @@ const VrmCompanion = ({
   useEffect(() => {
     const main = async () => {
       if (voiceUrl) {
+        onSpeakStart?.();
         audioRef.current.src = voiceUrl;
         audioRef.current.play();
 
@@ -222,6 +227,7 @@ const VrmCompanion = ({
         lipsAction.play();
 
         audioRef.current.onended = async () => {
+          onSpeakEnd?.();
           animationMixer.clipAction(talkClip).fadeOut(2);
           lipsAction.fadeOut(1);
           const randomIdle = getRandomAnimation("idle");
@@ -263,7 +269,7 @@ const VrmCompanion = ({
   );
 };
 
-const Scene = ({ virtualText, voiceUrl }: VrmCompanionProps) => {
+const Scene = ({ virtualText, voiceUrl, onSpeakStart, onSpeakEnd }: VrmCompanionProps) => {
   const audioRef = useRef<HTMLAudioElement>(null);
 
   return (
@@ -287,6 +293,8 @@ const Scene = ({ virtualText, voiceUrl }: VrmCompanionProps) => {
           virtualText={virtualText}
           voiceUrl={voiceUrl}
           audioRef={audioRef}
+          onSpeakStart={onSpeakStart}
+          onSpeakEnd={onSpeakEnd}
         />
       </Canvas>
       <audio autoPlay ref={audioRef} src={""} />
diff --git a/src/overlay/Overlay.tsx b/src/overlay/Overlay.tsx
index dd813e8..1ed45dc 100644
--- a/src/overlay/Overlay.tsx
+++ b/src/overlay/Overlay.tsx
@@ -1,10 +1,14 @@
 import React, { useCallback, useEffect, useState } from "react";
 import VRMCompanion from "../components/VRMCompanion";
 import { useChat } from "../../node_modules/ai/react/dist/index";
+import hark from "hark";
+import WaveSurfer from "wavesurfer.js";
+import RecordPlugin from "wavesurfer.js/dist/plugins/record";
 
 const Overlay = () => {
   const [voiceUrl, setVoiceUrl] = useState<string>("");
   const [recentResponse, setRecentResponse] = useState<string>("");
+  const [isLalaSpeaking, setIsLalaSpeaking] = useState<boolean>(false);
 
   const getVoiceAudio = useCallback(async (text: string) => {
     try {
@@ -52,6 +56,94 @@ const Overlay = () => {
     });
   }, []);
 
+  // whisper chunking magic here
+  useEffect(() => {
+    let stream: MediaStream = null;
+    let speechEvents: hark.Harker = null;
+    let wavesurfer: WaveSurfer = null;
+    let recorder: RecordPlugin = null;
+    let isUserSpeaking = false;
+    let isLoading = false;
+
+    const main = async () => {
+      stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      speechEvents = hark(stream);
+
+      wavesurfer = WaveSurfer.create({
+        container: "#recorder",
+        height: 0,
+      });
+
+      recorder = wavesurfer.registerPlugin(
+        RecordPlugin.create({
+          scrollingWaveform: true,
+          renderRecordedAudio: false,
+        })
+      );
+
+      speechEvents.on("speaking", () => {
+        if (isLalaSpeaking || isLoading) return;
+        isUserSpeaking = true;
+        recorder.startRecording();
+        console.log("Started speaking");
+      });
+
+      speechEvents.on("stopped_speaking", () => {
+        if (isLalaSpeaking) return;
+        isLoading = true;
+        recorder.stopRecording();
+        isUserSpeaking = false;
+        console.log("Stopped speaking");
+      });
+
+      recorder.on("record-end", async (blob) => {
+        console.log("recording stopped");
+        const formData = new FormData();
+
+        const file = new File([blob], "voice.wav", {
+          type: "audio/wav",
+        });
+
+        console.log(file);
+
+        formData.append("file", file);
+
+        const whisperResp = await fetch(
+          "https://lalaland.chat/api/magic/whisper",
+          {
+            method: "POST",
+            body: formData,
+          }
+        );
+
+        if (whisperResp.ok) {
+          const whisperText = await whisperResp.json();
+          console.log(whisperText);
+          await append({
+            role: "user",
+            content: whisperText,
+          });
+          setTimeout(() => {
+            isLoading = false;
+          }, 5000);
+        } else {
+          console.log("error whispering", whisperResp);
+          isLoading = false;
+        }
+      });
+    };
+    main();
+
+    return () => {
+      stream?.getTracks().forEach((track) => track.stop());
+      speechEvents?.stop();
+      wavesurfer?.destroy();
+      recorder?.destroy();
+      isUserSpeaking = false;
+      isLoading = false;
+    };
+  }, [isLalaSpeaking]);
+
   return (
     <div
       style={{
@@ -59,7 +151,13 @@ const Overlay = () => {
         width: "100%",
       }}
     >
-      <VRMCompanion virtualText={recentResponse} voiceUrl={voiceUrl} />
+      <VRMCompanion
+        virtualText={recentResponse}
+        voiceUrl={voiceUrl}
+        onSpeakStart={() => setIsLalaSpeaking(true)}
+        onSpeakEnd={() => setIsLalaSpeaking(false)}
+      />
+      <div id="recorder" />
     </div>
   );
 };