new VRMCompanion, mouth lip audio sync, Scene cleanup

package.json

@@ -1,7 +1,7 @@
 {
   "name": "lala-companion",
   "productName": "lala-companion",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "description": "3D personified desktop assistants, tuned for you, powered by AI vision and voice.",
   "main": ".webpack/main",
   "scripts": {
@@ -55,6 +55,7 @@
     "@pixiv/three-vrm": "^2.0.10",
     "@react-three/drei": "^9.97.3",
     "@react-three/fiber": "^8.15.16",
+    "@react-three/rapier": "^1.2.1",
     "ai": "^2.2.34",
     "dotenv": "^16.4.1",
     "electron-squirrel-startup": "^1.0.0",

src/Scene.tsx

@@ -0,0 +1,84 @@
+import { VRM } from "@pixiv/three-vrm";
+import { OrbitControls } from "@react-three/drei";
+import { Canvas } from "@react-three/fiber";
+import { RapierRigidBody } from "@react-three/rapier";
+import React, { useRef, useEffect, MutableRefObject } from "react";
+import { Mesh } from "three";
+import { animations } from "./constants/animations";
+import VrmCompanion from "./components/VRMCompanion";
+
+interface SceneProps {
+  virtualText: string;
+  voiceUrl: string;
+  audioRef?: MutableRefObject<HTMLAudioElement>;
+  onSpeakStart?: () => void;
+  onSpeakEnd?: () => void;
+}
+
+const Scene = ({
+  virtualText,
+  voiceUrl,
+  onSpeakStart,
+  onSpeakEnd,
+}: SceneProps) => {
+  const vrmRef = useRef<VRM>(null);
+  const vrmMeshRef = useRef<Mesh>(null);
+  const vrmPhysicsRef = useRef<RapierRigidBody>(null);
+
+  useEffect(() => {
+    if (virtualText) {
+      (vrmRef as any)?.current?.setText?.(virtualText);
+    }
+  }, [virtualText]);
+
+  useEffect(() => {
+    const speak = async () => {
+      if (voiceUrl) {
+        onSpeakStart?.();
+        await (vrmRef as any)?.current?.talk?.(voiceUrl);
+        onSpeakEnd?.();
+      }
+    };
+    speak();
+  }, [voiceUrl]);
+
+  return (
+    <>
+      <Canvas
+        style={{
+          zIndex: 1,
+          height: "100vh",
+          width: "100%",
+        }}
+      >
+        <OrbitControls
+          makeDefault
+          minDistance={0.75}
+          maxDistance={1.5}
+          enableDamping
+        />
+        <ambientLight />
+
+        {/* left */}
+        <pointLight position={[1, 2, 1]} intensity={2.5} castShadow />
+
+        {/* right */}
+        <pointLight position={[-1, 2, 1]} intensity={2.5} castShadow />
+
+        <VrmCompanion
+          ref={vrmRef}
+          meshRef={vrmMeshRef}
+          physicsRef={vrmPhysicsRef}
+          vrmUrl={"https://lalaland.chat/vrms/purple-girl.vrm"}
+          animations={animations}
+          scale={[1, 1, 1]}
+          position={[0, -1, 0]}
+          rotation={[0, Math.PI, 0]}
+          isStaticPosition
+        />
+      </Canvas>
+    </>
+  );
+};
+
+export default Scene;