fix: update ui

TEN-framework · Dec 13, 2024 · 53777b8 · 53777b8
1 parent 41a05db
commit 53777b8
Show file tree

Hide file tree

Showing 9 changed files with 72 additions and 59 deletions.
diff --git a/agents/examples/demo/property.json b/agents/examples/demo/property.json
@@ -1013,15 +1013,6 @@
             "extension_group": "transcriber",
             "property": {}
           },
-          {
-            "type": "extension",
-            "name": "bingsearch_tool_python",
-            "addon": "bingsearch_tool_python",
-            "extension_group": "default",
-            "property": {
-              "api_key": "${env:BING_API_KEY|}"
-            }
-          },
           {
             "type": "extension",
             "name": "weatherapi_tool_python",
@@ -1104,10 +1095,6 @@
               {
                 "name": "tool_call",
                 "dest": [
-                  {
-                    "extension_group": "default",
-                    "extension": "bingsearch_tool_python"
-                  },
                   {
                     "extension_group": "default",
                     "extension": "weatherapi_tool_python"
@@ -1153,21 +1140,6 @@
               }
             ]
           },
-          {
-            "extension_group": "default",
-            "extension": "bingsearch_tool_python",
-            "cmd": [
-              {
-                "name": "tool_register",
-                "dest": [
-                  {
-                    "extension_group": "llm",
-                    "extension": "v2v"
-                  }
-                ]
-              }
-            ]
-          },
           {
             "extension_group": "default",
             "extension": "weatherapi_tool_python",

diff --git a/agents/ten_packages/extension/gemini_v2v_python/extension.py b/agents/ten_packages/extension/gemini_v2v_python/extension.py
@@ -237,7 +237,7 @@ async def _loop(self, ten_env: AsyncTenEnv) -> None:
                                         ten_env.log_info(f"Setup complete")
                                     elif response.tool_call:
                                         func_calls = response.tool_call.function_calls
-                                        await self._handle_tool_call(func_calls)
+                                        self.loop.create_task(self._handle_tool_call(func_calls))
                                 except Exception as e:
                                     traceback.print_exc()
                                     ten_env.log_error(f"Failed to handle response")
@@ -421,7 +421,15 @@ def tool_dict(tool: LLMToolMetadata):
 
             return t
 
-        tools = [tool_dict(t) for t in self.available_tools] if len(self.available_tools) > 0 else None
+        tools = [tool_dict(t) for t in self.available_tools] if len(self.available_tools) > 0 else []
+
+        tools.append(Tool(
+            google_search={}
+        ))
+        tools.append(Tool(
+            code_execution={}
+        ))
+
         config = LiveConnectConfig(
             response_modalities=["AUDIO"],
             system_instruction=Content(parts=[Part(text=self.config.prompt)]),

diff --git a/demo/src/app/api/agents/start/graph.ts b/demo/src/app/api/agents/start/graph.ts
@@ -187,7 +187,7 @@ export const getGraphProperties = (
         return {
             "v2v": {
                 "prompt": prompt,
-                "greeting": combined_greeting,
+                // "greeting": combined_greeting,
             }
         }
     }

diff --git a/demo/src/components/Agent/View.tsx b/demo/src/components/Agent/View.tsx
@@ -18,7 +18,7 @@ export default function AgentView(props: AgentViewProps) {
   return (
     <div
       className={cn(
-        "flex h-72 w-full flex-col items-center justify-center px-4 py-5",
+        "flex h-auto w-full flex-col items-center justify-center px-4 py-5",
         "bg-[#0F0F11] bg-gradient-to-br from-[rgba(27,66,166,0.16)] via-[rgba(27,45,140,0.00)] to-[#11174E] shadow-[0px_3.999px_48.988px_0px_rgba(0,7,72,0.12)] backdrop-blur-[7px]",
       )}
     >

diff --git a/demo/src/components/Dynamic/RTCCard.tsx b/demo/src/components/Dynamic/RTCCard.tsx
@@ -54,7 +54,8 @@ export default function RTCCard(props: { className?: string }) {
     rtcManager.on("localTracksChanged", onLocalTracksChanged)
     rtcManager.on("textChanged", onTextChanged)
     rtcManager.on("remoteUserChanged", onRemoteUserChanged)
-    await rtcManager.createTracks()
+    await rtcManager.createCameraTracks()
+    await rtcManager.createMicrophoneTracks()
     await rtcManager.join({
       channel,
       userId,
@@ -135,9 +136,6 @@ export default function RTCCard(props: { className?: string }) {
 
           {/* -- You */}
           <div className="w-full space-y-2 px-2">
-            <div className="flex w-full items-center justify-between py-2">
-              <h2 className="mb-2 text-xl font-semibold">You</h2>
-            </div>
             <MicrophoneBlock audioTrack={audioTrack} />
             <CameraBlock videoTrack={videoTrack} />
           </div>

diff --git a/demo/src/manager/rtc/rtc.ts b/demo/src/manager/rtc/rtc.ts
@@ -5,7 +5,7 @@ import AgoraRTC, {
   IAgoraRTCClient,
   IMicrophoneAudioTrack,
   IRemoteAudioTrack,
-  UID,
+  UID, ICameraVideoTrack,
 } from "agora-rtc-sdk-ng"
 import { ITextItem } from "@/types"
 import { AGEventEmitter } from "../events"
@@ -52,13 +52,17 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
     }
   }
 
-  async createTracks() {
+  async createCameraTracks() {
     try {
       const videoTrack = await AgoraRTC.createCameraVideoTrack()
       this.localTracks.videoTrack = videoTrack
     } catch (err) {
       console.error("Failed to create video track", err)
     }
+    this.emit("localTracksChanged", this.localTracks)
+  }
+
+  async createMicrophoneTracks() {
     try {
       const audioTrack = await AgoraRTC.createMicrophoneAudioTrack()
       this.localTracks.audioTrack = audioTrack
@@ -68,6 +72,22 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
     this.emit("localTracksChanged", this.localTracks)
   }
 
+  async createScreenShareTrack() {
+    try {
+      const screenTrack = await AgoraRTC.createScreenVideoTrack({
+        encoderConfig: {
+          width: 640,
+          height: 480,
+          frameRate: 5,
+        }
+      }, "disable")
+      this.localTracks.screenTrack = screenTrack
+    } catch (err) {
+      console.error("Failed to create screen track", err)
+    }
+    this.emit("localTracksChanged", this.localTracks)
+  }
+
   async publish() {
     const tracks = []
     if (this.localTracks.videoTrack) {
@@ -81,6 +101,26 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
     }
   }
 
+  async switchVideoTrack() {
+    if (this.localTracks.videoTrack) {
+      await this.createScreenShareTrack()
+      if (this.localTracks.screenTrack) {
+        this.client.unpublish(this.localTracks.videoTrack)
+        this.localTracks.videoTrack.close()
+        this.localTracks.videoTrack = undefined
+        await this.client.publish([this.localTracks.screenTrack])
+      }
+    } else if (this.localTracks.screenTrack) {
+      await this.createCameraTracks()
+      if (this.localTracks.videoTrack) {
+        this.client.unpublish(this.localTracks.screenTrack)
+        this.localTracks.screenTrack.close()
+        this.localTracks.screenTrack = undefined
+        await this.client.publish([this.localTracks.videoTrack])
+      }
+    }
+  }
+
   async destroy() {
     this.localTracks?.audioTrack?.close()
     this.localTracks?.videoTrack?.close()

diff --git a/demo/src/manager/rtc/types.ts b/demo/src/manager/rtc/types.ts
@@ -5,6 +5,7 @@ import {
   ICameraVideoTrack,
   IMicrophoneAudioTrack,
   NetworkQuality,
+  ILocalVideoTrack,
 } from "agora-rtc-sdk-ng"
 import { ITextItem } from "@/types"
 
@@ -22,4 +23,5 @@ export interface RtcEvents {
 export interface IUserTracks {
   videoTrack?: ICameraVideoTrack
   audioTrack?: IMicrophoneAudioTrack
+  screenTrack?: ILocalVideoTrack
 }
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -18,7 +18,7 @@ services:
     networks:
       - ten_agent_network
   ten_agent_playground:
-    image: ghcr.io/ten-framework/ten_agent_playground:0.6.2-9-gdaec880
+    image: ghcr.io/ten-framework/ten_agent_playground:0.6.2-22-g41a05db
     container_name: ten_agent_playground
     restart: always
     ports:
@@ -29,7 +29,7 @@ services:
       - AGENT_SERVER_URL=http://ten_agent_dev:8080
       - TEN_DEV_SERVER_URL=http://ten_agent_dev:49483
   ten_agent_demo:
-    image: ghcr.io/ten-framework/ten_agent_demo:0.6.1-40-ge6c28a4
+    image: ghcr.io/ten-framework/ten_agent_demo:0.6.2-22-g41a05db
     container_name: ten_agent_demo
     restart: always
     ports:

diff --git a/playground/src/components/Chat/ChatCfgModuleSelect.tsx b/playground/src/components/Chat/ChatCfgModuleSelect.tsx
@@ -140,29 +140,22 @@ export function RemoteModuleCfgSheet() {
                                         node.addon = value;
                                         node.property = addonModules.find((module) => module.name === value)?.defaultProperty;
 
-                                        if (node.addon === "gemini_v2v_python") {
-                                            GraphEditor.addOrUpdateConnection(
-                                                selectedGraphCopy,
-                                                `${agoraRtcNode.extensionGroup}.${agoraRtcNode.name}`,
-                                                `${node.extensionGroup}.${node.name}`,
-                                                ProtocolLabel.VIDEO_FRAME,
-                                                "video_frame"
-                                            );
-                                            enableRTCVideoSubscribe = true;
-                                        } else {
-                                            GraphEditor.removeConnection(
-                                                selectedGraphCopy,
-                                                `${agoraRtcNode.extensionGroup}.${agoraRtcNode.name}`,
-                                                `${node.extensionGroup}.${node.name}`,
-                                                ProtocolLabel.VIDEO_FRAME,
-                                                "video_frame"
-                                            );
-                                        }
-
                                         needUpdate = true;
                                     }
                                 });
 
+                                const geminiV2VNode = GraphEditor.findNodeByPredicate(selectedGraphCopy, (node) => node.addon === "gemini_v2v_python");
+                                if (geminiV2VNode) {
+                                    GraphEditor.addOrUpdateConnection(
+                                        selectedGraphCopy,
+                                        `${agoraRtcNode.extensionGroup}.${agoraRtcNode.name}`,
+                                        `${geminiV2VNode.extensionGroup}.${geminiV2VNode.name}`,
+                                        ProtocolLabel.VIDEO_FRAME,
+                                        "video_frame"
+                                    );
+                                    enableRTCVideoSubscribe = true;
+                                }
+
                                 // Identify removed tools and process them
                                 const currentToolsInGraph = nodes
                                     .filter((node) => installedAndRegisteredToolModules.map((module) => module.name).includes(node.addon))