From f55cc7ecd787059d1678049e737f5754c75a99ff Mon Sep 17 00:00:00 2001 From: zhangqianze Date: Fri, 13 Dec 2024 15:14:32 +0800 Subject: [PATCH] feat: adjust demo graphs --- agents/examples/demo/property.json | 1180 +++++------------ demo/src/app/api/agents/start/graph.ts | 39 +- demo/src/common/constant.ts | 14 +- playground/src/common/graph.ts | 39 +- .../components/Chat/ChatCfgModuleSelect.tsx | 160 +-- 5 files changed, 442 insertions(+), 990 deletions(-) diff --git a/agents/examples/demo/property.json b/agents/examples/demo/property.json index 24efddd9..59d3679b 100644 --- a/agents/examples/demo/property.json +++ b/agents/examples/demo/property.json @@ -4,13 +4,13 @@ "predefined_graphs": [ { "name": "va_openai_azure", - "auto_start": false, + "auto_start": true, "nodes": [ { "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", "property": { "app_id": "${env:AGORA_APP_ID}", "token": "", @@ -23,39 +23,35 @@ "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" + "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}", + "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}", + "agora_asr_session_control_file_path": "session_control.conf", + "subscribe_video_pix_fmt": 4, + "subscribe_video": true } }, { "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", + "name": "llm", + "addon": "openai_chatgpt_python", "extension_group": "chatgpt", - "addon": "openai_chatgpt", - "name": "openai_chatgpt", "property": { - "base_url": "${env:OPENAI_API_BASE}", "api_key": "${env:OPENAI_API_KEY}", + "base_url": "", "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", + "greeting": "TEN Agent connected. How can I help you today?", + "max_memory_length": 10, "max_tokens": 512, + "model": "${env:OPENAI_MODEL}", "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "max_memory_length": 10 + "proxy_url": "${env:OPENAI_PROXY_URL}" } }, { "type": "extension", - "extension_group": "tts", + "name": "tts", "addon": "azure_tts", - "name": "azure_tts", + "extension_group": "tts", "property": { "azure_subscription_key": "${env:AZURE_TTS_KEY}", "azure_subscription_region": "${env:AZURE_TTS_REGION}", @@ -64,524 +60,88 @@ }, { "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - } - ], - "connections": [ - { + "name": "interrupt_detector", + "addon": "interrupt_detector_python", "extension_group": "default", - "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] + "property": {} }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - } - ] - }, - { - "name": "camera_va_openai_azure", - "auto_start": false, - "nodes": [ { "type": "extension", - "extension_group": "default", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "subscribe_video": true, - "publish_audio": true, - "publish_data": true, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf", - "subscribe_video_pix_fmt": 4 - } + "name": "message_collector", + "addon": "message_collector", + "extension_group": "transcriber", + "property": {} }, { "type": "extension", + "name": "weatherapi_tool_python", + "addon": "weatherapi_tool_python", "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt_python", - "name": "openai_chatgpt", "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "gpt-4o", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", - "max_memory_length": 10, - "enable_tools": true + "api_key": "${env:WEATHERAPI_API_KEY|}" } }, { "type": "extension", - "extension_group": "chatgpt", + "name": "vision_tool_python", "addon": "vision_tool_python", - "name": "vision_tool" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "weatherapi_tool_python", - "name": "weatherapi_tool_python", - "property": { - "api_key": "${env:WEATHERAPI_API_KEY}" - } + "extension_group": "default", + "property": {} }, { "type": "extension", - "extension_group": "chatgpt", - "addon": "bingsearch_tool_python", "name": "bingsearch_tool_python", + "addon": "bingsearch_tool_python", + "extension_group": "default", "property": { - "api_key": "${env:BING_API_KEY}" - } - }, - { - "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", - "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + "api_key": "${env:BING_API_KEY|}" } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" } ], "connections": [ { "extension_group": "default", "extension": "agora_rtc", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "video_frame": [ - { - "name": "video_frame", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "vision_tool" - } - ] - } - ], "cmd": [ { "name": "on_user_joined", "dest": [ { "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - }, - { - "extension_group": "transcriber", - "extension": "message_collector" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "tts", - "extension": "azure_tts" - } - ] - }, - { - "name": "tool_call", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "vision_tool" - }, - { - "extension_group": "chatgpt", - "extension": "weatherapi_tool_python" - }, - { - "extension_group": "chatgpt", - "extension": "bingsearch_tool_python" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "vision_tool", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "weatherapi_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - }, - { - "extension_group": "chatgpt", - "extension": "bingsearch_tool_python", - "cmd": [ - { - "name": "tool_register", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ - { - "name": "pcm_frame", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ], - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ - { - "name": "data", - "dest": [ - { - "extension_group": "default", - "extension": "agora_rtc" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", - "cmd": [ - { - "name": "flush", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" + "extension": "llm" } ] - } - ] - } - ] - }, - { - "name": "va_openai_v2v", - "auto_start": true, - "nodes": [ - { - "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", - "name": "agora_rtc", - "property": { - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "channel": "ten_agent_test", - "stream_id": 1234, - "remote_stream_id": 123, - "subscribe_audio": true, - "subscribe_video": true, - "publish_audio": true, - "publish_data": true, - "subscribe_audio_sample_rate": 24000, - "subscribe_video_pix_fmt": 4 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_v2v_python", - "name": "openai_v2v_python", - "property": { - "api_key": "${env:OPENAI_REALTIME_API_KEY}", - "temperature": 0.9, - "model": "gpt-4o-realtime-preview", - "max_tokens": 2048, - "voice": "alloy", - "language": "en-US", - "server_vad": true, - "dump": true, - "greeting": "TEN Agent connected. How can I help you today?", - "max_history": 10 - } - }, - { - "type": "extension", - "extension_group": "llm", - "addon": "openai_chatgpt_python", - "name": "openai_chatgpt", - "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", - "max_memory_length": 10, - "enable_tools": true - } - }, - { - "type": "extension", - "extension_group": "transcriber", - "addon": "message_collector", - "name": "message_collector" - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "weatherapi_tool_python", - "name": "weatherapi_tool_python", - "property": { - "api_key": "${env:WEATHERAPI_API_KEY}" - } - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "bingsearch_tool_python", - "name": "bingsearch_tool_python", - "property": { - "api_key": "${env:BING_API_KEY}" - } - }, - { - "type": "extension", - "extension_group": "tools", - "addon": "vision_analyze_tool_python", - "name": "vision_analyze_tool_python" - } - ], - "connections": [ - { - "extension_group": "rtc", - "extension": "agora_rtc", - "audio_frame": [ + }, { - "name": "pcm_frame", + "name": "on_user_left", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "chatgpt", + "extension": "llm" + } + ] + }, + { + "name": "on_connection_failure", + "dest": [ + { + "extension_group": "chatgpt", + "extension": "llm" + } + ] + } + ], + "data": [ + { + "name": "text_data", + "dest": [ + { + "extension_group": "default", + "extension": "interrupt_detector" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" } ] } @@ -591,97 +151,111 @@ "name": "video_frame", "dest": [ { - "extension_group": "tools", - "extension": "vision_analyze_tool_python" + "extension_group": "default", + "extension": "vision_tool_python" } ] } - ], + ] + }, + { + "extension_group": "chatgpt", + "extension": "llm", "cmd": [ { - "name": "on_user_joined", + "name": "flush", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "tts", + "extension": "tts" } ] }, { - "name": "on_user_left", + "name": "tool_call", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "default", + "extension": "weatherapi_tool_python" + }, + { + "extension_group": "default", + "extension": "vision_tool_python" + }, + { + "extension_group": "default", + "extension": "bingsearch_tool_python" } ] } - ] - }, - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "cmd": [ + ], + "data": [ { - "name": "tool_register", + "name": "text_data", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "tts", + "extension": "tts" + }, + { + "extension_group": "transcriber", + "extension": "message_collector" } ] } ] }, { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "cmd": [ + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ { - "name": "tool_register", + "name": "data", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "default", + "extension": "agora_rtc" } ] } ] }, { - "extension_group": "tools", - "extension": "vision_analyze_tool_python", + "extension_group": "tts", + "extension": "tts", "cmd": [ { - "name": "tool_register", + "name": "flush", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "default", + "extension": "agora_rtc" } ] - }, + } + ], + "audio_frame": [ { - "name": "chat_completion_call", + "name": "pcm_frame", "dest": [ { - "extension_group": "llm", - "extension": "openai_chatgpt" + "extension_group": "default", + "extension": "agora_rtc" } ] } ] }, { - "extension_group": "llm", - "extension": "openai_v2v_python", - "audio_frame": [ + "extension_group": "default", + "extension": "interrupt_detector", + "cmd": [ { - "name": "pcm_frame", + "name": "flush", "dest": [ { - "extension_group": "rtc", - "extension": "agora_rtc" + "extension_group": "chatgpt", + "extension": "llm" } ] } @@ -691,51 +265,53 @@ "name": "text_data", "dest": [ { - "extension_group": "transcriber", - "extension": "message_collector" + "extension_group": "chatgpt", + "extension": "llm" } ] } - ], + ] + }, + { + "extension_group": "default", + "extension": "weatherapi_tool_python", "cmd": [ { - "name": "flush", + "name": "tool_register", "dest": [ { - "extension_group": "rtc", - "extension": "agora_rtc" + "extension_group": "chatgpt", + "extension": "llm" } ] - }, + } + ] + }, + { + "extension_group": "default", + "extension": "vision_tool_python", + "cmd": [ { - "name": "tool_call", + "name": "tool_register", "dest": [ { - "extension_group": "tools", - "extension": "vision_analyze_tool_python" - }, - { - "extension_group": "tools", - "extension": "weatherapi_tool_python" - }, - { - "extension_group": "tools", - "extension": "bingsearch_tool_python" + "extension_group": "chatgpt", + "extension": "llm" } ] } ] }, { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ + "extension_group": "default", + "extension": "bingsearch_tool_python", + "cmd": [ { - "name": "data", + "name": "tool_register", "dest": [ { - "extension_group": "rtc", - "extension": "agora_rtc" + "extension_group": "chatgpt", + "extension": "llm" } ] } @@ -744,14 +320,14 @@ ] }, { - "name": "va_openai_v2v_fish", - "auto_start": false, + "name": "va_openai_v2v", + "auto_start": true, "nodes": [ { "type": "extension", - "extension_group": "rtc", - "addon": "agora_rtc", "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "rtc", "property": { "app_id": "${env:AGORA_APP_ID}", "token": "", @@ -761,69 +337,49 @@ "subscribe_audio": true, "publish_audio": true, "publish_data": true, - "subscribe_audio_sample_rate": 24000, - "enable_agora_asr": true, - "agora_asr_vendor_name": "microsoft", - "agora_asr_language": "en-US", - "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", - "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf" + "subscribe_audio_sample_rate": 24000 } }, { "type": "extension", - "extension_group": "llm", + "name": "v2v", "addon": "openai_v2v_python", - "name": "openai_v2v_python", + "extension_group": "llm", "property": { "api_key": "${env:OPENAI_REALTIME_API_KEY}", "temperature": 0.9, "model": "gpt-4o-realtime-preview", "max_tokens": 2048, - "audio_out": false, - "input_transcript": false, + "voice": "alloy", "language": "en-US", "server_vad": true, "dump": true, - "greeting": "TEN Agent connected. How can I help you today?", "max_history": 10 } }, { "type": "extension", - "extension_group": "tts", - "addon": "fish_audio_tts", - "name": "fish_audio_tts", - "property": { - "api_key": "${env:FISH_AUDIO_TTS_KEY}", - "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", - "optimize_streaming_latency": true, - "request_timeout_seconds": 30, - "base_url": "https://api.fish.audio" - } - }, - { - "type": "extension", - "extension_group": "transcriber", + "name": "message_collector", "addon": "message_collector", - "name": "message_collector" + "extension_group": "transcriber", + "property": {} }, { "type": "extension", - "extension_group": "tools", - "addon": "weatherapi_tool_python", - "name": "weatherapi_tool_python", + "name": "bingsearch_tool_python", + "addon": "bingsearch_tool_python", + "extension_group": "default", "property": { - "api_key": "${env:WEATHERAPI_API_KEY}" + "api_key": "${env:BING_API_KEY|}" } }, { "type": "extension", - "extension_group": "tools", - "addon": "bingsearch_tool_python", - "name": "bingsearch_tool_python", + "name": "weatherapi_tool_python", + "addon": "weatherapi_tool_python", + "extension_group": "default", "property": { - "api_key": "${env:BING_API_KEY}" + "api_key": "${env:WEATHERAPI_API_KEY|}" } } ], @@ -831,54 +387,42 @@ { "extension_group": "rtc", "extension": "agora_rtc", - "audio_frame": [ + "cmd": [ { - "name": "pcm_frame", + "name": "on_user_joined", "dest": [ { "extension_group": "llm", - "extension": "openai_v2v_python" + "extension": "v2v" } ] - } - ], - "data": [ + }, { - "name": "text_data", + "name": "on_user_left", "dest": [ { - "extension_group": "transcriber", - "extension": "message_collector" + "extension_group": "llm", + "extension": "v2v" } ] - } - ] - }, - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "cmd": [ + }, { - "name": "tool_register", + "name": "on_connection_failure", "dest": [ { "extension_group": "llm", - "extension": "openai_v2v_python" + "extension": "v2v" } ] } - ] - }, - { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "cmd": [ + ], + "audio_frame": [ { - "name": "tool_register", + "name": "pcm_frame", "dest": [ { "extension_group": "llm", - "extension": "openai_v2v_python" + "extension": "v2v" } ] } @@ -886,29 +430,14 @@ }, { "extension_group": "llm", - "extension": "openai_v2v_python", - "data": [ - { - "name": "text_data", - "dest": [ - { - "extension_group": "transcriber", - "extension": "message_collector" - }, - { - "extension_group": "tts", - "extension": "fish_audio_tts" - } - ] - } - ], + "extension": "v2v", "cmd": [ { "name": "flush", "dest": [ { - "extension_group": "tts", - "extension": "fish_audio_tts" + "extension_group": "rtc", + "extension": "agora_rtc" } ] }, @@ -916,41 +445,45 @@ "name": "tool_call", "dest": [ { - "extension_group": "tools", - "extension": "weatherapi_tool_python" + "extension_group": "default", + "extension": "bingsearch_tool_python" }, { - "extension_group": "tools", - "extension": "bingsearch_tool_python" + "extension_group": "default", + "extension": "weatherapi_tool_python" } ] - }, + } + ], + "data": [ { - "name": "on_user_joined", + "name": "text_data", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "transcriber", + "extension": "message_collector" } ] - }, + } + ], + "audio_frame": [ { - "name": "on_user_left", + "name": "pcm_frame", "dest": [ { - "extension_group": "llm", - "extension": "openai_v2v_python" + "extension_group": "rtc", + "extension": "agora_rtc" } ] } ] }, { - "extension_group": "tts", - "extension": "fish_audio_tts", - "audio_frame": [ + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ { - "name": "pcm_frame", + "name": "data", "dest": [ { "extension_group": "rtc", @@ -958,29 +491,33 @@ } ] } - ], + ] + }, + { + "extension_group": "default", + "extension": "bingsearch_tool_python", "cmd": [ { - "name": "flush", + "name": "tool_register", "dest": [ { - "extension_group": "rtc", - "extension": "agora_rtc" + "extension_group": "llm", + "extension": "v2v" } ] } ] }, { - "extension_group": "transcriber", - "extension": "message_collector", - "data": [ + "extension_group": "default", + "extension": "weatherapi_tool_python", + "cmd": [ { - "name": "data", + "name": "tool_register", "dest": [ { - "extension_group": "rtc", - "extension": "agora_rtc" + "extension_group": "llm", + "extension": "v2v" } ] } @@ -989,219 +526,160 @@ ] }, { - "name": "camera_va_openai_azure_rtm", - "auto_start": true, + "name": "va_openai_v2v_fish", + "auto_start": false, "nodes": [ { "type": "extension", - "extension_group": "default", + "extension_group": "rtc", "addon": "agora_rtc", "name": "agora_rtc", "property": { "app_id": "${env:AGORA_APP_ID}", - "token": "", + "token": "", "channel": "ten_agent_test", "stream_id": 1234, "remote_stream_id": 123, "subscribe_audio": true, - "subscribe_video": true, "publish_audio": true, "publish_data": true, + "subscribe_audio_sample_rate": 24000, "enable_agora_asr": true, "agora_asr_vendor_name": "microsoft", "agora_asr_language": "en-US", "agora_asr_vendor_key": "${env:AZURE_STT_KEY}", "agora_asr_vendor_region": "${env:AZURE_STT_REGION}", - "agora_asr_session_control_file_path": "session_control.conf", - "subscribe_video_pix_fmt": 4 + "agora_asr_session_control_file_path": "session_control.conf" } }, { "type": "extension", - "extension_group": "rtm", - "addon": "agora_rtm", - "name": "agora_rtm", + "extension_group": "llm", + "addon": "openai_v2v_python", + "name": "v2v", "property": { - "channel": "ten_agent_test", - "app_id": "${env:AGORA_APP_ID}", - "token": "", - "user_id": "0", - "rtm_enabled": true, - "rtm_is_stream": false, - "rtm_presence_enabled": false, - "rtm_metadata_enabled": false, - "rtm_lock_enabled": false + "api_key": "${env:OPENAI_REALTIME_API_KEY}", + "temperature": 0.9, + "model": "gpt-4o-realtime-preview", + "max_tokens": 2048, + "audio_out": false, + "input_transcript": false, + "language": "en-US", + "server_vad": true, + "dump": true, + "greeting": "TEN Agent connected. How can I help you today?", + "max_history": 10 } }, { "type": "extension", - "extension_group": "default", - "addon": "interrupt_detector", - "name": "interrupt_detector" - }, - { - "type": "extension", - "extension_group": "chatgpt", - "addon": "openai_chatgpt_python", - "name": "openai_chatgpt", + "extension_group": "tts", + "addon": "fish_audio_tts", + "name": "tts", "property": { - "base_url": "${env:OPENAI_API_BASE}", - "api_key": "${env:OPENAI_API_KEY}", - "frequency_penalty": 0.9, - "model": "${env:OPENAI_MODEL}", - "max_tokens": 512, - "prompt": "", - "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", - "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", - "max_memory_length": 10, - "enable_tools": true + "api_key": "${env:FISH_AUDIO_TTS_KEY}", + "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5", + "optimize_streaming_latency": true, + "request_timeout_seconds": 30, + "base_url": "https://api.fish.audio" } }, { "type": "extension", - "extension_group": "chatgpt", - "addon": "vision_tool_python", - "name": "vision_tool" + "extension_group": "transcriber", + "addon": "message_collector", + "name": "message_collector" }, { "type": "extension", - "extension_group": "tts", - "addon": "azure_tts", - "name": "azure_tts", + "extension_group": "tools", + "addon": "weatherapi_tool_python", + "name": "weatherapi_tool_python", "property": { - "azure_subscription_key": "${env:AZURE_TTS_KEY}", - "azure_subscription_region": "${env:AZURE_TTS_REGION}", - "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural" + "api_key": "${env:WEATHERAPI_API_KEY}" } }, { "type": "extension", - "extension_group": "message_collector_rtm", - "addon": "message_collector_rtm", - "name": "message_collector_rtm" + "extension_group": "tools", + "addon": "bingsearch_tool_python", + "name": "bingsearch_tool_python", + "property": { + "api_key": "${env:BING_API_KEY}" + } } ], "connections": [ { - "extension_group": "default", + "extension_group": "rtc", "extension": "agora_rtc", - "data": [ + "audio_frame": [ { - "name": "text_data", + "name": "pcm_frame", "dest": [ { - "extension_group": "default", - "extension": "interrupt_detector" - }, - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - }, - { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm" + "extension_group": "llm", + "extension": "v2v" } ] } ], - "cmd": [ - { - "name": "on_user_joined", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - }, - { - "name": "on_user_left", - "dest": [ - { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" - } - ] - }, + "data": [ { - "name": "on_user_audio_track_state_changed", + "name": "text_data", "dest": [ { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm" + "extension_group": "transcriber", + "extension": "message_collector" } ] } - ], - "video_frame": [ + ] + }, + { + "extension_group": "tools", + "extension": "weatherapi_tool_python", + "cmd": [ { - "name": "video_frame", + "name": "tool_register", "dest": [ { - "extension_group": "chatgpt", - "extension": "vision_tool" + "extension_group": "llm", + "extension": "v2v" } ] } ] }, { - "extension": "agora_rtm", - "extension_group": "rtm", - "data": [ - { - "name": "rtm_message_event", - "dest": [ - { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm" - } - ] - }, - { - "name": "rtm_storage_event", - "dest": [ - { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm" - } - ] - }, - { - "name": "rtm_presence_event", - "dest": [ - { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm" - } - ] - }, + "extension_group": "tools", + "extension": "bingsearch_tool_python", + "cmd": [ { - "name": "rtm_lock_event", + "name": "tool_register", "dest": [ { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm" + "extension_group": "llm", + "extension": "v2v" } ] } ] }, { - "extension_group": "chatgpt", - "extension": "openai_chatgpt", + "extension_group": "llm", + "extension": "v2v", "data": [ { "name": "text_data", "dest": [ { - "extension_group": "tts", - "extension": "azure_tts" + "extension_group": "transcriber", + "extension": "message_collector" }, { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm" + "extension_group": "tts", + "extension": "tts" } ] } @@ -1212,7 +690,7 @@ "dest": [ { "extension_group": "tts", - "extension": "azure_tts" + "extension": "tts" } ] }, @@ -1220,99 +698,71 @@ "name": "tool_call", "dest": [ { - "extension_group": "chatgpt", - "extension": "vision_tool" + "extension_group": "tools", + "extension": "weatherapi_tool_python" + }, + { + "extension_group": "tools", + "extension": "bingsearch_tool_python" } ] - } - ] - }, - { - "extension_group": "tts", - "extension": "azure_tts", - "audio_frame": [ + }, { - "name": "pcm_frame", + "name": "on_user_joined", "dest": [ { - "extension_group": "default", - "extension": "agora_rtc" + "extension_group": "llm", + "extension": "v2v" } ] - } - ], - "cmd": [ + }, { - "name": "flush", + "name": "on_user_left", "dest": [ { - "extension_group": "default", - "extension": "agora_rtc" + "extension_group": "llm", + "extension": "v2v" } ] } ] }, { - "extension_group": "message_collector_rtm", - "extension": "message_collector_rtm", - "data": [ + "extension_group": "tts", + "extension": "tts", + "audio_frame": [ { - "name": "text_data", + "name": "pcm_frame", "dest": [ { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" + "extension_group": "rtc", + "extension": "agora_rtc" } ] } ], - "cmd": [ - { - "name": "publish", - "dest": [ - { - "extension_group": "rtm", - "extension": "agora_rtm" - } - ] - }, - { - "name": "set_presence_state", - "dest": [ - { - "extension_group": "rtm", - "extension": "agora_rtm" - } - ] - } - ] - }, - { - "extension_group": "default", - "extension": "interrupt_detector", "cmd": [ { "name": "flush", "dest": [ { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" + "extension_group": "rtc", + "extension": "agora_rtc" } ] } ] }, { - "extension_group": "chatgpt", - "extension": "vision_tool", - "cmd": [ + "extension_group": "transcriber", + "extension": "message_collector", + "data": [ { - "name": "tool_register", + "name": "data", "dest": [ { - "extension_group": "chatgpt", - "extension": "openai_chatgpt" + "extension_group": "rtc", + "extension": "agora_rtc" } ] } @@ -1369,7 +819,7 @@ "type": "extension", "extension_group": "tts", "addon": "azure_tts", - "name": "azure_tts", + "name": "tts", "property": { "azure_subscription_key": "${env:AZURE_TTS_KEY}", "azure_subscription_region": "${env:AZURE_TTS_REGION}", @@ -1436,7 +886,7 @@ "dest": [ { "extension_group": "tts", - "extension": "azure_tts" + "extension": "tts" }, { "extension_group": "transcriber", @@ -1451,7 +901,7 @@ "dest": [ { "extension_group": "tts", - "extension": "azure_tts" + "extension": "tts" } ] } @@ -1459,7 +909,7 @@ }, { "extension_group": "tts", - "extension": "azure_tts", + "extension": "tts", "audio_frame": [ { "name": "pcm_frame", diff --git a/demo/src/app/api/agents/start/graph.ts b/demo/src/app/api/agents/start/graph.ts index cff387b1..c8543233 100644 --- a/demo/src/app/api/agents/start/graph.ts +++ b/demo/src/app/api/agents/start/graph.ts @@ -71,40 +71,47 @@ export const getGraphProperties = ( let localizationOptions = { "greeting": "Hey, I\'m TEN Agent, I can speak, see, and reason from a knowledge base, ask me anything!", "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", + "coze_greeting": "Hey, I'm Coze Bot, I can chat with you, ask me anything!", } if (language === "zh-CN") { localizationOptions = { "greeting": "嗨，我是 TEN Agent，我可以说话、看东西，还能从知识库中推理，问我任何问题吧！", "checking_vision_text_items": "[\"让我看看你的摄像头...\",\"让我看一下...\",\"我看一下，请稍候...\"]", + "coze_greeting": "嗨，我是扣子机器人，我可以和你聊天，问我任何问题吧！", } } else if (language === "ja-JP") { localizationOptions = { "greeting": "こんにちは、TEN Agentです。私は話したり、見たり、知識ベースから推論したりできます。何でも聞いてください！", "checking_vision_text_items": "[\"ちょっと見てみます...\",\"カメラをチェックします...\",\"少々お待ちください...\"]", + "coze_greeting": "こんにちは、私はCoze Botです。お話しできますので、何でも聞いてください！", } } else if (language === "ko-KR") { localizationOptions = { "greeting": "안녕하세요, 저는 TEN Agent입니다. 말하고, 보고, 지식 베이스에서 추론할 수 있어요. 무엇이든 물어보세요!", "checking_vision_text_items": "[\"조금만 기다려 주세요...\",\"카메라를 확인해 보겠습니다...\",\"잠시만 기다려 주세요...\"]", + "coze_greeting": "안녕하세요, 저는 Coze Bot입니다. 대화할 수 있어요. 무엇이든 물어보세요!", } } + let combined_greeting = greeting || localizationOptions["greeting"]; + if (graphName == "camera_va_openai_azure") { return { "agora_rtc": { "agora_asr_language": language, }, - "openai_chatgpt": { + "llm": { ...localizationOptions, "prompt": prompt, - "greeting": greeting, + "greeting": combined_greeting, }, - "azure_tts": { + "tts": { "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } } else if (graphName == "va_coze_azure") { + combined_greeting = greeting || localizationOptions["coze_greeting"]; return { "agora_rtc": { "agora_asr_language": language, @@ -112,9 +119,9 @@ export const getGraphProperties = ( "coze_python_async": { ...localizationOptions, "prompt": prompt, - "greeting": greeting, + "greeting": combined_greeting, }, - "azure_tts": { + "tts": { "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } @@ -123,36 +130,36 @@ export const getGraphProperties = ( "agora_rtc": { "agora_asr_language": language, }, - "openai_chatgpt": { + "llm": { "model": "gpt-4o", ...localizationOptions, "prompt": prompt, - "greeting": greeting, + "greeting": combined_greeting, }, - "azure_tts": { + "tts": { "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } } else if (graphName == "va_openai_v2v") { return { - "openai_v2v_python": { + "v2v": { "model": "gpt-4o-realtime-preview", "voice": voiceNameMap[language]["openai"][voiceType], "language": language, ...localizationOptions, "prompt": prompt, - "greeting": greeting, + "greeting": combined_greeting, } } } else if (graphName == "va_openai_v2v_fish") { return { - "openai_v2v_python": { + "v2v": { "model": "gpt-4o-realtime-preview", "voice": voiceNameMap[language]["openai"][voiceType], "language": language, ...localizationOptions, "prompt": prompt, - "greeting": greeting, + "greeting": combined_greeting, }, "agora_rtc": { "agora_asr_language": language, @@ -163,13 +170,13 @@ export const getGraphProperties = ( "agora_rtc": { "agora_asr_language": language, }, - "openai_chatgpt": { - "model": "gpt-4o-mini", + "llm": { + "model": "gpt-4o", ...localizationOptions, "prompt": prompt, - "greeting": greeting, + "greeting": combined_greeting, }, - "azure_tts": { + "tts": { "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType] } } diff --git a/demo/src/common/constant.ts b/demo/src/common/constant.ts index e3d5886b..6a1c6d9e 100644 --- a/demo/src/common/constant.ts +++ b/demo/src/common/constant.ts @@ -57,29 +57,21 @@ export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [ ] export const GRAPH_OPTIONS: GraphOptionItem[] = [ { - label: "Voice Agent - OpenAI LLM + Azure TTS", + label: "Voice Agent / STT + LLM + TTS", value: "va_openai_azure", }, - { - label: "Voice Agent with Vision - OpenAI LLM + Azure TTS", - value: "camera_va_openai_azure", - }, // { // label: "Voice Agent with Knowledge - RAG + Qwen LLM + Cosy TTS", // value: "va_qwen_rag" // }, { - label: "Voice Agent with OpenAI Realtime API (Beta)", + label: "Voice Agent OpenAI Realtime", value: "va_openai_v2v", }, { - label: "Voice Agent with OpenAI Realtime API (Beta) + FishAudio TTS", + label: "Voice Agent OpenAI Realtime + Custom STT/TTS", value: "va_openai_v2v_fish", }, - { - label: "Voice Agent with Vision - OpenAI LLM + Azure TTS + RTM", - value: "camera_va_openai_azure_rtm", - }, { label: "Voice Agent Coze Bot + Azure TTS", value: "va_coze_azure", diff --git a/playground/src/common/graph.ts b/playground/src/common/graph.ts index 3239aecf..a62f2186 100644 --- a/playground/src/common/graph.ts +++ b/playground/src/common/graph.ts @@ -357,8 +357,8 @@ class GraphEditor { } } /** - * Remove a connection from the graph across all protocols (cmd, data, audio_frame, video_frame) - */ + * Remove a connection from the graph across all protocols (cmd, data, audio_frame, video_frame) + */ static removeConnection( graph: Graph, source: string, @@ -371,54 +371,57 @@ class GraphEditor { (conn) => conn.extensionGroup === source.split(".")[0] && conn.extension === source.split(".")[1], - ) + ); if (connectionIndex === -1) { - throw new Error(`Source "${source}" not found in the graph.`) + console.warn(`Source "${source}" not found in the graph. Operation ignored.`); + return; // Exit the function if the connection does not exist } - const connection = graph.connections[connectionIndex] + const connection = graph.connections[connectionIndex]; // If protocolLabel is provided, handle protocol-specific removal if (protocolLabel) { - const protocolField = protocolLabel.toLowerCase() as keyof Connection + const protocolField = protocolLabel.toLowerCase() as keyof Connection; const protocolArray = connection[protocolField] as Array< Command | Data | AudioFrame | VideoFrame - > + >; if (!protocolArray) { - throw new Error( - `Protocol "${protocolLabel}" does not exist for source "${source}".`, - ) + console.warn( + `Protocol "${protocolLabel}" does not exist for source "${source}". Operation ignored.` + ); + return; // Exit the function if the protocol does not exist } const protocolObjectIndex = protocolArray.findIndex( (item) => item.name === protocolName, - ) + ); if (protocolObjectIndex === -1) { - throw new Error( - `Protocol object with name "${protocolName}" not found in protocol "${protocolLabel}".`, - ) + console.warn( + `Protocol object with name "${protocolName}" not found in protocol "${protocolLabel}". Operation ignored.` + ); + return; // Exit the function if the protocol object does not exist } if (destination) { // Remove a specific destination protocolArray[protocolObjectIndex].dest = protocolArray[ protocolObjectIndex - ].dest.filter((dest) => dest.extension !== destination.split(".")[1]) + ].dest.filter((dest) => dest.extension !== destination.split(".")[1]); // Remove the protocol object if it has no destinations if (protocolArray[protocolObjectIndex].dest.length === 0) { - protocolArray.splice(protocolObjectIndex, 1) + protocolArray.splice(protocolObjectIndex, 1); } } else { // Remove the entire protocol object - protocolArray.splice(protocolObjectIndex, 1) + protocolArray.splice(protocolObjectIndex, 1); } } else { // If no protocolLabel is provided, remove the entire connection - graph.connections.splice(connectionIndex, 1) + graph.connections.splice(connectionIndex, 1); } // Clean up empty connections diff --git a/playground/src/components/Chat/ChatCfgModuleSelect.tsx b/playground/src/components/Chat/ChatCfgModuleSelect.tsx index fa9842a8..8321f886 100644 --- a/playground/src/components/Chat/ChatCfgModuleSelect.tsx +++ b/playground/src/components/Chat/ChatCfgModuleSelect.tsx @@ -118,99 +118,99 @@ export function RemoteModuleCfgSheet() { initialData={initialData} metadata={metadata} onUpdate={async (data, tools) => { - // Clone the selectedGraph to avoid mutating the original graph - const selectedGraphCopy: Graph = JSON.parse(JSON.stringify(selectedGraph)); - const nodes = selectedGraphCopy.nodes; - let needUpdate = false; - let enableRTCVideoSubscribe = false; - - - // Retrieve the agora_rtc node - const agoraRtcNode = GraphEditor.findNode(selectedGraphCopy, "agora_rtc"); - if (!agoraRtcNode) { - toast.error("agora_rtc node not found in the graph"); - return; - } - - // Update graph nodes with selected modules - Object.entries(data).forEach(([key, value]) => { - const node = nodes.find((n) => n.name === key); - if (node && value && node.addon !== value) { - node.addon = value; - node.property = addonModules.find((module) => module.name === value)?.defaultProperty; - - if(node.addon === "gemini_v2v_python") { - GraphEditor.addOrUpdateConnection( - selectedGraphCopy, - `${agoraRtcNode.extensionGroup}.${agoraRtcNode.name}`, - `${node.extensionGroup}.${node.name}`, - ProtocolLabel.VIDEO_FRAME, - "video_frame" - ); - enableRTCVideoSubscribe = true; - } else { - GraphEditor.removeConnection( - selectedGraphCopy, - `${agoraRtcNode.extensionGroup}.${agoraRtcNode.name}`, - `${node.extensionGroup}.${node.name}`, - ProtocolLabel.VIDEO_FRAME, - "video_frame" - ); - } - - needUpdate = true; + try { + // Clone the selectedGraph to avoid mutating the original graph + const selectedGraphCopy: Graph = JSON.parse(JSON.stringify(selectedGraph)); + const nodes = selectedGraphCopy.nodes; + let needUpdate = false; + let enableRTCVideoSubscribe = false; + + + // Retrieve the agora_rtc node + const agoraRtcNode = GraphEditor.findNode(selectedGraphCopy, "agora_rtc"); + if (!agoraRtcNode) { + toast.error("agora_rtc node not found in the graph"); + return; } - }); - - // Identify removed tools and process them - const currentToolsInGraph = nodes - .filter((node) => installedAndRegisteredToolModules.map((module) => module.name).includes(node.addon)) - .map((node) => node.addon); - - const removedTools = currentToolsInGraph.filter((tool) => !tools.includes(tool)); - removedTools.forEach((tool) => { - GraphEditor.removeNodeAndConnections(selectedGraphCopy, tool); - needUpdate = true; - }); - - // Process tool modules - if (tools.length > 0) { - if(!enableRTCVideoSubscribe) { - enableRTCVideoSubscribe = tools.some((tool) => tool.includes("vision")) - } - tools.forEach((tool) => { - if (!currentToolsInGraph.includes(tool)) { - const toolModule = addonModules.find((module) => module.name === tool); - if (!toolModule) { - toast.error(`Module ${tool} not found`); - return; + // Update graph nodes with selected modules + Object.entries(data).forEach(([key, value]) => { + const node = nodes.find((n) => n.name === key); + if (node && value && node.addon !== value) { + node.addon = value; + node.property = addonModules.find((module) => module.name === value)?.defaultProperty; + + if (node.addon === "gemini_v2v_python") { + GraphEditor.addOrUpdateConnection( + selectedGraphCopy, + `${agoraRtcNode.extensionGroup}.${agoraRtcNode.name}`, + `${node.extensionGroup}.${node.name}`, + ProtocolLabel.VIDEO_FRAME, + "video_frame" + ); + enableRTCVideoSubscribe = true; + } else { + GraphEditor.removeConnection( + selectedGraphCopy, + `${agoraRtcNode.extensionGroup}.${agoraRtcNode.name}`, + `${node.extensionGroup}.${node.name}`, + ProtocolLabel.VIDEO_FRAME, + "video_frame" + ); } - const toolNode = GraphEditor.addNode(selectedGraphCopy, tool, tool, "default", toolModule.defaultProperty) - - // Create or update connections - const llmNode = GraphEditor.findNodeByPredicate(selectedGraphCopy, (node) => isLLM(node.name)); - if (llmNode) { - GraphEditor.linkTool(selectedGraphCopy, llmNode, toolNode); - } + needUpdate = true; } }); - needUpdate = true; - } - GraphEditor.enableRTCVideoSubscribe(selectedGraphCopy, enableRTCVideoSubscribe); + // Identify removed tools and process them + const currentToolsInGraph = nodes + .filter((node) => installedAndRegisteredToolModules.map((module) => module.name).includes(node.addon)) + .map((node) => node.addon); + + const removedTools = currentToolsInGraph.filter((tool) => !tools.includes(tool)); + removedTools.forEach((tool) => { + GraphEditor.removeNodeAndConnections(selectedGraphCopy, tool); + needUpdate = true; + }); + + // Process tool modules + if (tools.length > 0) { + if (!enableRTCVideoSubscribe) { + enableRTCVideoSubscribe = tools.some((tool) => tool.includes("vision")) + } + tools.forEach((tool) => { + if (!currentToolsInGraph.includes(tool)) { + const toolModule = addonModules.find((module) => module.name === tool); + + if (!toolModule) { + toast.error(`Module ${tool} not found`); + return; + } + + const toolNode = GraphEditor.addNode(selectedGraphCopy, tool, tool, "default", toolModule.defaultProperty) + + // Create or update connections + const llmNode = GraphEditor.findNodeByPredicate(selectedGraphCopy, (node) => isLLM(node.name)); + if (llmNode) { + GraphEditor.linkTool(selectedGraphCopy, llmNode, toolNode); + } + } + }); + needUpdate = true; + } + + GraphEditor.enableRTCVideoSubscribe(selectedGraphCopy, enableRTCVideoSubscribe); - // Perform the update if changes are detected - if (needUpdate) { - try { + // Perform the update if changes are detected + if (needUpdate) { await updateGraph(selectedGraphCopy.id, selectedGraphCopy); toast.success("Modules updated", { description: `Graph: ${selectedGraphCopy.id}`, }); - } catch (e:any) { - toast.error(`Failed to update modules: ${e}`); } + } catch (e: any) { + toast.error(`Failed to update modules: ${e}`); } }}