From 5da2247aa8b98d1d2e5f3abff687c582bd97f56a Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Fri, 26 Apr 2024 13:18:10 -0300 Subject: [PATCH 01/12] Added 'role' as a summary_args and to the reflection_with_llm flow to be able to pass the role for the summarizing prompt --- autogen/agentchat/conversable_agent.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index 262fc513d23..07cb1c7ab3f 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -921,6 +921,7 @@ def my_summary_method( summary_args (dict): a dictionary of arguments to be passed to the summary_method. One example key is "summary_prompt", and value is a string of text used to prompt a LLM-based agent (the sender or receiver agent) to reflect on the conversation and extract a summary when summary_method is "reflection_with_llm". + Another argument is 'role', the default for which is 'system'. The default summary_prompt is DEFAULT_SUMMARY_PROMPT, i.e., "Summarize takeaway from the conversation. Do not add any introductory phrases. If the intended request is NOT properly addressed, please point it out." message (str, dict or Callable): the initial message to be sent to the recipient. Needs to be provided. Otherwise, input() will be called to get the initial message. - If a string or a dict is provided, it will be used as the initial message. `generate_init_message` is called to generate the initial message for the agent based on this string and the context. @@ -1153,8 +1154,11 @@ def _reflection_with_llm_as_summary(sender, recipient, summary_args): raise ValueError("The summary_prompt must be a string.") msg_list = recipient.chat_messages_for_summary(sender) agent = sender if recipient is None else recipient + role = summary_args.get("role", None) + if role and not isinstance(role, str): + raise ValueError("The role (summary_arg) must be a string.") try: - summary = sender._reflection_with_llm(prompt, msg_list, llm_agent=agent, cache=summary_args.get("cache")) + summary = sender._reflection_with_llm(prompt, msg_list, llm_agent=agent, cache=summary_args.get("cache"), role=role) except BadRequestError as e: warnings.warn( f"Cannot extract summary using reflection_with_llm: {e}. Using an empty str as summary.", UserWarning @@ -1163,7 +1167,7 @@ def _reflection_with_llm_as_summary(sender, recipient, summary_args): return summary def _reflection_with_llm( - self, prompt, messages, llm_agent: Optional[Agent] = None, cache: Optional[AbstractCache] = None + self, prompt, messages, llm_agent: Optional[Agent] = None, cache: Optional[AbstractCache] = None, role: Union[str, None] = None ) -> str: """Get a chat summary using reflection with an llm client based on the conversation history. @@ -1172,14 +1176,19 @@ def _reflection_with_llm( messages (list): The messages generated as part of a chat conversation. llm_agent: the agent with an llm client. cache (AbstractCache or None): the cache client to be used for this conversation. + role (str): the role of the message, usually "system" or "user". Default is "system". """ + if not role: + role = "system" + system_msg = [ { - "role": "system", + "role": role, "content": prompt, } ] + messages = messages + system_msg if llm_agent and llm_agent.client is not None: llm_client = llm_agent.client From fa927f18251bc0d1945b986f9ab8650c131f13cb Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Fri, 26 Apr 2024 13:51:58 -0300 Subject: [PATCH 02/12] Added 'role' as a summary_args and to the reflection_with_llm flow to be able to pass the role for the summarizing prompt, minor docstring adjustments --- autogen/agentchat/conversable_agent.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index 07cb1c7ab3f..dc7129f0190 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -921,7 +921,6 @@ def my_summary_method( summary_args (dict): a dictionary of arguments to be passed to the summary_method. One example key is "summary_prompt", and value is a string of text used to prompt a LLM-based agent (the sender or receiver agent) to reflect on the conversation and extract a summary when summary_method is "reflection_with_llm". - Another argument is 'role', the default for which is 'system'. The default summary_prompt is DEFAULT_SUMMARY_PROMPT, i.e., "Summarize takeaway from the conversation. Do not add any introductory phrases. If the intended request is NOT properly addressed, please point it out." message (str, dict or Callable): the initial message to be sent to the recipient. Needs to be provided. Otherwise, input() will be called to get the initial message. - If a string or a dict is provided, it will be used as the initial message. `generate_init_message` is called to generate the initial message for the agent based on this string and the context. @@ -930,8 +929,7 @@ def my_summary_method( 1. "content": content of the message, can be None. 2. "function_call": a dictionary containing the function name and arguments. (deprecated in favor of "tool_calls") 3. "tool_calls": a list of dictionaries containing the function name and arguments. - 4. "role": role of the message, can be "assistant", "user", "function". - This field is only needed to distinguish between "function" or "assistant"/"user". + 4. "role": role of the message, default is "system". Can be "assistant", "user", "function" or "system" 5. "name": In most cases, this field is not needed. When the role is "function", this field is needed to indicate the function name. 6. "context" (dict): the context of the message, which will be passed to [OpenAIWrapper.create](../oai/client#create). From 04b059783a3a2cbd30c7a877d213f8e5bbe6eeae Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Sat, 27 Apr 2024 13:51:30 -0300 Subject: [PATCH 03/12] Added test for summary prompt role assignment --- test/agentchat/test_groupchat.py | 48 ++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index 8a4758d2d37..f0720ce3bf6 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -3,14 +3,14 @@ import builtins import json from typing import Any, Dict, List, Optional -from unittest import mock +from unittest import mock, TestCase import pytest import autogen from autogen import Agent, GroupChat from autogen.exception_utils import AgentNameConflict, UndefinedNextAgent - +from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST def test_func_call_groupchat(): agent1 = autogen.ConversableAgent( @@ -1425,6 +1425,47 @@ def test_speaker_selection_agent_name_match(): ) assert result == {} +def test_role_for_reflection_summary(): + config_list = autogen.config_list_from_json( + OAI_CONFIG_LIST, + file_location=KEY_LOC, + ) + llm_config={ + "config_list": config_list, + "model": "gpt-3.5-turbo-0613", + }, + agent1 = autogen.ConversableAgent( + "alice", + max_consecutive_auto_reply=10, + human_input_mode="NEVER", + llm_config=False, + default_auto_reply="This is alice speaking.", + ) + agent2 = autogen.ConversableAgent( + "bob", + max_consecutive_auto_reply=10, + human_input_mode="NEVER", + llm_config=False, + default_auto_reply="This is bob speaking.", + ) + groupchat = autogen.GroupChat(agents=[agent1, agent2], messages=[], max_round=3, speaker_selection_method="round_robin") + group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config) + + role_name = "user" + with mock.patch.object(autogen.ConversableAgent, '_generate_oai_reply_from_client') as mock_generate_oai_reply_from_client: + mock_generate_oai_reply_from_client.return_value = "Mocked summary" + + res = agent1.initiate_chat( + group_chat_manager, + max_turns=2, + message="hello", + summary_method="reflection_with_llm", + summary_args={"role": role_name} + ) + + mock_generate_oai_reply_from_client.assert_called_once() + args, kwargs = mock_generate_oai_reply_from_client.call_args + assert kwargs['messages'][-1]["role"] == role_name if __name__ == "__main__": # test_func_call_groupchat() @@ -1443,5 +1484,6 @@ def test_speaker_selection_agent_name_match(): # test_custom_speaker_selection_overrides_transition_graph() # test_role_for_select_speaker_messages() # test_select_speaker_message_and_prompt_templates() - test_speaker_selection_agent_name_match() + # test_speaker_selection_agent_name_match() + test_role_for_reflection_summary() # pass From 953464a41708204ba524172dbc61be98a1c4656b Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Tue, 30 Apr 2024 15:19:23 -0300 Subject: [PATCH 04/12] Fixed docstrings and mocked llm-config in the test --- autogen/agentchat/conversable_agent.py | 6 ++++-- test/agentchat/test_groupchat.py | 9 +-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index dc7129f0190..d06524063ad 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -922,6 +922,7 @@ def my_summary_method( One example key is "summary_prompt", and value is a string of text used to prompt a LLM-based agent (the sender or receiver agent) to reflect on the conversation and extract a summary when summary_method is "reflection_with_llm". The default summary_prompt is DEFAULT_SUMMARY_PROMPT, i.e., "Summarize takeaway from the conversation. Do not add any introductory phrases. If the intended request is NOT properly addressed, please point it out." + Another available key is "summary_role", which is the role of the message sent to the agent in charge of summarizing. Default is "system". message (str, dict or Callable): the initial message to be sent to the recipient. Needs to be provided. Otherwise, input() will be called to get the initial message. - If a string or a dict is provided, it will be used as the initial message. `generate_init_message` is called to generate the initial message for the agent based on this string and the context. If dict, it may contain the following reserved fields (either content or tool_calls need to be provided). @@ -929,7 +930,8 @@ def my_summary_method( 1. "content": content of the message, can be None. 2. "function_call": a dictionary containing the function name and arguments. (deprecated in favor of "tool_calls") 3. "tool_calls": a list of dictionaries containing the function name and arguments. - 4. "role": role of the message, default is "system". Can be "assistant", "user", "function" or "system" + 4. "role": role of the message, can be "assistant", "user", "function". + This field is only needed to distinguish between "function" or "assistant"/"user". 5. "name": In most cases, this field is not needed. When the role is "function", this field is needed to indicate the function name. 6. "context" (dict): the context of the message, which will be passed to [OpenAIWrapper.create](../oai/client#create). @@ -1152,7 +1154,7 @@ def _reflection_with_llm_as_summary(sender, recipient, summary_args): raise ValueError("The summary_prompt must be a string.") msg_list = recipient.chat_messages_for_summary(sender) agent = sender if recipient is None else recipient - role = summary_args.get("role", None) + role = summary_args.get("summary_role", None) if role and not isinstance(role, str): raise ValueError("The role (summary_arg) must be a string.") try: diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index f0720ce3bf6..3887b8e387f 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -1426,14 +1426,7 @@ def test_speaker_selection_agent_name_match(): assert result == {} def test_role_for_reflection_summary(): - config_list = autogen.config_list_from_json( - OAI_CONFIG_LIST, - file_location=KEY_LOC, - ) - llm_config={ - "config_list": config_list, - "model": "gpt-3.5-turbo-0613", - }, + llm_config={"config_list": [{"model": "mock", "api_key": "mock"}]} agent1 = autogen.ConversableAgent( "alice", max_consecutive_auto_reply=10, From d309e15f95135c44d4341ebc07a4a2e323eed754 Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Thu, 2 May 2024 11:59:27 -0300 Subject: [PATCH 05/12] Update autogen/agentchat/conversable_agent.py Co-authored-by: Chi Wang --- autogen/agentchat/conversable_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index d06524063ad..c2e5c039dfd 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -1156,7 +1156,7 @@ def _reflection_with_llm_as_summary(sender, recipient, summary_args): agent = sender if recipient is None else recipient role = summary_args.get("summary_role", None) if role and not isinstance(role, str): - raise ValueError("The role (summary_arg) must be a string.") + raise ValueError("The summary_role in summary_arg must be a string.") try: summary = sender._reflection_with_llm(prompt, msg_list, llm_agent=agent, cache=summary_args.get("cache"), role=role) except BadRequestError as e: From 2dd6b14aaf1b4457b0754f3711fe93927a6d7e04 Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Fri, 3 May 2024 14:40:14 -0300 Subject: [PATCH 06/12] ran pre-commit --- test/agentchat/test_groupchat.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index 3d58beddef9..1a8cec9767b 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -3,14 +3,15 @@ import builtins import json from typing import Any, Dict, List, Optional -from unittest import mock, TestCase +from unittest import TestCase, mock import pytest +from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST import autogen from autogen import Agent, GroupChat from autogen.exception_utils import AgentNameConflict, UndefinedNextAgent -from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST + def test_func_call_groupchat(): agent1 = autogen.ConversableAgent( @@ -1443,8 +1444,9 @@ def test_speaker_selection_agent_name_match(): ) assert result == {} + def test_role_for_reflection_summary(): - llm_config={"config_list": [{"model": "mock", "api_key": "mock"}]} + llm_config = {"config_list": [{"model": "mock", "api_key": "mock"}]} agent1 = autogen.ConversableAgent( "alice", max_consecutive_auto_reply=10, @@ -1459,24 +1461,29 @@ def test_role_for_reflection_summary(): llm_config=False, default_auto_reply="This is bob speaking.", ) - groupchat = autogen.GroupChat(agents=[agent1, agent2], messages=[], max_round=3, speaker_selection_method="round_robin") + groupchat = autogen.GroupChat( + agents=[agent1, agent2], messages=[], max_round=3, speaker_selection_method="round_robin" + ) group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config) role_name = "user" - with mock.patch.object(autogen.ConversableAgent, '_generate_oai_reply_from_client') as mock_generate_oai_reply_from_client: + with mock.patch.object( + autogen.ConversableAgent, "_generate_oai_reply_from_client" + ) as mock_generate_oai_reply_from_client: mock_generate_oai_reply_from_client.return_value = "Mocked summary" - res = agent1.initiate_chat( + agent1.initiate_chat( group_chat_manager, max_turns=2, message="hello", summary_method="reflection_with_llm", - summary_args={"role": role_name} + summary_args={"role": role_name}, ) mock_generate_oai_reply_from_client.assert_called_once() args, kwargs = mock_generate_oai_reply_from_client.call_args - assert kwargs['messages'][-1]["role"] == role_name + assert kwargs["messages"][-1]["role"] == role_name + def test_speaker_selection_auto_process_result(): """ @@ -1822,5 +1829,4 @@ def test_select_speaker_auto_messages(): test_speaker_selection_auto_process_result() test_speaker_selection_validate_speaker_name() test_select_speaker_auto_messages() -main - # pass +# pass From bf32bf06aea7f8704957af07c266bf9bdca11989 Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Fri, 3 May 2024 14:45:43 -0300 Subject: [PATCH 07/12] ran pre-commit2 --- autogen/agentchat/conversable_agent.py | 12 +- ...33\357\200\233\357\200\233\357\200\233asd" | 385 ++++++++++++++++++ 2 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 "dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index 9689b3de016..35af969673b 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -1163,7 +1163,9 @@ def _reflection_with_llm_as_summary(sender, recipient, summary_args): if role and not isinstance(role, str): raise ValueError("The summary_role in summary_arg must be a string.") try: - summary = sender._reflection_with_llm(prompt, msg_list, llm_agent=agent, cache=summary_args.get("cache"), role=role) + summary = sender._reflection_with_llm( + prompt, msg_list, llm_agent=agent, cache=summary_args.get("cache"), role=role + ) except BadRequestError as e: warnings.warn( f"Cannot extract summary using reflection_with_llm: {e}. Using an empty str as summary.", UserWarning @@ -1172,7 +1174,12 @@ def _reflection_with_llm_as_summary(sender, recipient, summary_args): return summary def _reflection_with_llm( - self, prompt, messages, llm_agent: Optional[Agent] = None, cache: Optional[AbstractCache] = None, role: Union[str, None] = None + self, + prompt, + messages, + llm_agent: Optional[Agent] = None, + cache: Optional[AbstractCache] = None, + role: Union[str, None] = None, ) -> str: """Get a chat summary using reflection with an llm client based on the conversation history. @@ -1193,7 +1200,6 @@ def _reflection_with_llm( } ] - messages = messages + system_msg if llm_agent and llm_agent.client is not None: llm_client = llm_agent.client diff --git "a/dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" "b/dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" new file mode 100644 index 00000000000..3867aa00b78 --- /dev/null +++ "b/dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" @@ -0,0 +1,385 @@ +commit 9b3555eec4cec5b75af6b49558aee7c8ccad0ee1 (HEAD -> add-role-to-reflection-with-llm, origin/add-role-to-reflection-with-llm) +Merge: d309e15f 3e693578 +Author: MarianoMolina +Date: Thu May 2 12:03:26 2024 -0300 + + Merge branch 'main' into add-role-to-reflection-with-llm + +diff --cc test/agentchat/test_groupchat.py +index 3887b8e3,a4689bd5..3d58bedd +--- a/test/agentchat/test_groupchat.py ++++ b/test/agentchat/test_groupchat.py +@@@ -1425,41 -1443,329 +1443,363 @@@ def test_speaker_selection_agent_name_m + ) + assert result == {} +  + +def test_role_for_reflection_summary(): + + llm_config={"config_list": [{"model": "mock", "api_key": "mock"}]} + + agent1 = autogen.ConversableAgent( + + "alice", + + max_consecutive_auto_reply=10, + + human_input_mode="NEVER", + + llm_config=False, + + default_auto_reply="This is alice speaking.", + + ) + + agent2 = autogen.ConversableAgent( + + "bob", + + max_consecutive_auto_reply=10, + + human_input_mode="NEVER", + + llm_config=False, + + default_auto_reply="This is bob speaking.", + + ) + + groupchat = autogen.GroupChat(agents=[agent1, agent2], messages=[], max_round=3, speaker_selection_method="round_robin") + + group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config) + + + + role_name = "user" + + with mock.patch.object(autogen.ConversableAgent, '_generate_oai_reply_from_client') as mock_generate_oai_reply_from_client: + + mock_generate_oai_reply_from_client.return_value = "Mocked summary" + + + + res = agent1.initiate_chat( + + group_chat_manager, + + max_turns=2, + + message="hello", + + summary_method="reflection_with_llm", + + summary_args={"role": role_name} + + ) + + + + mock_generate_oai_reply_from_client.assert_called_once() + + args, kwargs = mock_generate_oai_reply_from_client.call_args + + assert kwargs['messages'][-1]["role"] == role_name +  ++ def test_speaker_selection_auto_process_result(): ++ """ ++ Tests the return result of the 2-agent chat used for speaker selection for the auto method. ++ The last message of the messages passed in will contain a pass or fail. ++ If passed, the message will contain the name of the correct agent and that agent will be returned. ++ If failed, the message will contain the reason for failure for the last attempt and the next ++ agent in the sequence will be returned. ++ """ ++ cmo = autogen.ConversableAgent( ++ name="Chief_Marketing_Officer", ++ human_input_mode="NEVER", ++ llm_config=False, ++ default_auto_reply="This is alice speaking.", ++ ) ++ pm = autogen.ConversableAgent( ++ name="Product_Manager", ++ human_input_mode="NEVER", ++ llm_config=False, ++ default_auto_reply="This is bob speaking.", ++ function_map={"test_func": lambda x: x}, ++ ) ++  ++ agent_list = [cmo, pm] ++ groupchat = autogen.GroupChat(agents=agent_list, messages=[], max_round=3) ++  ++ chat_result = autogen.ChatResult( ++ chat_id=None, ++ chat_history=[ ++ { ++ "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.", ++ "name": "Chairperson", ++ "role": "assistant", ++ }, ++ {"content": "You are an expert at finding the next speaker.", "role": "assistant"}, ++ {"content": "Product_Manager", "role": "user"}, ++ {"content": "UPDATED_BELOW", "role": "user"}, ++ ], ++ ) ++  ++ ### Agent selected successfully ++ chat_result.chat_history[3]["content"] = "[AGENT SELECTED]Product_Manager" ++  ++ # Product_Manager should be returned ++ assert groupchat._process_speaker_selection_result(chat_result, cmo, agent_list) == pm ++  ++ ### Agent not selected successfully ++ chat_result.chat_history[3][ ++ "content" ++ ] = "[AGENT SELECTION FAILED]Select speaker attempt #3 of 3 failed as it did not include any agent names." ++  ++ # The next speaker in the list will be selected, which will be the Product_Manager (as the last speaker is the Chief_Marketing_Officer) ++ assert groupchat._process_speaker_selection_result(chat_result, cmo, agent_list) == pm ++  ++ ### Invalid result messages, will return the next agent ++ chat_result.chat_history[3]["content"] = "This text should not be here." ++  ++ # The next speaker in the list will be selected, which will be the Chief_Marketing_Officer (as the last speaker is the Product_Maanger) ++ assert groupchat._process_speaker_selection_result(chat_result, pm, agent_list) == cmo ++  ++  ++ def test_speaker_selection_validate_speaker_name(): ++ """ ++ Tests the speaker name validation function used to evaluate the return result of the LLM ++ during speaker selection in 'auto' mode. ++  ++ Function: _validate_speaker_name ++  ++ If a single agent name is returned by the LLM, it will add a relevant message to the chat messages and return True, None ++ If multiple agent names are returned and there are attempts left, it will return a message to be used to prompt the LLM to try again ++ If multiple agent names are return and there are no attempts left, it will add a relevant message to the chat messages and return True, None ++ If no agent names are returned and there are attempts left, it will return a message to be used to prompt the LLM to try again ++ If no agent names are returned and there are no attempts left, it will add a relevant message to the chat messages and return True, None ++  ++ When returning a message, it will include the 'override_role' key and value to support the GroupChat role_for_select_speaker_messages attribute ++ """ ++  ++ # Group Chat setup ++ cmo = autogen.ConversableAgent( ++ name="Chief_Marketing_Officer", ++ human_input_mode="NEVER", ++ llm_config=False, ++ default_auto_reply="This is alice speaking.", ++ ) ++ pm = autogen.ConversableAgent( ++ name="Product_Manager", ++ human_input_mode="NEVER", ++ llm_config=False, ++ default_auto_reply="This is bob speaking.", ++ function_map={"test_func": lambda x: x}, ++ ) ++  ++ agent_list = [cmo, pm] ++ agent_list_string = f"{[agent.name for agent in agent_list]}" ++ groupchat = autogen.GroupChat(agents=agent_list, messages=[], max_round=3) ++  ++ # Speaker Selection 2-agent chat setup ++  ++ # Agent for selecting a single agent name from the response ++ speaker_selection_agent = autogen.ConversableAgent( ++ "speaker_selection_agent", ++ ) ++  ++ # Agent for checking the response from the speaker_select_agent ++ checking_agent = autogen.ConversableAgent("checking_agent") ++  ++ # Select speaker messages ++ select_speaker_messages = [ ++ { ++ "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.", ++ "name": "Chairperson", ++ "role": "assistant", ++ }, ++ {"content": "You are an expert at finding the next speaker.", "role": "assistant"}, ++ {"content": "UPDATED_BELOW", "role": "user"}, ++ ] ++  ++ ### Single agent name returned ++ attempts_left = 2 ++ attempt = 1 ++ select_speaker_messages[-1]["content"] = "Product_Manager is the next to speak" ++  ++ result = groupchat._validate_speaker_name( ++ recipient=checking_agent, ++ messages=select_speaker_messages, ++ sender=speaker_selection_agent, ++ config=None, ++ attempts_left=attempts_left, ++ attempt=attempt, ++ agents=agent_list, ++ ) ++  ++ assert result == (True, None) ++ assert select_speaker_messages[-1]["content"] == "[AGENT SELECTED]Product_Manager" ++  ++ select_speaker_messages.pop(-1) # Remove the last message before the next test ++  ++ ### Multiple agent names returned with attempts left ++ attempts_left = 2 ++ attempt = 1 ++ select_speaker_messages[-1]["content"] = "Product_Manager must speak after the Chief_Marketing_Officer" ++  ++ result = groupchat._validate_speaker_name( ++ recipient=checking_agent, ++ messages=select_speaker_messages, ++ sender=speaker_selection_agent, ++ config=None, ++ attempts_left=attempts_left, ++ attempt=attempt, ++ agents=agent_list, ++ ) ++  ++ assert result == ( ++ True, ++ { ++ "content": groupchat.select_speaker_auto_multiple_template.format(agentlist=agent_list_string), ++ "override_role": groupchat.role_for_select_speaker_messages, ++ }, ++ ) ++  ++ ### Multiple agent names returned with no attempts left ++ attempts_left = 0 ++ attempt = 1 ++ select_speaker_messages[-1]["content"] = "Product_Manager must speak after the Chief_Marketing_Officer" ++  ++ result = groupchat._validate_speaker_name( ++ recipient=checking_agent, ++ messages=select_speaker_messages, ++ sender=speaker_selection_agent, ++ config=None, ++ attempts_left=attempts_left, ++ attempt=attempt, ++ agents=agent_list, ++ ) ++  ++ assert result == (True, None) ++ assert ( ++ select_speaker_messages[-1]["content"] ++ == f"[AGENT SELECTION FAILED]Select speaker attempt #{attempt} of {attempt + attempts_left} failed as it returned multiple names." ++ ) ++  ++ select_speaker_messages.pop(-1) # Remove the last message before the next test ++  ++ ### No agent names returned with attempts left ++ attempts_left = 3 ++ attempt = 2 ++ select_speaker_messages[-1]["content"] = "The PM must speak after the CMO" ++  ++ result = groupchat._validate_speaker_name( ++ recipient=checking_agent, ++ messages=select_speaker_messages, ++ sender=speaker_selection_agent, ++ config=None, ++ attempts_left=attempts_left, ++ attempt=attempt, ++ agents=agent_list, ++ ) ++  ++ assert result == ( ++ True, ++ { ++ "content": groupchat.select_speaker_auto_none_template.format(agentlist=agent_list_string), ++ "override_role": groupchat.role_for_select_speaker_messages, ++ }, ++ ) ++  ++ ### Multiple agents returned with no attempts left ++ attempts_left = 0 ++ attempt = 3 ++ select_speaker_messages[-1]["content"] = "The PM must speak after the CMO" ++  ++ result = groupchat._validate_speaker_name( ++ recipient=checking_agent, ++ messages=select_speaker_messages, ++ sender=speaker_selection_agent, ++ config=None, ++ attempts_left=attempts_left, ++ attempt=attempt, ++ agents=agent_list, ++ ) ++  ++ assert result == (True, None) ++ assert ( ++ select_speaker_messages[-1]["content"] ++ == f"[AGENT SELECTION FAILED]Select speaker attempt #{attempt} of {attempt + attempts_left} failed as it did not include any agent names." ++ ) ++  ++  ++ def test_select_speaker_auto_messages(): ++ """ ++ In this test, two agents are part of a group chat which has customized select speaker "auto" multiple and no-name prompt messages. Both valid and empty string values will be used. ++ The expected behaviour is that the customized speaker selection "auto" messages will override the default values or throw exceptions if empty. ++ """ ++  ++ agent1 = autogen.ConversableAgent( ++ "Alice", ++ description="A wonderful employee named Alice.", ++ human_input_mode="NEVER", ++ llm_config=False, ++ ) ++ agent2 = autogen.ConversableAgent( ++ "Bob", ++ description="An amazing employee named Bob.", ++ human_input_mode="NEVER", ++ llm_config=False, ++ ) ++  ++ # Customised message for select speaker auto method where multiple agent names are returned ++ custom_multiple_names_msg = "You mentioned multiple names but we need just one. Select the best one. A reminder that the options are {agentlist}." ++  ++ # Customised message for select speaker auto method where no agent names are returned ++ custom_no_names_msg = "You forgot to select a single names and we need one, and only one. Select the best one. A reminder that the options are {agentlist}." ++  ++ # Test empty is_termination_msg function ++ groupchat = autogen.GroupChat( ++ agents=[agent1, agent2], ++ messages=[], ++ speaker_selection_method="auto", ++ max_round=10, ++ select_speaker_auto_multiple_template=custom_multiple_names_msg, ++ select_speaker_auto_none_template=custom_no_names_msg, ++ ) ++  ++ # Test using the _validate_speaker_name function, checking for the correct string and agentlist to be included ++ agents = [agent1, agent2] ++  ++ messages = [{"content": "Alice and Bob should both speak.", "name": "speaker_selector", "role": "user"}] ++ assert groupchat._validate_speaker_name(None, messages, None, None, 1, 1, agents) == ( ++ True, ++ { ++ "content": custom_multiple_names_msg.replace("{agentlist}", "['Alice', 'Bob']"), ++ "override_role": groupchat.role_for_select_speaker_messages, ++ }, ++ ) ++  ++ messages = [{"content": "Fred should both speak.", "name": "speaker_selector", "role": "user"}] ++ assert groupchat._validate_speaker_name(None, messages, None, None, 1, 1, agents) == ( ++ True, ++ { ++ "content": custom_no_names_msg.replace("{agentlist}", "['Alice', 'Bob']"), ++ "override_role": groupchat.role_for_select_speaker_messages, ++ }, ++ ) ++  ++ # Test with empty strings ++ with pytest.raises(ValueError, match="select_speaker_auto_multiple_template cannot be empty or None."): ++ groupchat = autogen.GroupChat( ++ agents=[agent1, agent2], ++ messages=[], ++ speaker_selection_method="auto", ++ max_round=10, ++ select_speaker_auto_multiple_template="", ++ ) ++  ++ with pytest.raises(ValueError, match="select_speaker_auto_none_template cannot be empty or None."): ++ groupchat = autogen.GroupChat( ++ agents=[agent1, agent2], ++ messages=[], ++ speaker_selection_method="auto", ++ max_round=10, ++ select_speaker_auto_none_template="", ++ ) ++  ++ # Test with None ++ with pytest.raises(ValueError, match="select_speaker_auto_multiple_template cannot be empty or None."): ++ groupchat = autogen.GroupChat( ++ agents=[agent1, agent2], ++ messages=[], ++ speaker_selection_method="auto", ++ max_round=10, ++ select_speaker_auto_multiple_template=None, ++ ) ++  ++ with pytest.raises(ValueError, match="select_speaker_auto_none_template cannot be empty or None."): ++ groupchat = autogen.GroupChat( ++ agents=[agent1, agent2], ++ messages=[], ++ speaker_selection_method="auto", ++ max_round=10, ++ select_speaker_auto_none_template=None, ++ ) ++  ++  + if __name__ == "__main__": + # test_func_call_groupchat() + # test_broadcast() +@@@ -1478,5 -1784,7 +1818,9 @@@ + # test_role_for_select_speaker_messages() + # test_select_speaker_message_and_prompt_templates() + # test_speaker_selection_agent_name_match() + + test_role_for_reflection_summary() ++ test_speaker_selection_auto_process_result() ++ test_speaker_selection_validate_speaker_name() ++ test_select_speaker_auto_messages() +++main + # pass From 0f3f5d546ddd6d98f0292911fba1373d523edba6 Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Sun, 5 May 2024 15:15:02 -0300 Subject: [PATCH 08/12] fixed old arg name --- test/agentchat/test_groupchat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index 1a8cec9767b..6e5e6736e37 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -1477,7 +1477,7 @@ def test_role_for_reflection_summary(): max_turns=2, message="hello", summary_method="reflection_with_llm", - summary_args={"role": role_name}, + summary_args={"summary_role": role_name}, ) mock_generate_oai_reply_from_client.assert_called_once() From ed1cdf225b857f2a4139bbb96211cbfcc9835937 Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Sat, 11 May 2024 14:40:42 -0300 Subject: [PATCH 09/12] =?UTF-8?q?Delete=20dasda=EF=80=9B=EF=80=9B=EF=80=9B?= =?UTF-8?q?=EF=80=9Basd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No idea what this file was about --- ...33\357\200\233\357\200\233\357\200\233asd" | 385 ------------------ 1 file changed, 385 deletions(-) delete mode 100644 "dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" diff --git "a/dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" "b/dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" deleted file mode 100644 index 3867aa00b78..00000000000 --- "a/dasda\357\200\233\357\200\233\357\200\233\357\200\233asd" +++ /dev/null @@ -1,385 +0,0 @@ -commit 9b3555eec4cec5b75af6b49558aee7c8ccad0ee1 (HEAD -> add-role-to-reflection-with-llm, origin/add-role-to-reflection-with-llm) -Merge: d309e15f 3e693578 -Author: MarianoMolina -Date: Thu May 2 12:03:26 2024 -0300 - - Merge branch 'main' into add-role-to-reflection-with-llm - -diff --cc test/agentchat/test_groupchat.py -index 3887b8e3,a4689bd5..3d58bedd ---- a/test/agentchat/test_groupchat.py -+++ b/test/agentchat/test_groupchat.py -@@@ -1425,41 -1443,329 +1443,363 @@@ def test_speaker_selection_agent_name_m - ) - assert result == {} -  - +def test_role_for_reflection_summary(): - + llm_config={"config_list": [{"model": "mock", "api_key": "mock"}]} - + agent1 = autogen.ConversableAgent( - + "alice", - + max_consecutive_auto_reply=10, - + human_input_mode="NEVER", - + llm_config=False, - + default_auto_reply="This is alice speaking.", - + ) - + agent2 = autogen.ConversableAgent( - + "bob", - + max_consecutive_auto_reply=10, - + human_input_mode="NEVER", - + llm_config=False, - + default_auto_reply="This is bob speaking.", - + ) - + groupchat = autogen.GroupChat(agents=[agent1, agent2], messages=[], max_round=3, speaker_selection_method="round_robin") - + group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config) - + - + role_name = "user" - + with mock.patch.object(autogen.ConversableAgent, '_generate_oai_reply_from_client') as mock_generate_oai_reply_from_client: - + mock_generate_oai_reply_from_client.return_value = "Mocked summary" - + - + res = agent1.initiate_chat( - + group_chat_manager, - + max_turns=2, - + message="hello", - + summary_method="reflection_with_llm", - + summary_args={"role": role_name} - + ) - + - + mock_generate_oai_reply_from_client.assert_called_once() - + args, kwargs = mock_generate_oai_reply_from_client.call_args - + assert kwargs['messages'][-1]["role"] == role_name -  -+ def test_speaker_selection_auto_process_result(): -+ """ -+ Tests the return result of the 2-agent chat used for speaker selection for the auto method. -+ The last message of the messages passed in will contain a pass or fail. -+ If passed, the message will contain the name of the correct agent and that agent will be returned. -+ If failed, the message will contain the reason for failure for the last attempt and the next -+ agent in the sequence will be returned. -+ """ -+ cmo = autogen.ConversableAgent( -+ name="Chief_Marketing_Officer", -+ human_input_mode="NEVER", -+ llm_config=False, -+ default_auto_reply="This is alice speaking.", -+ ) -+ pm = autogen.ConversableAgent( -+ name="Product_Manager", -+ human_input_mode="NEVER", -+ llm_config=False, -+ default_auto_reply="This is bob speaking.", -+ function_map={"test_func": lambda x: x}, -+ ) -+  -+ agent_list = [cmo, pm] -+ groupchat = autogen.GroupChat(agents=agent_list, messages=[], max_round=3) -+  -+ chat_result = autogen.ChatResult( -+ chat_id=None, -+ chat_history=[ -+ { -+ "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.", -+ "name": "Chairperson", -+ "role": "assistant", -+ }, -+ {"content": "You are an expert at finding the next speaker.", "role": "assistant"}, -+ {"content": "Product_Manager", "role": "user"}, -+ {"content": "UPDATED_BELOW", "role": "user"}, -+ ], -+ ) -+  -+ ### Agent selected successfully -+ chat_result.chat_history[3]["content"] = "[AGENT SELECTED]Product_Manager" -+  -+ # Product_Manager should be returned -+ assert groupchat._process_speaker_selection_result(chat_result, cmo, agent_list) == pm -+  -+ ### Agent not selected successfully -+ chat_result.chat_history[3][ -+ "content" -+ ] = "[AGENT SELECTION FAILED]Select speaker attempt #3 of 3 failed as it did not include any agent names." -+  -+ # The next speaker in the list will be selected, which will be the Product_Manager (as the last speaker is the Chief_Marketing_Officer) -+ assert groupchat._process_speaker_selection_result(chat_result, cmo, agent_list) == pm -+  -+ ### Invalid result messages, will return the next agent -+ chat_result.chat_history[3]["content"] = "This text should not be here." -+  -+ # The next speaker in the list will be selected, which will be the Chief_Marketing_Officer (as the last speaker is the Product_Maanger) -+ assert groupchat._process_speaker_selection_result(chat_result, pm, agent_list) == cmo -+  -+  -+ def test_speaker_selection_validate_speaker_name(): -+ """ -+ Tests the speaker name validation function used to evaluate the return result of the LLM -+ during speaker selection in 'auto' mode. -+  -+ Function: _validate_speaker_name -+  -+ If a single agent name is returned by the LLM, it will add a relevant message to the chat messages and return True, None -+ If multiple agent names are returned and there are attempts left, it will return a message to be used to prompt the LLM to try again -+ If multiple agent names are return and there are no attempts left, it will add a relevant message to the chat messages and return True, None -+ If no agent names are returned and there are attempts left, it will return a message to be used to prompt the LLM to try again -+ If no agent names are returned and there are no attempts left, it will add a relevant message to the chat messages and return True, None -+  -+ When returning a message, it will include the 'override_role' key and value to support the GroupChat role_for_select_speaker_messages attribute -+ """ -+  -+ # Group Chat setup -+ cmo = autogen.ConversableAgent( -+ name="Chief_Marketing_Officer", -+ human_input_mode="NEVER", -+ llm_config=False, -+ default_auto_reply="This is alice speaking.", -+ ) -+ pm = autogen.ConversableAgent( -+ name="Product_Manager", -+ human_input_mode="NEVER", -+ llm_config=False, -+ default_auto_reply="This is bob speaking.", -+ function_map={"test_func": lambda x: x}, -+ ) -+  -+ agent_list = [cmo, pm] -+ agent_list_string = f"{[agent.name for agent in agent_list]}" -+ groupchat = autogen.GroupChat(agents=agent_list, messages=[], max_round=3) -+  -+ # Speaker Selection 2-agent chat setup -+  -+ # Agent for selecting a single agent name from the response -+ speaker_selection_agent = autogen.ConversableAgent( -+ "speaker_selection_agent", -+ ) -+  -+ # Agent for checking the response from the speaker_select_agent -+ checking_agent = autogen.ConversableAgent("checking_agent") -+  -+ # Select speaker messages -+ select_speaker_messages = [ -+ { -+ "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.", -+ "name": "Chairperson", -+ "role": "assistant", -+ }, -+ {"content": "You are an expert at finding the next speaker.", "role": "assistant"}, -+ {"content": "UPDATED_BELOW", "role": "user"}, -+ ] -+  -+ ### Single agent name returned -+ attempts_left = 2 -+ attempt = 1 -+ select_speaker_messages[-1]["content"] = "Product_Manager is the next to speak" -+  -+ result = groupchat._validate_speaker_name( -+ recipient=checking_agent, -+ messages=select_speaker_messages, -+ sender=speaker_selection_agent, -+ config=None, -+ attempts_left=attempts_left, -+ attempt=attempt, -+ agents=agent_list, -+ ) -+  -+ assert result == (True, None) -+ assert select_speaker_messages[-1]["content"] == "[AGENT SELECTED]Product_Manager" -+  -+ select_speaker_messages.pop(-1) # Remove the last message before the next test -+  -+ ### Multiple agent names returned with attempts left -+ attempts_left = 2 -+ attempt = 1 -+ select_speaker_messages[-1]["content"] = "Product_Manager must speak after the Chief_Marketing_Officer" -+  -+ result = groupchat._validate_speaker_name( -+ recipient=checking_agent, -+ messages=select_speaker_messages, -+ sender=speaker_selection_agent, -+ config=None, -+ attempts_left=attempts_left, -+ attempt=attempt, -+ agents=agent_list, -+ ) -+  -+ assert result == ( -+ True, -+ { -+ "content": groupchat.select_speaker_auto_multiple_template.format(agentlist=agent_list_string), -+ "override_role": groupchat.role_for_select_speaker_messages, -+ }, -+ ) -+  -+ ### Multiple agent names returned with no attempts left -+ attempts_left = 0 -+ attempt = 1 -+ select_speaker_messages[-1]["content"] = "Product_Manager must speak after the Chief_Marketing_Officer" -+  -+ result = groupchat._validate_speaker_name( -+ recipient=checking_agent, -+ messages=select_speaker_messages, -+ sender=speaker_selection_agent, -+ config=None, -+ attempts_left=attempts_left, -+ attempt=attempt, -+ agents=agent_list, -+ ) -+  -+ assert result == (True, None) -+ assert ( -+ select_speaker_messages[-1]["content"] -+ == f"[AGENT SELECTION FAILED]Select speaker attempt #{attempt} of {attempt + attempts_left} failed as it returned multiple names." -+ ) -+  -+ select_speaker_messages.pop(-1) # Remove the last message before the next test -+  -+ ### No agent names returned with attempts left -+ attempts_left = 3 -+ attempt = 2 -+ select_speaker_messages[-1]["content"] = "The PM must speak after the CMO" -+  -+ result = groupchat._validate_speaker_name( -+ recipient=checking_agent, -+ messages=select_speaker_messages, -+ sender=speaker_selection_agent, -+ config=None, -+ attempts_left=attempts_left, -+ attempt=attempt, -+ agents=agent_list, -+ ) -+  -+ assert result == ( -+ True, -+ { -+ "content": groupchat.select_speaker_auto_none_template.format(agentlist=agent_list_string), -+ "override_role": groupchat.role_for_select_speaker_messages, -+ }, -+ ) -+  -+ ### Multiple agents returned with no attempts left -+ attempts_left = 0 -+ attempt = 3 -+ select_speaker_messages[-1]["content"] = "The PM must speak after the CMO" -+  -+ result = groupchat._validate_speaker_name( -+ recipient=checking_agent, -+ messages=select_speaker_messages, -+ sender=speaker_selection_agent, -+ config=None, -+ attempts_left=attempts_left, -+ attempt=attempt, -+ agents=agent_list, -+ ) -+  -+ assert result == (True, None) -+ assert ( -+ select_speaker_messages[-1]["content"] -+ == f"[AGENT SELECTION FAILED]Select speaker attempt #{attempt} of {attempt + attempts_left} failed as it did not include any agent names." -+ ) -+  -+  -+ def test_select_speaker_auto_messages(): -+ """ -+ In this test, two agents are part of a group chat which has customized select speaker "auto" multiple and no-name prompt messages. Both valid and empty string values will be used. -+ The expected behaviour is that the customized speaker selection "auto" messages will override the default values or throw exceptions if empty. -+ """ -+  -+ agent1 = autogen.ConversableAgent( -+ "Alice", -+ description="A wonderful employee named Alice.", -+ human_input_mode="NEVER", -+ llm_config=False, -+ ) -+ agent2 = autogen.ConversableAgent( -+ "Bob", -+ description="An amazing employee named Bob.", -+ human_input_mode="NEVER", -+ llm_config=False, -+ ) -+  -+ # Customised message for select speaker auto method where multiple agent names are returned -+ custom_multiple_names_msg = "You mentioned multiple names but we need just one. Select the best one. A reminder that the options are {agentlist}." -+  -+ # Customised message for select speaker auto method where no agent names are returned -+ custom_no_names_msg = "You forgot to select a single names and we need one, and only one. Select the best one. A reminder that the options are {agentlist}." -+  -+ # Test empty is_termination_msg function -+ groupchat = autogen.GroupChat( -+ agents=[agent1, agent2], -+ messages=[], -+ speaker_selection_method="auto", -+ max_round=10, -+ select_speaker_auto_multiple_template=custom_multiple_names_msg, -+ select_speaker_auto_none_template=custom_no_names_msg, -+ ) -+  -+ # Test using the _validate_speaker_name function, checking for the correct string and agentlist to be included -+ agents = [agent1, agent2] -+  -+ messages = [{"content": "Alice and Bob should both speak.", "name": "speaker_selector", "role": "user"}] -+ assert groupchat._validate_speaker_name(None, messages, None, None, 1, 1, agents) == ( -+ True, -+ { -+ "content": custom_multiple_names_msg.replace("{agentlist}", "['Alice', 'Bob']"), -+ "override_role": groupchat.role_for_select_speaker_messages, -+ }, -+ ) -+  -+ messages = [{"content": "Fred should both speak.", "name": "speaker_selector", "role": "user"}] -+ assert groupchat._validate_speaker_name(None, messages, None, None, 1, 1, agents) == ( -+ True, -+ { -+ "content": custom_no_names_msg.replace("{agentlist}", "['Alice', 'Bob']"), -+ "override_role": groupchat.role_for_select_speaker_messages, -+ }, -+ ) -+  -+ # Test with empty strings -+ with pytest.raises(ValueError, match="select_speaker_auto_multiple_template cannot be empty or None."): -+ groupchat = autogen.GroupChat( -+ agents=[agent1, agent2], -+ messages=[], -+ speaker_selection_method="auto", -+ max_round=10, -+ select_speaker_auto_multiple_template="", -+ ) -+  -+ with pytest.raises(ValueError, match="select_speaker_auto_none_template cannot be empty or None."): -+ groupchat = autogen.GroupChat( -+ agents=[agent1, agent2], -+ messages=[], -+ speaker_selection_method="auto", -+ max_round=10, -+ select_speaker_auto_none_template="", -+ ) -+  -+ # Test with None -+ with pytest.raises(ValueError, match="select_speaker_auto_multiple_template cannot be empty or None."): -+ groupchat = autogen.GroupChat( -+ agents=[agent1, agent2], -+ messages=[], -+ speaker_selection_method="auto", -+ max_round=10, -+ select_speaker_auto_multiple_template=None, -+ ) -+  -+ with pytest.raises(ValueError, match="select_speaker_auto_none_template cannot be empty or None."): -+ groupchat = autogen.GroupChat( -+ agents=[agent1, agent2], -+ messages=[], -+ speaker_selection_method="auto", -+ max_round=10, -+ select_speaker_auto_none_template=None, -+ ) -+  -+  - if __name__ == "__main__": - # test_func_call_groupchat() - # test_broadcast() -@@@ -1478,5 -1784,7 +1818,9 @@@ - # test_role_for_select_speaker_messages() - # test_select_speaker_message_and_prompt_templates() - # test_speaker_selection_agent_name_match() - + test_role_for_reflection_summary() -+ test_speaker_selection_auto_process_result() -+ test_speaker_selection_validate_speaker_name() -+ test_select_speaker_auto_messages() -++main - # pass From ba633865cc7436a15db7fd69a13b7584acbddeec Mon Sep 17 00:00:00 2001 From: MarianoMolina Date: Tue, 18 Jun 2024 14:12:07 -0300 Subject: [PATCH 10/12] Added notebook with stateflow example of optional user input --- autogen/agentchat/conversable_agent.py | 2 +- ...tchat_groupchat_user_input_stateflow.ipynb | 426 ++++++++++++++++++ test/agentchat/test_groupchat.py | 1 + 3 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 notebook/agentchat_groupchat_user_input_stateflow.ipynb diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index 89b2dd94345..8e7e08f0887 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -1218,7 +1218,7 @@ def _reflection_with_llm( else: raise ValueError("No OpenAIWrapper client is found.") response = self._generate_oai_reply_from_client(llm_client=llm_client, messages=messages, cache=cache) - return response + return self.generate_oai_reply(messages=messages, config=llm_client) def _check_chat_queue_for_sender(self, chat_queue: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ diff --git a/notebook/agentchat_groupchat_user_input_stateflow.ipynb b/notebook/agentchat_groupchat_user_input_stateflow.ipynb new file mode 100644 index 00000000000..62b96a5a328 --- /dev/null +++ b/notebook/agentchat_groupchat_user_input_stateflow.ipynb @@ -0,0 +1,426 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# User Input + StateFlow GroupChat: Build Workflows through State-Oriented Actions that can intelligently include the user in the process\n", + "\n", + "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. In this notebook, we introduce how to use groupchat to build workflows with AutoGen agents from a state-oriented perspective and allow for user input during the process.\n", + "\n", + "This notebook is an expansion of [StateFlow: Build Workflows through State-Oriented Actions](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat_stateflow/) to allow for user input during the process. \n", + "\n", + "\n", + "````{=mdx}\n", + ":::info Requirements\n", + "Install `pyautogen`:\n", + "```bash\n", + "pip install pyautogen\n", + "```\n", + "\n", + "For more information, please refer to the [installation guide](/docs/installation/).\n", + ":::\n", + "````" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set your API Endpoint\n", + "\n", + "The [`config_list_from_json`](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils#config_list_from_json) function loads a list of configurations from an environment variable or a json file." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import autogen\n", + "\n", + "config_list = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\n", + " \"tags\": [\"gpt-4o-2024-05-13\"],\n", + " },\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "````{=mdx}\n", + ":::tip\n", + "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n", + ":::\n", + "````\n", + "## Workflow: Arxiv Research\n", + "We define the following agents:\n", + "- Planner: Create a plan.\n", + "- Admin: Human in the loop to provide optional feedback.\n", + "- Engineer: Retrieve papers from the internet by writing code.\n", + "- Executor: Execute the code.\n", + "- Scientist: Read the papers and write a summary.\n", + "\n", + "The pipeline is the following:\n", + "0. Input: a str prompt -> a research topic from Arxiv\n", + "1. Plan: The planner will create a plan based on the prompt and decide if it would like user input. \n", + "2. Optional_plan_input: If they don't, workflow continues, if they do, the Admin will be asked for input before returning to step 1.\n", + "3. Code: The engineer will write code to retrieve papers from the internet. \n", + "4. Optional_code_input: If the engineer is stuck or doesn't know how to solve the task, they can ask the user for input before returning to step 3. \n", + "5. Execution: The code will be executed by executor. If the code execution fails, we go back to step 3. \n", + "6. Summary: If the code is executed successfully, the scientist will read the papers and write a summary. They can choose to ask the user for input. \n", + "7. Optional_summary_input: Once the summary is received, if user input was asked, the Admin will be asked for input, and we go back to step 5. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "gpt4_config = {\n", + " \"cache_seed\": 42, # change the cache_seed for different trials\n", + " \"temperature\": 0,\n", + " \"config_list\": config_list,\n", + " \"timeout\": 120,\n", + "}\n", + "\n", + "planner = autogen.AssistantAgent(\n", + " name=\"Planner\",\n", + " system_message=\"\"\"You are the Planner for a research workflow. The user will provide a task, and your job is to suggest a plan. \n", + "The plan can involve an engineer who can write code and a scientist who doesn't write code. The engineer always acts first, and then the scientist. \n", + "Do not suggest recursive plans where agents need to interact multiple times. Keep it simple. \n", + "Explain the plan first. Be clear which step is performed by an engineer, and which step is performed by a scientist. \n", + "If you need to ask the user for more information, you can do so by including ´ASK_USER:´ in your response. If you do, use the answer and respond with a complete plan. \n", + "\"\"\",\n", + " llm_config=gpt4_config,\n", + ")\n", + "\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"Admin\",\n", + " system_message=\"A human admin. Interacts with the planner and scientist if asked.\",\n", + " code_execution_config=False,\n", + " default_auto_reply=\"No user input received\"\n", + ")\n", + "\n", + "engineer = autogen.AssistantAgent(\n", + " name=\"Engineer\",\n", + " llm_config=gpt4_config,\n", + " system_message=\"\"\"You are an Engineer. You follow an approved plan. You write python/shell code to solve tasks. Wrap the code in a code block that specifies the script type. The user can't modify your code. So do not suggest incomplete code which requires others to modify. Don't use a code block if it's not intended to be executed by the executor.\n", + "Don't include multiple code blocks in one response. Do not ask others to copy and paste the result. Check the execution result returned by the executor.\n", + "If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.\n", + "If you can't solve the task, you can ask the user for more information by including ´ASK_USER:´ in your response. If you do, use the answer and respond with a complete code solution.\n", + "\"\"\",\n", + ")\n", + "scientist = autogen.AssistantAgent(\n", + " name=\"Scientist\",\n", + " llm_config=gpt4_config,\n", + " system_message=\"\"\"You are a Scientist. You are part of a research workflow, which has been planned. \n", + "Follow the plan the Planner suggested, and provide a summary of the information retrieved. You don't write code. \n", + "If you need to ask the user for more information, you can do so by including ´ASK_USER:´ in your response. If you do, use the answer and respond with a complete summary.\"\"\",\n", + ")\n", + "\n", + "executor = autogen.UserProxyAgent(\n", + " name=\"Executor\",\n", + " system_message=\"Executor. Execute the code written by the engineer and report the result.\",\n", + " human_input_mode=\"NEVER\",\n", + " code_execution_config={\n", + " \"last_n_messages\": 3,\n", + " \"work_dir\": \"paper\",\n", + " \"use_docker\": False,\n", + " }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly.\n", + ")\n", + "\n", + "from typing import Dict, List\n", + "from autogen import Agent\n", + "\n", + "\n", + "def custom_speaker_selection_func(last_speaker: Agent, groupchat: autogen.GroupChat):\n", + " \"\"\"Define a customized speaker selection function.\n", + " A recommended way is to define a transition for each speaker in the groupchat.\n", + "\n", + " Returns:\n", + " Return an `Agent` class or a string from ['auto', 'manual', 'random', 'round_robin'] to select a default method to use.\n", + " \"\"\"\n", + " messages = groupchat.messages\n", + "\n", + " if len(messages) <= 1:\n", + " return planner\n", + " \n", + " if last_speaker is user_proxy:\n", + " if messages[-2][\"name\"] == \"Planner\":\n", + " # If it is the planning stage, let the planner continue\n", + " return planner\n", + " elif messages[-2][\"name\"] == \"Scientist\":\n", + " # If the last message is from the scientist, let the scientist continue\n", + " return scientist\n", + " elif messages[-2][\"name\"] == \"Engineer\":\n", + " # If the last message is from the engineer, let the engineer continue\n", + " return engineer\n", + " \n", + " elif \"ASK_USER:\" in messages[-1][\"content\"] and last_speaker is not executor:\n", + " # If someone asks for user input, let the user speak\n", + " return user_proxy\n", + "\n", + " elif last_speaker is planner:\n", + " # If the last message is from the planner, we continue to the engineer because they didn't ask for user input\n", + " return engineer\n", + "\n", + " elif last_speaker is engineer:\n", + " if \"```python\" in messages[-1][\"content\"]:\n", + " # If the last message is a python code block, let the executor speak\n", + " return executor\n", + " else:\n", + " # Otherwise, let the engineer continue -> This can be confusing, so adding a message here to clarify the issue would be better\n", + " return engineer\n", + "\n", + " elif last_speaker is executor:\n", + " if \"exitcode: 1\" in messages[-1][\"content\"]:\n", + " # If the last message indicates an error, let the engineer improve the code\n", + " return engineer\n", + " else:\n", + " # Otherwise, let the scientist speak\n", + " return scientist\n", + "\n", + " elif last_speaker is scientist:\n", + " # If the last message is from the scientist, we end because they didn't request user input\n", + " return None\n", + " else:\n", + " return \"random\"\n", + "\n", + "\n", + "groupchat = autogen.GroupChat(\n", + " agents=[user_proxy, engineer, scientist, planner, executor],\n", + " messages=[],\n", + " max_round=5,\n", + " speaker_selection_method=custom_speaker_selection_func,\n", + ")\n", + "manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=gpt4_config)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mAdmin\u001b[0m (to chat_manager):\n", + "\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mPlanner\u001b[0m (to chat_manager):\n", + "\n", + "### Plan\n", + "\n", + "1. **Engineer Step:**\n", + " - The engineer will write a script to search for the latest papers about GPT-4 on arXiv. This script will use the arXiv API to fetch the most recent papers related to GPT-4.\n", + " - The engineer will then extract the relevant information from the fetched papers, such as the title, authors, abstract, and publication date.\n", + " - The engineer will provide a summary of the latest paper found, including a link to the full paper.\n", + "\n", + "2. **Scientist Step:**\n", + " - The scientist will read the summary and the full paper provided by the engineer.\n", + " - The scientist will analyze the paper to identify potential applications of GPT-4 in software.\n", + " - The scientist will document these potential applications in a detailed report.\n", + "\n", + "### Execution\n", + "\n", + "1. **Engineer Step:**\n", + " - Write a Python script to interact with the arXiv API and search for the latest papers about GPT-4.\n", + " - Extract and summarize the relevant information from the latest paper found.\n", + "\n", + "2. **Scientist Step:**\n", + " - Read and analyze the summary and full paper.\n", + " - Identify and document potential applications of GPT-4 in software.\n", + "\n", + "Would you like to proceed with this plan, or do you have any additional requirements or constraints?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mEngineer\u001b[0m (to chat_manager):\n", + "\n", + "Let's proceed with the plan. I'll start by writing a Python script to search for the latest papers about GPT-4 on arXiv and extract relevant information.\n", + "\n", + "```python\n", + "import requests\n", + "import xml.etree.ElementTree as ET\n", + "\n", + "def search_arxiv(query, max_results=1):\n", + " url = f'http://export.arxiv.org/api/query?search_query={query}&start=0&max_results={max_results}'\n", + " response = requests.get(url)\n", + " if response.status_code == 200:\n", + " return response.text\n", + " else:\n", + " return None\n", + "\n", + "def parse_arxiv_response(response):\n", + " root = ET.fromstring(response)\n", + " ns = {'arxiv': 'http://www.w3.org/2005/Atom'}\n", + " entries = root.findall('arxiv:entry', ns)\n", + " papers = []\n", + " for entry in entries:\n", + " title = entry.find('arxiv:title', ns).text\n", + " authors = [author.find('arxiv:name', ns).text for author in entry.findall('arxiv:author', ns)]\n", + " summary = entry.find('arxiv:summary', ns).text\n", + " published = entry.find('arxiv:published', ns).text\n", + " link = entry.find('arxiv:id', ns).text\n", + " papers.append({\n", + " 'title': title,\n", + " 'authors': authors,\n", + " 'summary': summary,\n", + " 'published': published,\n", + " 'link': link\n", + " })\n", + " return papers\n", + "\n", + "def main():\n", + " query = 'all: \"GPT-4\"'\n", + " response = search_arxiv(query)\n", + " if response:\n", + " papers = parse_arxiv_response(response)\n", + " if papers:\n", + " latest_paper = papers[0]\n", + " print(\"Title:\", latest_paper['title'])\n", + " print(\"Authors:\", \", \".join(latest_paper['authors']))\n", + " print(\"Published:\", latest_paper['published'])\n", + " print(\"Summary:\", latest_paper['summary'])\n", + " print(\"Link:\", latest_paper['link'])\n", + " else:\n", + " print(\"No papers found.\")\n", + " else:\n", + " print(\"Failed to fetch data from arXiv.\")\n", + "\n", + "if __name__ == \"__main__\":\n", + " main()\n", + "```\n", + "\n", + "This script will search for the latest paper about GPT-4 on arXiv, extract relevant information, and print it out. Let's execute this script to find the latest paper and its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33mExecutor\u001b[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Title: Can LLMs like GPT-4 outperform traditional AI tools in dementia\n", + " diagnosis? Maybe, but not today\n", + "Authors: Zhuo Wang, Rongzhen Li, Bowen Dong, Jie Wang, Xiuxing Li, Ning Liu, Chenhui Mao, Wei Zhang, Liling Dong, Jing Gao, Jianyong Wang\n", + "Published: 2023-06-02T12:47:45Z\n", + "Summary: Recent investigations show that large language models (LLMs), specifically\n", + "GPT-4, not only have remarkable capabilities in common Natural Language\n", + "Processing (NLP) tasks but also exhibit human-level performance on various\n", + "professional and academic benchmarks. However, whether GPT-4 can be directly\n", + "used in practical applications and replace traditional artificial intelligence\n", + "(AI) tools in specialized domains requires further experimental validation. In\n", + "this paper, we explore the potential of LLMs such as GPT-4 to outperform\n", + "traditional AI tools in dementia diagnosis. Comprehensive comparisons between\n", + "GPT-4 and traditional AI tools are conducted to examine their diagnostic\n", + "accuracy in a clinical setting. Experimental results on two real clinical\n", + "datasets show that, although LLMs like GPT-4 demonstrate potential for future\n", + "advancements in dementia diagnosis, they currently do not surpass the\n", + "performance of traditional AI tools. The interpretability and faithfulness of\n", + "GPT-4 are also evaluated by comparison with real doctors. We discuss the\n", + "limitations of GPT-4 in its current state and propose future research\n", + "directions to enhance GPT-4 in dementia diagnosis.\n", + "\n", + "Link: http://arxiv.org/abs/2306.01499v1\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mScientist\u001b[0m (to chat_manager):\n", + "\n", + "### Summary of the Latest Paper on GPT-4\n", + "\n", + "**Title:** Can LLMs like GPT-4 outperform traditional AI tools in dementia diagnosis? Maybe, but not today\n", + "\n", + "**Authors:** Zhuo Wang, Rongzhen Li, Bowen Dong, Jie Wang, Xiuxing Li, Ning Liu, Chenhui Mao, Wei Zhang, Liling Dong, Jing Gao, Jianyong Wang\n", + "\n", + "**Published Date:** 2023-06-02\n", + "\n", + "**Summary:**\n", + "The paper investigates the potential of large language models (LLMs), specifically GPT-4, in outperforming traditional AI tools in the domain of dementia diagnosis. While GPT-4 has shown remarkable capabilities in various Natural Language Processing (NLP) tasks and has achieved human-level performance on several professional and academic benchmarks, its effectiveness in specialized domains like dementia diagnosis remains uncertain.\n", + "\n", + "The authors conducted comprehensive comparisons between GPT-4 and traditional AI tools to evaluate their diagnostic accuracy in clinical settings. The experimental results, based on two real clinical datasets, indicate that although GPT-4 shows promise for future advancements in dementia diagnosis, it currently does not surpass the performance of traditional AI tools. Additionally, the paper evaluates the interpretability and faithfulness of GPT-4 by comparing its outputs with real doctors' assessments. The authors discuss the limitations of GPT-4 in its current state and propose future research directions to enhance its capabilities in dementia diagnosis.\n", + "\n", + "**Link to Full Paper:** [arXiv:2306.01499v1](http://arxiv.org/abs/2306.01499v1)\n", + "\n", + "### Potential Applications of GPT-4 in Software\n", + "\n", + "Based on the findings and discussions in the paper, here are some potential applications of GPT-4 in software:\n", + "\n", + "1. **Medical Diagnosis Assistance:**\n", + " - **Clinical Decision Support Systems:** GPT-4 can be integrated into clinical decision support systems to assist healthcare professionals by providing preliminary diagnostic suggestions and insights based on patient data.\n", + " - **Symptom Checker Tools:** GPT-4 can be used to develop advanced symptom checker tools that help patients understand potential health issues and recommend when to seek medical attention.\n", + "\n", + "2. **Healthcare Data Analysis:**\n", + " - **Medical Record Summarization:** GPT-4 can be employed to summarize patient medical records, making it easier for healthcare providers to quickly review patient histories and make informed decisions.\n", + " - **Natural Language Processing of Medical Texts:** GPT-4 can be used to analyze and extract valuable information from large volumes of medical texts, such as research papers, clinical notes, and patient feedback.\n", + "\n", + "3. **Patient Interaction and Support:**\n", + " - **Virtual Health Assistants:** GPT-4 can power virtual health assistants that provide patients with information about their conditions, treatment options, and medication instructions.\n", + " - **Mental Health Support:** GPT-4 can be used to develop chatbots and virtual therapists that offer mental health support and counseling to patients.\n", + "\n", + "4. **Research and Development:**\n", + " - **Medical Research Assistance:** GPT-4 can assist researchers by analyzing and summarizing scientific literature, identifying trends, and generating hypotheses for further investigation.\n", + " - **Clinical Trial Matching:** GPT-4 can help match patients with suitable clinical trials based on their medical history and condition, improving the efficiency of clinical research.\n", + "\n", + "5. **Education and Training:**\n", + " - **Medical Education Tools:** GPT-4 can be used to create interactive educational tools and simulations for medical students and professionals, enhancing their learning experience.\n", + " - **Continuing Medical Education:** GPT-4 can provide up-to-date information and training materials to healthcare professionals, helping them stay current with the latest advancements in their field.\n", + "\n", + "While GPT-4 shows potential in these applications, it is important to note that its current limitations must be addressed through further research and development to ensure its reliability and effectiveness in specialized domains like healthcare.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "chat_result = user_proxy.initiate_chat(\n", + " manager, \n", + " message=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\",\n", + " max_turns=20,\n", + ")" + ] + } + ], + "metadata": { + "front_matter": { + "description": "StateFlow: Build Workflows through State-Oriented Actions", + "tags": [ + "orchestration", + "group chat", + "research" + ] + }, + "kernelspec": { + "display_name": "flaml", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index 517a88dd10b..0176c9e8990 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -2037,3 +2037,4 @@ def test_manager_resume_messages(): # test_manager_resume_functions() # test_manager_resume_returns() # test_manager_resume_messages() + pass From 73208114d77c2e9bb4feea99bfe0e1c0c4e2131d Mon Sep 17 00:00:00 2001 From: Eric Zhu Date: Wed, 2 Oct 2024 15:27:57 -0700 Subject: [PATCH 11/12] Update notebook/agentchat_groupchat_user_input_stateflow.ipynb --- notebook/agentchat_groupchat_user_input_stateflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_groupchat_user_input_stateflow.ipynb b/notebook/agentchat_groupchat_user_input_stateflow.ipynb index 62b96a5a328..ead0d0ebae4 100644 --- a/notebook/agentchat_groupchat_user_input_stateflow.ipynb +++ b/notebook/agentchat_groupchat_user_input_stateflow.ipynb @@ -14,7 +14,7 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", "pip install pyautogen\n", "```\n", From 8acbb1976aa5cfc4f00a10ec113a14511dc53c72 Mon Sep 17 00:00:00 2001 From: Eric Zhu Date: Wed, 2 Oct 2024 15:28:07 -0700 Subject: [PATCH 12/12] Update notebook/agentchat_groupchat_user_input_stateflow.ipynb --- notebook/agentchat_groupchat_user_input_stateflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_groupchat_user_input_stateflow.ipynb b/notebook/agentchat_groupchat_user_input_stateflow.ipynb index ead0d0ebae4..4c8043958d2 100644 --- a/notebook/agentchat_groupchat_user_input_stateflow.ipynb +++ b/notebook/agentchat_groupchat_user_input_stateflow.ipynb @@ -16,7 +16,7 @@ ":::info Requirements\n", "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n",