From 899b250c08c192a88385c0485a277fad77869d2b Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Sun, 11 Feb 2024 21:25:09 -0500 Subject: [PATCH] Adding callable summary_method support and enhancements to initiate_chats (#1628) * initiate_chats enhancements * callable summary_method * summary method * summary method default * docstr * add timeout to slient pip install test * consolidate_chat_info * a_initiate_chat * AssertionError test * update tests --------- Co-authored-by: Eric Zhu --- autogen/agentchat/conversable_agent.py | 124 +++++++++++++----- .../coding/embedded_ipython_code_executor.py | 2 +- test/agentchat/test_chats.py | 110 +++++++++++++++- test/agentchat/test_function_call.py | 24 +++- .../test_embedded_ipython_code_executor.py | 2 +- 5 files changed, 221 insertions(+), 41 deletions(-) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index fbba53e5e8d..895eb7b5122 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -66,6 +66,7 @@ class ConversableAgent(LLMAgent): MAX_CONSECUTIVE_AUTO_REPLY = 100 # maximum number of consecutive auto replies (subject to future change) DEFAULT_summary_prompt = "Summarize the takeaway from the conversation. Do not add any introductory phrases." + DEFAULT_summary_method = "last_msg" llm_config: Union[Dict, Literal[False]] def __init__( @@ -766,12 +767,20 @@ def initiate_chat( cache (Cache or None): the cache client to be used for this conversation. **context: any context information. It has the following reserved fields: "message": a str of message. Needs to be provided. Otherwise, input() will be called to get the initial message. - "summary_method": a string specify the method to get a summary from the chat. - Supported methods are "last_msg" and "reflection_with_llm". - when set "last_msg", it returns the last message of the dialog as the summary. - when set "reflection_with_llm", it returns a summary extracted using an llm client. - "llm" requires the llm_config to be set in either the sender or the recipient so that an llm client is available. - When both the sender and the recipient have an llm client, the recipient's llm client will be used. + "summary_method": a string or callable specifying the method to get a summary from the chat. Default is DEFAULT_summary_method, i.e., "last_msg". + - Supported string are "last_msg" and "reflection_with_llm": + when set "last_msg", it returns the last message of the dialog as the summary. + when set "reflection_with_llm", it returns a summary extracted using an llm client. + `llm_config` must be set in either the recipient or sender. + "reflection_with_llm" requires the llm_config to be set in either the sender or the recipient. + - A callable summary_method should take the recipient and sender agent in a chat as input and return a string of summary. E.g, + ```python + def my_summary_method( + sender: ConversableAgent, + recipient: ConversableAgent, + ): + return recipient.last_message(sender)["content"] + ``` "summary_prompt": a string of text used to prompt a LLM-based agent (the sender or receiver agent) to reflext on the conversation and extract a summary when summary_method is "reflection_with_llm". Default is DEFAULT_summary_prompt, i.e., "Summarize takeaway from the conversation. Do not add any introductory phrases. If the intended request is NOT properly addressed, please point it out." @@ -785,6 +794,9 @@ def initiate_chat( Returns: ChatResult: an ChatResult object. """ + _chat_info = context.copy() + _chat_info["recipient"] = recipient + self._consolidate_chat_info(_chat_info) for agent in [self, recipient]: agent._raise_exception_on_async_reply_functions() agent.previous_cache = agent.client_cache @@ -792,7 +804,7 @@ def initiate_chat( self._prepare_chat(recipient, clear_history) self.send(self.generate_init_message(**context), recipient, silent=silent) summary = self._summarize_chat( - context.get("summary_method"), + context.get("summary_method", ConversableAgent.DEFAULT_summary_method), recipient, prompt=context.get("summary_prompt"), cache=cache, @@ -827,13 +839,16 @@ async def a_initiate_chat( Returns: ChatResult: an ChatResult object. """ + _chat_info = context.copy() + _chat_info["recipient"] = recipient + self._consolidate_chat_info(_chat_info) self._prepare_chat(recipient, clear_history) for agent in [self, recipient]: agent.previous_cache = agent.client_cache agent.client_cache = cache await self.a_send(await self.a_generate_init_message(**context), recipient, silent=silent) summary = self._summarize_chat( - context.get("summary_method"), + context.get("summary_method", ConversableAgent.DEFAULT_summary_method), recipient, prompt=context.get("summary_prompt"), cache=cache, @@ -851,24 +866,34 @@ async def a_initiate_chat( def _summarize_chat( self, - method, - agent: Optional[Agent] = None, + summary_method, + recipient: Optional[Agent] = None, prompt: Optional[str] = None, cache: Optional[Cache] = None, ) -> str: """Get a chat summary from an agent participating in a chat. Args: - method (str): the method to get the summary. - agent: the participating agent in a chat. + summary_method (str or callable): the summary_method to get the summary. + The callable summary_method should take the recipient and sender agent in a chat as input and return a string of summary. E.g, + ```python + def my_summary_method( + sender: ConversableAgent, + recipient: ConversableAgent, + ): + return recipient.last_message(sender)["content"] + ``` + recipient: the recipient agent in a chat. prompt (str): the prompt used to get a summary when summary_method is "reflection_with_llm". Returns: str: a chat summary from the agent. """ - agent = self if agent is None else agent + agent = self if recipient is None else recipient summary = "" - if method == "reflection_with_llm": + if summary_method is None: + return summary + if summary_method == "reflection_with_llm": prompt = ConversableAgent.DEFAULT_summary_prompt if prompt is None else prompt if not isinstance(prompt, str): raise ValueError("The summary_prompt must be a string.") @@ -877,13 +902,13 @@ def _summarize_chat( summary = self._reflection_with_llm(prompt, msg_list, llm_agent=agent, cache=cache) except BadRequestError as e: warnings.warn(f"Cannot extract summary using reflection_with_llm: {e}", UserWarning) - elif method == "last_msg" or method is None: + elif summary_method == "last_msg" or summary_method is None: try: summary = agent.last_message(self)["content"].replace("TERMINATE", "") except (IndexError, AttributeError) as e: warnings.warn(f"Cannot extract summary using last_msg: {e}", UserWarning) - else: - warnings.warn(f"Unsupported summary method: {method}", UserWarning) + elif isinstance(summary_method, Callable): + summary = summary_method(recipient, self) return summary def _reflection_with_llm( @@ -914,6 +939,22 @@ def _reflection_with_llm( response = self._generate_oai_reply_from_client(llm_client=llm_client, messages=messages, cache=cache) return response + def _consolidate_chat_info(self, chat_info: Union[Dict, List[Dict]]): + if isinstance(chat_info, dict): + chat_info = [chat_info] + for c in chat_info: + assert "recipient" in c, "recipient must be provided." + summary_method = c.get("summary_method") + assert ( + summary_method is None + or isinstance(summary_method, Callable) + or summary_method in ("last_msg", "reflection_with_llm") + ), "summary_method must be a string chosen from 'reflection_with_llm' or 'last_msg' or a callable, or None." + if summary_method == "reflection_with_llm": + assert ( + self.client is not None or c["recipient"].client is not None + ), "llm client must be set in either the recipient or sender when summary_method is reflection_with_llm." + def initiate_chats(self, chat_queue: List[Dict[str, Any]]) -> Dict[Agent, ChatResult]: """(Experimental) Initiate chats with multiple agents. TODO: add async version of this method. @@ -925,12 +966,20 @@ def initiate_chats(self, chat_queue: List[Dict[str, Any]]) -> Dict[Agent, ChatRe - "context": any context information, e.g., the request message. The following fields are reserved: "message" needs to be provided if the `generate_init_message` method is not overridden. Otherwise, input() will be called to get the initial message. - "summary_method" can be used to specify the method to extract a summary from the chat. - Supported methods are "last_msg" and "reflection_with_llm". - when set "last_msg", it returns the last message of the dialog as the summary. - when set "reflection_with_llm", it returns a summary extracted using an llm client. - `llm_config` must be set in either the recipient or sender. - "reflection_with_llm" requires the llm_config to be set in either the sender or the recipient. + "summary_method": a string or callable specifying the method to get a summary from the chat. Default is DEFAULT_summary_method, i.e., "last_msg". + - Supported string are "last_msg" and "reflection_with_llm": + when set "last_msg", it returns the last message of the dialog as the summary. + when set "reflection_with_llm", it returns a summary extracted using an llm client. + `llm_config` must be set in either the recipient or sender. + "reflection_with_llm" requires the llm_config to be set in either the sender or the recipient. + - A callable summary_method should take the recipient and sender agent in a chat as input and return a string of summary. E.g, + ```python + def my_summary_method( + sender: ConversableAgent, + recipient: ConversableAgent, + ): + return recipient.last_message(sender)["content"] + ``` "summary_prompt" can be used to specify the prompt used to extract a summary when summary_method is "reflection_with_llm". Default is None and the following default prompt will be used when "summary_method" is set to "reflection_with_llm": "Identify and extract the final solution to the originally asked question based on the conversation." @@ -940,13 +989,17 @@ def initiate_chats(self, chat_queue: List[Dict[str, Any]]) -> Dict[Agent, ChatRe Returns: a dictionary of ChatResult object from the finished chats of particular agents. """ + self._consolidate_chat_info(chat_queue) receipts_set = set() for chat_info in chat_queue: assert "recipient" in chat_info, "recipient must be provided." receipts_set.add(chat_info["recipient"]) - assert len(receipts_set) == len(chat_queue), "recipients must be different." - - self._chat_queue = chat_queue + if len(receipts_set) < len(chat_queue): + warnings.warn( + "Repetitive recipients detected: The chat history will be cleared by default if a recipient appears more than once. To retain the chat history, please set 'clear_history=False' in the configuration of the repeating agent.", + UserWarning, + ) + self._chat_queue = chat_queue.copy() self._finished_chats = {} while self._chat_queue: chat_info = self._chat_queue.pop(0) @@ -1971,11 +2024,20 @@ def generate_init_message(self, **context) -> Union[str, Dict]: Args: **context: any context information. It has the following reserved fields: "message": a str of message. - "summary_method": a string specify the method to get a summary from the chat. - Supported methods are "last_msg" and "reflection_with_llm". - when set "last_msg", it returns the last message of the dialog as the summary. - when set "reflection_with_llm", it returns a summary extracted using an llm client. - "llm" requires the llm_config to be set in either the sender or the recipient so that an llm client is available. + "summary_method": a string or callable specifying the method to get a summary from the chat. Default is DEFAULT_summary_method, i.e., "last_msg". + - Supported string are "last_msg" and "reflection_with_llm": + when set "last_msg", it returns the last message of the dialog as the summary. + when set "reflection_with_llm", it returns a summary extracted using an llm client. + `llm_config` must be set in either the recipient or sender. + "reflection_with_llm" requires the llm_config to be set in either the sender or the recipient. + - A callable summary_method should take the recipient and sender agent in a chat as input and return a string of summary. E.g, + ```python + def my_summary_method( + sender: ConversableAgent, + recipient: ConversableAgent, + ): + return recipient.last_message(sender)["content"] + ``` When both the sender and the recipient have an llm client, the recipient's llm client will be used. "summary_prompt": a string of text used to prompt a LLM-based agent (the sender or receiver agent) to reflext on the conversation and extract a summary when summary_method is "reflection_with_llm". diff --git a/autogen/coding/embedded_ipython_code_executor.py b/autogen/coding/embedded_ipython_code_executor.py index 01a640015dd..c85798f7503 100644 --- a/autogen/coding/embedded_ipython_code_executor.py +++ b/autogen/coding/embedded_ipython_code_executor.py @@ -14,7 +14,7 @@ from .base import CodeBlock, CodeExtractor, CodeResult from .markdown_code_extractor import MarkdownCodeExtractor -__all__ = ("EmbeddedIPythonCodeExecutor",) +__all__ = ("EmbeddedIPythonCodeExecutor", "IPythonCodeResult") class IPythonCodeResult(CodeResult): diff --git a/test/agentchat/test_chats.py b/test/agentchat/test_chats.py index 9b578c00c70..dfa1617cba5 100644 --- a/test/agentchat/test_chats.py +++ b/test/agentchat/test_chats.py @@ -148,8 +148,8 @@ def test_chats(): financial_tasks = [ """What are the full names of NVDA and TESLA.""", - """Investigate the reasons.""", - """Pros and cons of the companies I'm interested in. Keep it short.""", + """Get their stock price.""", + """Analyze pros and cons. Keep it short.""", ] writing_tasks = ["""Develop a short but engaging blog post using any information provided."""] @@ -185,20 +185,29 @@ def test_chats(): }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly. ) + def my_summary_method(recipient, sender): + return recipient.chat_messages[sender][0].get("content", "") + chat_res = user.initiate_chats( [ { "recipient": financial_assistant_1, "message": financial_tasks[0], - "clear_history": True, "silent": False, - "summary_method": "last_msg", + "summary_method": my_summary_method, }, { "recipient": financial_assistant_2, "message": financial_tasks[1], + "silent": True, "summary_method": "reflection_with_llm", }, + { + "recipient": financial_assistant_1, + "message": financial_tasks[2], + "summary_method": "last_msg", + "clear_history": False, + }, { "recipient": writer, "message": writing_tasks[0], @@ -216,9 +225,97 @@ def test_chats(): print(writer_res.summary, writer_res.cost) print(all_res[financial_assistant_1].human_input) print(all_res[financial_assistant_1].summary) + print(all_res[financial_assistant_1].chat_history) + print(all_res[financial_assistant_2].summary) # print(blogpost.summary, insights_and_blogpost) +@pytest.mark.skipif(skip_openai, reason="requested to skip openai tests") +def test_chats_exceptions(): + config_list = autogen.config_list_from_json( + OAI_CONFIG_LIST, + file_location=KEY_LOC, + ) + + financial_tasks = [ + """What are the full names of NVDA and TESLA.""", + """Get their stock price.""", + """Analyze pros and cons. Keep it short.""", + ] + + financial_assistant_1 = AssistantAgent( + name="Financial_assistant_1", + llm_config={"config_list": config_list}, + ) + financial_assistant_2 = AssistantAgent( + name="Financial_assistant_2", + llm_config={"config_list": config_list}, + ) + user = UserProxyAgent( + name="User", + human_input_mode="NEVER", + is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0, + code_execution_config={ + "last_n_messages": 1, + "work_dir": "tasks", + "use_docker": False, + }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly. + ) + + user_2 = UserProxyAgent( + name="User", + human_input_mode="NEVER", + is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0, + code_execution_config={ + "last_n_messages": 1, + "work_dir": "tasks", + "use_docker": False, + }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly. + ) + + with pytest.raises( + AssertionError, + match="summary_method must be a string chosen from 'reflection_with_llm' or 'last_msg' or a callable, or None.", + ): + user.initiate_chats( + [ + { + "recipient": financial_assistant_1, + "message": financial_tasks[0], + "silent": False, + "summary_method": "last_msg", + }, + { + "recipient": financial_assistant_2, + "message": financial_tasks[2], + "summary_method": "llm", + "clear_history": False, + }, + ] + ) + + with pytest.raises( + AssertionError, + match="llm client must be set in either the recipient or sender when summary_method is reflection_with_llm.", + ): + user.initiate_chats( + [ + { + "recipient": financial_assistant_1, + "message": financial_tasks[0], + "silent": False, + "summary_method": "last_msg", + }, + { + "recipient": user_2, + "message": financial_tasks[2], + "clear_history": False, + "summary_method": "reflection_with_llm", + }, + ] + ) + + @pytest.mark.skipif(skip_openai, reason="requested to skip openai tests") def test_chats_w_func(): config_list = autogen.config_list_from_json( @@ -281,7 +378,8 @@ def currency_calculator( if __name__ == "__main__": - # test_chats() + test_chats() + # test_chats_exceptions() # test_chats_group() # test_chats_w_func() - test_chat_messages_for_summary() + # test_chat_messages_for_summary() diff --git a/test/agentchat/test_function_call.py b/test/agentchat/test_function_call.py index 52430c70bcb..09d17db5eb9 100644 --- a/test/agentchat/test_function_call.py +++ b/test/agentchat/test_function_call.py @@ -256,10 +256,30 @@ def test_update_function(): assert "greet_user" not in messages2 print("Chat summary and cost", res2.summary, res2.cost) + with pytest.raises( + AssertionError, + match="summary_method must be a string chosen from 'reflection_with_llm' or 'last_msg' or a callable, or None.", + ): + user_proxy.initiate_chat( + assistant, + message="What functions do you know about in the context of this conversation? End your response with 'TERMINATE'.", + summary_method="llm", + ) + + with pytest.raises( + AssertionError, + match="llm client must be set in either the recipient or sender when summary_method is reflection_with_llm.", + ): + user_proxy.initiate_chat( + recipient=user_proxy, + message="What functions do you know about in the context of this conversation? End your response with 'TERMINATE'.", + summary_method="reflection_with_llm", + ) + if __name__ == "__main__": # test_json_extraction() # test_execute_function() test_update_function() - asyncio.run(test_a_execute_function()) - test_eval_math_responses() + # asyncio.run(test_a_execute_function()) + # test_eval_math_responses() diff --git a/test/coding/test_embedded_ipython_code_executor.py b/test/coding/test_embedded_ipython_code_executor.py index d1669096000..3ecb318f959 100644 --- a/test/coding/test_embedded_ipython_code_executor.py +++ b/test/coding/test_embedded_ipython_code_executor.py @@ -93,7 +93,7 @@ def test_timeout() -> None: @pytest.mark.skipif(skip, reason=skip_reason) def test_silent_pip_install() -> None: - executor = EmbeddedIPythonCodeExecutor() + executor = EmbeddedIPythonCodeExecutor(timeout=600) code_blocks = [CodeBlock(code="!pip install matplotlib numpy", language="python")] code_result = executor.execute_code_blocks(code_blocks) assert code_result.exit_code == 0 and code_result.output.strip() == ""