tests websurfer

python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py

@@ -474,7 +474,7 @@ async def __generate_reply(self, cancellation_token: CancellationToken) -> UserC
         self._last_download = None
         if isinstance(message, str):
             # Answer directly
-            self.inner_messages.append(AgentMessage(content=message, source=self.name))
+            self.inner_messages.append(TextMessage(content=message, source=self.name))
             return message
         elif isinstance(message, list):
             # Take an action
@@ -504,7 +504,7 @@ async def _execute_tool(
                 message=f"{name}( {json.dumps(args)} )",
             )
         )
-        self.inner_messages.append(AgentMessage(content=f"{name}( {json.dumps(args)} )", source=self.name))
+        self.inner_messages.append(TextMessage(content=f"{name}( {json.dumps(args)} )", source=self.name))
 
         if name == "visit_url":
             url = args.get("url")

python/packages/autogen-ext/tests/test_playwright_controller.py

@@ -50,7 +50,7 @@ async def test_playwright_controller_click_id() -> None:
 
         controller = PlaywrightController()
         rects = await controller.get_interactive_rects(page)
-        click_me_id = 0
+        click_me_id = ""
         for rect in rects:
             if rects[rect]["aria_name"] == "Click Me":
                 click_me_id = str(rect)
@@ -68,7 +68,7 @@ async def test_playwright_controller_fill_id() -> None:
         page = await context.new_page()
         await page.set_content(FAKE_HTML)
         rects = await PlaywrightController().get_interactive_rects(page)
-        input_box_id = 0
+        input_box_id = ""
         for rect in rects:
             if rects[rect]["tag_name"] == "input, type=text":
                 input_box_id = str(rect)

python/packages/autogen-ext/tests/test_websurfer_agent.py

@@ -7,26 +7,22 @@
 
 import pytest
 from autogen_agentchat import EVENT_LOGGER_NAME
-from autogen_agentchat.agents import AssistantAgent
-from autogen_agentchat.base import Handoff, TaskResult
 from autogen_agentchat.messages import (
     HandoffMessage,
     MultiModalMessage,
     TextMessage,
     ToolCallMessage,
     ToolCallResultMessage,
 )
-from autogen_core import Image
 from autogen_core.components.tools import FunctionTool
 from autogen_ext.agents.web_surfer import MultimodalWebSurfer
-from autogen_ext.models import OpenAIChatCompletionClient
+from autogen_ext.models.openai import OpenAIChatCompletionClient
 from openai.resources.chat.completions import AsyncCompletions
 from openai.types.chat.chat_completion import ChatCompletion, Choice
 from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
 from openai.types.chat.chat_completion_message import ChatCompletionMessage
 from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall, Function
 from openai.types.completion_usage import CompletionUsage
-from playwright.async_api import async_playwright
 from pydantic import BaseModel
 
 
@@ -69,19 +65,21 @@ async def mock_create(
 
 
 @pytest.mark.asyncio
-async def test_multimodal_websurfer_initialization() -> None:
-    model_client = OpenAIChatCompletionClient(model="gpt-4o-2024-08-06", api_key="")
-    agent = MultimodalWebSurfer(name="TestWebSurfer", model_client=model_client)
-    assert agent.name == "TestWebSurfer"
-    assert agent._model_client == model_client
-
-
-@pytest.mark.asyncio
-async def test_run_with_tools(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_run_websurfer(monkeypatch: pytest.MonkeyPatch) -> None:
     model = "gpt-4o-2024-05-13"
     chat_completions = [
         ChatCompletion(
-            id="id1",
+            id="id2",
+            choices=[
+                Choice(finish_reason="stop", index=0, message=ChatCompletionMessage(content="Hello", role="assistant"))
+            ],
+            created=0,
+            model=model,
+            object="chat.completion",
+            usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=0),
+        ),
+        ChatCompletion(
+            id="id2",
             choices=[
                 Choice(
                     finish_reason="tool_calls",
@@ -93,8 +91,8 @@ async def test_run_with_tools(monkeypatch: pytest.MonkeyPatch) -> None:
                                 id="1",
                                 type="function",
                                 function=Function(
-                                    name="_pass_function",
-                                    arguments=json.dumps({"input": "task"}),
+                                    name="sleep",
+                                    arguments=json.dumps({"reasoning": "sleep is important"}),
                                 ),
                             )
                         ],
@@ -107,33 +105,48 @@ async def test_run_with_tools(monkeypatch: pytest.MonkeyPatch) -> None:
             object="chat.completion",
             usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=0),
         ),
-        ChatCompletion(
-            id="id2",
-            choices=[
-                Choice(finish_reason="stop", index=0, message=ChatCompletionMessage(content="Hello", role="assistant"))
-            ],
-            created=0,
-            model=model,
-            object="chat.completion",
-            usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=0),
-        ),
-        ChatCompletion(
-            id="id2",
-            choices=[
-                Choice(
-                    finish_reason="stop", index=0, message=ChatCompletionMessage(content="TERMINATE", role="assistant")
-                )
-            ],
-            created=0,
-            model=model,
-            object="chat.completion",
-            usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=0),
-        ),
     ]
     mock = _MockChatCompletion(chat_completions)
     monkeypatch.setattr(AsyncCompletions, "create", mock.mock_create)
-    agent = AssistantAgent(
-        "tool_use_agent",
-        model_client=OpenAIChatCompletionClient(model=model, api_key=""),
+    agent = MultimodalWebSurfer(
+        "WebSurfer", model_client=OpenAIChatCompletionClient(model=model, api_key=""), use_ocr=False
+    )
+    # Before lazy init
+    assert agent._name == "WebSurfer"
+    assert agent._playwright is None
+    # After lazy init
+    result = await agent.run(task="task")
+    assert agent._playwright is not None
+    assert agent._page is not None
+    # now check result object
+    assert len(result.messages) == 3
+    # user message
+    assert isinstance(result.messages[0], TextMessage)
+    assert result.messages[0].models_usage is None
+    # inner message
+    assert isinstance(result.messages[1], TextMessage)
+    # final return
+    assert isinstance(result.messages[2], TextMessage)
+    assert result.messages[2].models_usage is not None
+    assert result.messages[2].models_usage.completion_tokens == 5
+    assert result.messages[2].models_usage.prompt_tokens == 10
+    assert result.messages[2].content == "Hello"
+    # check internal web surfer state
+    assert len(agent._chat_history) == 2
+    assert agent._chat_history[0].content == "task"
+    assert agent._chat_history[1].content == "Hello"
+    url_after_no_tool = agent._page.url  # type: ignore
+
+    # run again
+    result = await agent.run(task="task")
+    assert len(result.messages) == 3
+    assert isinstance(result.messages[2], MultiModalMessage)
+    assert (
+        result.messages[2]
+        .content[0]
+        .startswith(
+            "I am waiting a short period of time before taking further action.\n\n Here is a screenshot of the webpage: [Search - Microsoft Bing](https://www.bing.com/)"
+        )
     )
-    await agent.run(task="task")
+    url_after_sleep = agent._page.url  # type: ignore
+    assert url_after_no_tool == url_after_sleep