diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index b3b0d0daeed..00000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,57 +0,0 @@ -### Description - - -### Environment -- AutoGen version: -- Python version: -- Operating System: - -### Steps to Reproduce (for bugs) - - -1. Step 1 -2. Step 2 -3. ... - -### Expected Behavior - - -### Actual Behavior - - -### Screenshots / Logs (if applicable) - - -### Additional Information - - -### Possible Solution (if you have one) - - -### Is this a Bug or Feature Request? - - -### Priority - - -### Difficulty - - -### Any related issues? - - -### Any relevant discussions? - - -### Checklist - -- [ ] I have searched for similar issues and didn't find any duplicates. -- [ ] I have provided a clear and concise description of the issue. -- [ ] I have included the necessary environment details. -- [ ] I have outlined the steps to reproduce the issue. -- [ ] I have included any relevant logs or screenshots. -- [ ] I have indicated whether this is a bug or a feature request. -- [ ] I have set the priority and difficulty levels. - -### Additional Comments - diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 434226b3e88..090fa6cc593 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,53 +1,55 @@ name: Bug Report -description: File a bug report -title: "[Bug]: " +description: Report a bug labels: ["bug"] body: - type: textarea - id: description attributes: - label: Describe the bug - description: A clear and concise description of what the bug is. - placeholder: What went wrong? + label: What happened? + description: Please provide as much information as possible, this helps us address the issue. + validations: + required: true - type: textarea - id: reproduce attributes: - label: Steps to reproduce - description: | - Steps to reproduce the behavior: - - 1. Step 1 - 2. Step 2 - 3. ... - 4. See error - placeholder: How can we replicate the issue? + label: What did you expect to happen? + validations: + required: true - type: textarea - id: modelused attributes: - label: Model Used - description: A description of the model that was used when the error was encountered + label: How can we reproduce it (as minimally and precisely as possible)? + description: Please provide steps to reproduce. Provide code that can be run if possible. + validations: + required: true + - type: input + attributes: + label: AutoGen version + description: What version or commit of the library was used + validations: + required: true + - type: dropdown + attributes: + label: Which package was this bug in + options: + - Core + - AgentChat + - Extensions + - AutoGen Studio + - Magentic One + - AutoGen Bench + - Other + validations: + required: true + - type: input + attributes: + label: Model used + description: If a model was used, please describe it here, indicating whether it is a local model or a cloud-hosted model placeholder: gpt-4, mistral-7B etc - - type: textarea - id: expected_behavior + - type: input attributes: - label: Expected Behavior - description: A clear and concise description of what you expected to happen. - placeholder: What should have happened? - - type: textarea - id: screenshots + label: Python version + - type: input attributes: - label: Screenshots and logs - description: If applicable, add screenshots and logs to help explain your problem. - placeholder: Add screenshots here + label: Operating system - type: textarea - id: additional_information - attributes: - label: Additional Information - description: | - - AutoGen Version: - - Operating System: - - Python Version: - - Related Issues: - - Any other relevant information. - placeholder: Any additional details + attributes: + label: Any additional info you think would be helpful for fixing this bug diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 0086358db1e..76afcbcc5f8 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1 +1,5 @@ blank_issues_enabled: true +contact_links: + - name: Questions or general help 💬 + url: https://github.com/microsoft/autogen/discussions + about: Please ask and answer questions here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index e8a63df7a6e..57f360761a7 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,26 +1,18 @@ name: Feature Request -description: File a feature request +description: Request a new feature or enhancement labels: ["enhancement"] -title: "[Feature Request]: " body: - type: textarea - id: problem_description attributes: - label: Is your feature request related to a problem? Please describe. - description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - placeholder: What problem are you trying to solve? + label: What feature would you like to be added? + description: Please describe the desired feature. Be descriptive, provide examples and if possible, provide a proposed solution. + validations: + required: true - type: textarea - id: solution_description attributes: - label: Describe the solution you'd like - description: A clear and concise description of what you want to happen. - placeholder: How do you envision the solution? - - - type: textarea - id: additional_context - attributes: - label: Additional context - description: Add any other context or screenshots about the feature request here. - placeholder: Any additional information + label: Why is this needed? + description: Why is it important that this feature is implemented? What problem or need does it solve? + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/general_issue.yml b/.github/ISSUE_TEMPLATE/general_issue.yml deleted file mode 100644 index b585f4642f4..00000000000 --- a/.github/ISSUE_TEMPLATE/general_issue.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: General Issue -description: File a general issue -title: "[Issue]: " -labels: [] - -body: - - type: textarea - id: description - attributes: - label: Describe the issue - description: A clear and concise description of what the issue is. - placeholder: What went wrong? - - type: textarea - id: reproduce - attributes: - label: Steps to reproduce - description: | - Steps to reproduce the behavior: - - 1. Step 1 - 2. Step 2 - 3. ... - 4. See error - placeholder: How can we replicate the issue? - - type: textarea - id: screenshots - attributes: - label: Screenshots and logs - description: If applicable, add screenshots and logs to help explain your problem. - placeholder: Add screenshots here - - type: textarea - id: additional_information - attributes: - label: Additional Information - description: | - - AutoGen Version: - - Operating System: - - Python Version: - - Related Issues: - - Any other relevant information. - placeholder: Any additional details diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a92044f15b7..0107e7314fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,15 +5,13 @@ name: Build on: push: - branches: ["main"] + branches: ["0.2"] pull_request: - branches: ["main"] - merge_group: - types: [checks_requested] + branches: ["0.2"] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' }} permissions: {} jobs: paths-filter: diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index 7e8fb003317..d084e44bd60 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -5,7 +5,7 @@ name: OpenAI4ContribTests on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "test/agentchat/contrib/**" diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index f14a7f09ebe..7d779bc5fae 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -5,7 +5,7 @@ name: ContribTests on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "test/agentchat/contrib/**" @@ -16,7 +16,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' }} permissions: {} # actions: read diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index 2f2ba4d473f..e576ca1b4db 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -2,20 +2,18 @@ name: docs on: pull_request: - branches: [main] - path: + branches: ["0.2"] + paths: - "autogen/*" - "website/*" - ".github/workflows/deploy-website.yml" push: - branches: [main] - path: + branches: ["0.2"] + paths: - "autogen/*" - "website/*" - ".github/workflows/deploy-website.yml" workflow_dispatch: - merge_group: - types: [checks_requested] permissions: id-token: write pages: write @@ -67,57 +65,3 @@ jobs: npm i --legacy-peer-deps npm run build fi - gh-release: - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - defaults: - run: - working-directory: website - steps: - - uses: actions/checkout@v4 - with: - lfs: true - - uses: actions/setup-node@v4 - with: - node-version: 18.x - - name: setup python - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - name: pydoc-markdown install - run: | - python -m pip install --upgrade pip - pip install pydoc-markdown pyyaml termcolor - # Pin databind packages as version 4.5.0 is not compatible with pydoc-markdown. - pip install databind.core==4.4.2 databind.json==4.4.2 - - name: pydoc-markdown run - run: | - pydoc-markdown - - name: quarto install - working-directory: ${{ runner.temp }} - run: | - wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.tar.gz - tar -xzf quarto-1.5.23-linux-amd64.tar.gz - echo "$(pwd)/quarto-1.5.23/bin/" >> $GITHUB_PATH - - name: Process notebooks - run: | - python process_notebooks.py render - - name: Build website - run: | - if [ -e yarn.lock ]; then - yarn install --frozen-lockfile --ignore-engines - yarn build - elif [ -e package-lock.json ]; then - npm ci - npm run build - else - npm i --legacy-peer-deps - npm run build - fi - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: "website/build" - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/dotnet-build.yml b/.github/workflows/dotnet-build.yml index 6aac54d3818..b333d9065d6 100644 --- a/.github/workflows/dotnet-build.yml +++ b/.github/workflows/dotnet-build.yml @@ -6,15 +6,13 @@ name: dotnet-ci on: workflow_dispatch: pull_request: - branches: [ "main" ] + branches: [ "0.2" ] push: - branches: [ "main" ] - merge_group: - types: [checks_requested] + branches: [ "0.2" ] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' || github.ref != 'refs/heads/dotnet' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' || github.ref != 'refs/heads/dotnet' }} permissions: contents: read @@ -122,7 +120,7 @@ jobs: defaults: run: working-directory: dotnet - if: success() && (github.ref == 'refs/heads/main') + if: success() && (github.ref == 'refs/heads/0.2') needs: aot-test steps: - uses: actions/checkout@v4 @@ -228,4 +226,4 @@ jobs: env: MYGET_TOKEN: ${{ secrets.MYGET_TOKEN }} continue-on-error: true - + diff --git a/.github/workflows/issue-needs-triage.yml b/.github/workflows/issue-needs-triage.yml new file mode 100644 index 00000000000..59cb3479c80 --- /dev/null +++ b/.github/workflows/issue-needs-triage.yml @@ -0,0 +1,18 @@ +name: Label issues with needs-triage +on: + issues: + types: + - reopened + - opened +jobs: + label_issues: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - run: gh issue edit "$NUMBER" --add-label "$LABELS" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + NUMBER: ${{ github.event.issue.number }} + LABELS: needs-triage diff --git a/.github/workflows/lfs-check.yml b/.github/workflows/lfs-check.yml index 4baae925de3..dc5e1678be2 100644 --- a/.github/workflows/lfs-check.yml +++ b/.github/workflows/lfs-check.yml @@ -1,6 +1,7 @@ name: "Git LFS Check" - -on: pull_request +on: + pull_request: + branches: ["0.2"] permissions: {} jobs: lfs-check: diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml index a9ab8e9e0c5..e29f9d0f1ab 100644 --- a/.github/workflows/openai.yml +++ b/.github/workflows/openai.yml @@ -5,7 +5,7 @@ name: OpenAI on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "test/**" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 8404de61154..7ec5a4eb104 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -3,8 +3,7 @@ name: Code formatting # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows on: # Trigger the workflow on pull request or merge pull_request: - merge_group: - types: [checks_requested] + branches: ["0.2"] defaults: run: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b2274e64198..e34678ab003 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,14 +5,10 @@ name: python-package on: - release: - types: [published] - workflow_dispatch: + push: + tags: + - "0.2.*" permissions: {} - # actions: read - # checks: read - # contents: read - # deployments: read jobs: deploy: strategy: @@ -28,26 +24,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - # - name: Cache conda - # uses: actions/cache@v4 - # with: - # path: ~/conda_pkgs_dir - # key: conda-${{ matrix.os }}-python-${{ matrix.python-version }}-${{ hashFiles('environment.yml') }} - # - name: Setup Miniconda - # uses: conda-incubator/setup-miniconda@v2 - # with: - # auto-update-conda: true - # auto-activate-base: false - # activate-environment: hcrystalball - # python-version: ${{ matrix.python-version }} - # use-only-tar-bz2: true - - name: Install from source - # This is required for the pre-commit tests - shell: pwsh - run: pip install . - # - name: Conda list - # shell: pwsh - # run: conda list - name: Build shell: pwsh run: | diff --git a/.github/workflows/samples-tools-tests.yml b/.github/workflows/samples-tools-tests.yml index e774e5cb0b1..9452f0e377e 100644 --- a/.github/workflows/samples-tools-tests.yml +++ b/.github/workflows/samples-tools-tests.yml @@ -5,7 +5,7 @@ name: SamplesToolsTests on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "samples/tools/**" @@ -14,7 +14,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' }} permissions: {} jobs: SamplesToolsFineTuningTests: diff --git a/.github/workflows/type-check.yml b/.github/workflows/type-check.yml index c66fb6ad7b1..3afd32ad886 100644 --- a/.github/workflows/type-check.yml +++ b/.github/workflows/type-check.yml @@ -2,8 +2,8 @@ name: Type check # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows on: # Trigger the workflow on pull request or merge pull_request: - merge_group: - types: [checks_requested] + branches: ["0.2"] + defaults: run: shell: bash diff --git a/README.md b/README.md index e92083f0df3..0599300c539 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ AutoGen is an open-source programming framework for building AI agents and facil > -_Maintainers (Sept 6th, 2024)_ -![AutoGen Overview](https://github.com/microsoft/autogen/blob/main/website/static/img/autogen_agentchat.png) +![AutoGen Overview](https://github.com/microsoft/autogen/blob/0.2/website/static/img/autogen_agentchat.png) - AutoGen enables building next-gen LLM applications based on [multi-agent conversations](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat) with minimal effort. It simplifies the orchestration, automation, and optimization of a complex LLM workflow. It maximizes the performance of LLM models and overcomes their weaknesses. - It supports [diverse conversation patterns](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat#supporting-diverse-conversation-patterns) for complex workflows. With customizable and conversable agents, developers can use AutoGen to build a wide range of conversation patterns concerning conversation autonomy, @@ -171,7 +171,7 @@ Features of this use case include: - **Customization**: AutoGen agents can be customized to meet the specific needs of an application. This includes the ability to choose the LLMs to use, the types of human input to allow, and the tools to employ. - **Human participation**: AutoGen seamlessly allows human participation. This means that humans can provide input and feedback to the agents as needed. -For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py), +For [example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py), ```python from autogen import AssistantAgent, UserProxyAgent, config_list_from_json @@ -194,9 +194,9 @@ python test/twoagent.py After the repo is cloned. The figure below shows an example conversation flow with AutoGen. -![Agent Chat Example](https://github.com/microsoft/autogen/blob/main/website/static/img/chat_example.png) +![Agent Chat Example](https://github.com/microsoft/autogen/blob/0.2/website/static/img/chat_example.png) -Alternatively, the [sample code](https://github.com/microsoft/autogen/blob/main/samples/simple_chat.py) here allows a user to chat with an AutoGen agent in ChatGPT style. +Alternatively, the [sample code](https://github.com/microsoft/autogen/blob/0.2/samples/simple_chat.py) here allows a user to chat with an AutoGen agent in ChatGPT style. Please find more [code examples](https://microsoft.github.io/autogen/docs/Examples#automated-multi-agent-chat) for this feature.

@@ -240,7 +240,7 @@ You can find detailed documentation about AutoGen [here](https://microsoft.githu In addition, you can find: -- [Research](https://microsoft.github.io/autogen/docs/Research), [blogposts](https://microsoft.github.io/autogen/blog) around AutoGen, and [Transparency FAQs](https://github.com/microsoft/autogen/blob/main/TRANSPARENCY_FAQS.md) +- [Research](https://microsoft.github.io/autogen/docs/Research), [blogposts](https://microsoft.github.io/autogen/blog) around AutoGen, and [Transparency FAQs](https://github.com/microsoft/autogen/blob/0.2/TRANSPARENCY_FAQS.md) - [Contributing guide](https://microsoft.github.io/autogen/docs/Contribute) diff --git a/autogen/agentchat/groupchat.py b/autogen/agentchat/groupchat.py index c6355a13b94..b0f0b9520aa 100644 --- a/autogen/agentchat/groupchat.py +++ b/autogen/agentchat/groupchat.py @@ -12,9 +12,9 @@ from ..formatting_utils import colored from ..graph_utils import check_graph_validity, invert_disallowed_to_allowed from ..io.base import IOStream +from ..oai.client import ModelClient from ..runtime_logging import log_new_agent, logging_enabled from .agent import Agent -from .chat import ChatResult from .conversable_agent import ConversableAgent try: @@ -105,6 +105,8 @@ def custom_speaker_selection_func( "clear history" phrase in user prompt. This is experimental feature. See description of GroupChatManager.clear_agents_history function for more info. - send_introductions: send a round of introductions at the start of the group chat, so agents know who they can speak to (default: False) + - select_speaker_auto_model_client_cls: Custom model client class for the internal speaker select agent used during 'auto' speaker selection (optional) + - select_speaker_auto_llm_config: LLM config for the internal speaker select agent used during 'auto' speaker selection (optional) - role_for_select_speaker_messages: sets the role name for speaker selection when in 'auto' mode, typically 'user' or 'system'. (default: 'system') """ @@ -142,6 +144,8 @@ def custom_speaker_selection_func( Respond with ONLY the name of the speaker and DO NOT provide a reason.""" select_speaker_transform_messages: Optional[Any] = None select_speaker_auto_verbose: Optional[bool] = False + select_speaker_auto_model_client_cls: Optional[Union[ModelClient, List[ModelClient]]] = None + select_speaker_auto_llm_config: Optional[Union[Dict, Literal[False]]] = None role_for_select_speaker_messages: Optional[str] = "system" _VALID_SPEAKER_SELECTION_METHODS = ["auto", "manual", "random", "round_robin"] @@ -591,6 +595,79 @@ def _finalize_speaker(self, last_speaker: Agent, final: bool, name: str, agents: agent = self.agent_by_name(name) return agent if agent else self.next_agent(last_speaker, agents) + def _register_client_from_config(self, agent: Agent, config: Dict): + model_client_cls_to_match = config.get("model_client_cls") + if model_client_cls_to_match: + if not self.select_speaker_auto_model_client_cls: + raise ValueError( + "A custom model was detected in the config but no 'model_client_cls' " + "was supplied for registration in GroupChat." + ) + + if isinstance(self.select_speaker_auto_model_client_cls, list): + # Register the first custom model client class matching the name specified in the config + matching_model_cls = [ + client_cls + for client_cls in self.select_speaker_auto_model_client_cls + if client_cls.__name__ == model_client_cls_to_match + ] + if len(set(matching_model_cls)) > 1: + raise RuntimeError( + f"More than one unique 'model_client_cls' with __name__ '{model_client_cls_to_match}'." + ) + if not matching_model_cls: + raise ValueError( + "No model's __name__ matches the model client class " + f"'{model_client_cls_to_match}' specified in select_speaker_auto_llm_config." + ) + select_speaker_auto_model_client_cls = matching_model_cls[0] + else: + # Register the only custom model client + select_speaker_auto_model_client_cls = self.select_speaker_auto_model_client_cls + + agent.register_model_client(select_speaker_auto_model_client_cls) + + def _register_custom_model_clients(self, agent: ConversableAgent): + if not self.select_speaker_auto_llm_config: + return + + config_format_is_list = "config_list" in self.select_speaker_auto_llm_config.keys() + if config_format_is_list: + for config in self.select_speaker_auto_llm_config["config_list"]: + self._register_client_from_config(agent, config) + elif not config_format_is_list: + self._register_client_from_config(agent, self.select_speaker_auto_llm_config) + + def _create_internal_agents( + self, agents, max_attempts, messages, validate_speaker_name, selector: Optional[ConversableAgent] = None + ): + checking_agent = ConversableAgent("checking_agent", default_auto_reply=max_attempts) + + # Register the speaker validation function with the checking agent + checking_agent.register_reply( + [ConversableAgent, None], + reply_func=validate_speaker_name, # Validate each response + remove_other_reply_funcs=True, + ) + + # Override the selector's config if one was passed as a parameter to this class + speaker_selection_llm_config = self.select_speaker_auto_llm_config or selector.llm_config + + # Agent for selecting a single agent name from the response + speaker_selection_agent = ConversableAgent( + "speaker_selection_agent", + system_message=self.select_speaker_msg(agents), + chat_messages={checking_agent: messages}, + llm_config=speaker_selection_llm_config, + human_input_mode="NEVER", + # Suppresses some extra terminal outputs, outputs will be handled by select_speaker_auto_verbose + ) + + # Register any custom model passed in select_speaker_auto_llm_config with the speaker_selection_agent + self._register_custom_model_clients(speaker_selection_agent) + + return checking_agent, speaker_selection_agent + def _auto_select_speaker( self, last_speaker: Agent, @@ -644,28 +721,8 @@ def validate_speaker_name(recipient, messages, sender, config) -> Tuple[bool, Un # Two-agent chat for speaker selection # Agent for checking the response from the speaker_select_agent - checking_agent = ConversableAgent("checking_agent", default_auto_reply=max_attempts) - - # Register the speaker validation function with the checking agent - checking_agent.register_reply( - [ConversableAgent, None], - reply_func=validate_speaker_name, # Validate each response - remove_other_reply_funcs=True, - ) - - # NOTE: Do we have a speaker prompt (select_speaker_prompt_template is not None)? If we don't, we need to feed in the last message to start the nested chat - - # Agent for selecting a single agent name from the response - speaker_selection_agent = ConversableAgent( - "speaker_selection_agent", - system_message=self.select_speaker_msg(agents), - chat_messages=( - {checking_agent: messages} - if self.select_speaker_prompt_template is not None - else {checking_agent: messages[:-1]} - ), - llm_config=selector.llm_config, - human_input_mode="NEVER", # Suppresses some extra terminal outputs, outputs will be handled by select_speaker_auto_verbose + checking_agent, speaker_selection_agent = self._create_internal_agents( + agents, max_attempts, messages, validate_speaker_name, selector ) # Create the starting message @@ -747,24 +804,8 @@ def validate_speaker_name(recipient, messages, sender, config) -> Tuple[bool, Un # Two-agent chat for speaker selection # Agent for checking the response from the speaker_select_agent - checking_agent = ConversableAgent("checking_agent", default_auto_reply=max_attempts) - - # Register the speaker validation function with the checking agent - checking_agent.register_reply( - [ConversableAgent, None], - reply_func=validate_speaker_name, # Validate each response - remove_other_reply_funcs=True, - ) - - # NOTE: Do we have a speaker prompt (select_speaker_prompt_template is not None)? If we don't, we need to feed in the last message to start the nested chat - - # Agent for selecting a single agent name from the response - speaker_selection_agent = ConversableAgent( - "speaker_selection_agent", - system_message=self.select_speaker_msg(agents), - chat_messages={checking_agent: messages}, - llm_config=selector.llm_config, - human_input_mode="NEVER", # Suppresses some extra terminal outputs, outputs will be handled by select_speaker_auto_verbose + checking_agent, speaker_selection_agent = self._create_internal_agents( + agents, max_attempts, messages, validate_speaker_name, selector ) # Create the starting message diff --git a/autogen/oai/gemini.py b/autogen/oai/gemini.py index 33790c9851c..d44e41112ac 100644 --- a/autogen/oai/gemini.py +++ b/autogen/oai/gemini.py @@ -32,6 +32,8 @@ from __future__ import annotations import base64 +import copy +import json import logging import os import random @@ -39,24 +41,39 @@ import time import warnings from io import BytesIO -from typing import Any, Dict, List, Mapping, Union +from typing import Any, Dict, List, Union import google.generativeai as genai import requests import vertexai -from google.ai.generativelanguage import Content, Part +from google.ai.generativelanguage import Content, FunctionCall, FunctionDeclaration, FunctionResponse, Part, Tool from google.api_core.exceptions import InternalServerError from google.auth.credentials import Credentials -from openai.types.chat import ChatCompletion +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.chat.chat_completion_message_tool_call import Function from openai.types.completion_usage import CompletionUsage from PIL import Image -from vertexai.generative_models import Content as VertexAIContent +from vertexai.generative_models import ( + Content as VertexAIContent, +) +from vertexai.generative_models import ( + FunctionDeclaration as VertexAIFunctionDeclaration, +) +from vertexai.generative_models import ( + GenerationConfig as VertexAIGenerationConfig, +) from vertexai.generative_models import GenerativeModel from vertexai.generative_models import HarmBlockThreshold as VertexAIHarmBlockThreshold from vertexai.generative_models import HarmCategory as VertexAIHarmCategory from vertexai.generative_models import Part as VertexAIPart from vertexai.generative_models import SafetySetting as VertexAISafetySetting +from vertexai.generative_models import ( + Tool as VertexAITool, +) +from vertexai.generative_models import ( + ToolConfig as VertexAIToolConfig, +) logger = logging.getLogger(__name__) @@ -107,7 +124,7 @@ def __init__(self, **kwargs): Args: api_key (str): The API key for using Gemini. - credentials (google.auth.credentials.Credentials): credentials to be used for authentication with vertexai. + credentials (google.auth.credentials.Credentials): credentials to be used for authentication with vertexai. google_application_credentials (str): Path to the JSON service account key file of the service account. Alternatively, the GOOGLE_APPLICATION_CREDENTIALS environment variable can also be set instead of using this argument. @@ -171,6 +188,8 @@ def create(self, params: Dict) -> ChatCompletion: params.get("api_type", "google") # not used messages = params.get("messages", []) + tools = params.get("tools", []) + tool_config = params.get("tool_config", {}) stream = params.get("stream", False) n_response = params.get("n", 1) system_instruction = params.get("system_instruction", None) @@ -183,6 +202,7 @@ def create(self, params: Dict) -> ChatCompletion: } if self.use_vertexai: safety_settings = GeminiClient._to_vertexai_safety_settings(params.get("safety_settings", {})) + tool_config = GeminiClient._to_vertexai_tool_config(tool_config, tools) else: safety_settings = params.get("safety_settings", {}) @@ -198,12 +218,15 @@ def create(self, params: Dict) -> ChatCompletion: if "vision" not in model_name: # A. create and call the chat model. gemini_messages = self._oai_messages_to_gemini_messages(messages) + gemini_tools = self._oai_tools_to_gemini_tools(tools) if self.use_vertexai: model = GenerativeModel( model_name, generation_config=generation_config, safety_settings=safety_settings, system_instruction=system_instruction, + tools=gemini_tools, + tool_config=tool_config, ) chat = model.start_chat(history=gemini_messages[:-1], response_validation=response_validation) else: @@ -213,12 +236,13 @@ def create(self, params: Dict) -> ChatCompletion: generation_config=generation_config, safety_settings=safety_settings, system_instruction=system_instruction, + tools=gemini_tools, ) genai.configure(api_key=self.api_key) chat = model.start_chat(history=gemini_messages[:-1]) max_retries = 5 for attempt in range(max_retries): - ans = None + ans: Union[Content, VertexAIContent] = None try: response = chat.send_message( gemini_messages[-1].parts, stream=stream, safety_settings=safety_settings @@ -234,7 +258,7 @@ def create(self, params: Dict) -> ChatCompletion: raise RuntimeError(f"Google GenAI exception occurred while calling Gemini API: {e}") else: # `ans = response.text` is unstable. Use the following code instead. - ans: str = chat.history[-1].parts[0].text + ans: Union[Content, VertexAIContent] = chat.history[-1] break if ans is None: @@ -262,7 +286,7 @@ def create(self, params: Dict) -> ChatCompletion: # Gemini's vision model does not support chat history yet # chat = model.start_chat(history=gemini_messages[:-1]) # response = chat.send_message(gemini_messages[-1].parts) - user_message = self._oai_content_to_gemini_content(messages[-1]["content"]) + user_message = self._oai_content_to_gemini_content(messages[-1]) if len(messages) > 2: warnings.warn( "Warning: Gemini's vision model does not support chat history yet.", @@ -273,16 +297,14 @@ def create(self, params: Dict) -> ChatCompletion: response = model.generate_content(user_message, stream=stream) # ans = response.text if self.use_vertexai: - ans: str = response.candidates[0].content.parts[0].text + ans: VertexAIContent = response.candidates[0].content else: - ans: str = response._result.candidates[0].content.parts[0].text + ans: Content = response._result.candidates[0].content prompt_tokens = model.count_tokens(user_message).total_tokens - completion_tokens = model.count_tokens(ans).total_tokens + completion_tokens = model.count_tokens(ans.parts[0].text).total_tokens - # 3. convert output - message = ChatCompletionMessage(role="assistant", content=ans, function_call=None, tool_calls=None) - choices = [Choice(finish_reason="stop", index=0, message=message)] + choices = self._gemini_content_to_oai_choices(ans) response_oai = ChatCompletion( id=str(random.randint(0, 1000)), @@ -295,31 +317,87 @@ def create(self, params: Dict) -> ChatCompletion: completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ), - cost=calculate_gemini_cost(prompt_tokens, completion_tokens, model_name), + cost=self._calculate_gemini_cost(prompt_tokens, completion_tokens, model_name), ) return response_oai - def _oai_content_to_gemini_content(self, content: Union[str, List]) -> List: + # If str is not a json string return str as is + def _to_json(self, str) -> dict: + try: + return json.loads(str) + except ValueError: + return str + + def _oai_content_to_gemini_content(self, message: Dict[str, Any]) -> List: """Convert content from OAI format to Gemini format""" rst = [] - if isinstance(content, str): - if content == "": - content = "empty" # Empty content is not allowed. + if isinstance(message["content"], str): + if message["content"] == "": + message["content"] = "empty" # Empty content is not allowed. + if self.use_vertexai: + rst.append(VertexAIPart.from_text(message["content"])) + else: + rst.append(Part(text=message["content"])) + return rst + + if "tool_calls" in message: + if self.use_vertexai: + for tool_call in message["tool_calls"]: + rst.append( + VertexAIPart.from_dict( + { + "functionCall": { + "name": tool_call["function"]["name"], + "args": json.loads(tool_call["function"]["arguments"]), + } + } + ) + ) + else: + for tool_call in message["tool_calls"]: + rst.append( + Part( + function_call=FunctionCall( + name=tool_call["function"]["name"], + args=json.loads(tool_call["function"]["arguments"]), + ) + ) + ) + return rst + + if message["role"] == "tool": + if self.use_vertexai: + rst.append( + VertexAIPart.from_function_response( + name=message["name"], response={"result": self._to_json(message["content"])} + ) + ) + else: + rst.append( + Part( + function_response=FunctionResponse( + name=message["name"], response={"result": self._to_json(message["content"])} + ) + ) + ) + return rst + + if isinstance(message["content"], str): if self.use_vertexai: - rst.append(VertexAIPart.from_text(content)) + rst.append(VertexAIPart.from_text(message["content"])) else: - rst.append(Part(text=content)) + rst.append(Part(text=message["content"])) return rst - assert isinstance(content, list) + assert isinstance(message["content"], list) - for msg in content: + for msg in message["content"]: if isinstance(msg, dict): assert "type" in msg, f"Missing 'type' field in message: {msg}" if msg["type"] == "text": if self.use_vertexai: - rst.append(VertexAIPart.from_text(text=msg["text"])) + rst.append(VertexAIPart.from_text(msg["text"])) else: rst.append(Part(text=msg["text"])) elif msg["type"] == "image_url": @@ -340,34 +418,32 @@ def _oai_content_to_gemini_content(self, content: Union[str, List]) -> List: raise ValueError(f"Unsupported message type: {type(msg)}") return rst - def _concat_parts(self, parts: List[Part]) -> List: - """Concatenate parts with the same type. - If two adjacent parts both have the "text" attribute, then it will be joined into one part. - """ - if not parts: - return [] - - concatenated_parts = [] - previous_part = parts[0] - - for current_part in parts[1:]: - if previous_part.text != "": - if self.use_vertexai: - previous_part = VertexAIPart.from_text(previous_part.text + current_part.text) - else: - previous_part.text += current_part.text - else: - concatenated_parts.append(previous_part) - previous_part = current_part - - if previous_part.text == "": - if self.use_vertexai: - previous_part = VertexAIPart.from_text("empty") - else: - previous_part.text = "empty" # Empty content is not allowed. - concatenated_parts.append(previous_part) + def _calculate_gemini_cost(self, input_tokens: int, output_tokens: int, model_name: str) -> float: + if "1.5-pro" in model_name: + if (input_tokens + output_tokens) <= 128000: + # "gemini-1.5-pro" + # When total tokens is less than 128K cost is $3.5 per million input tokens and $10.5 per million output tokens + return 3.5 * input_tokens / 1e6 + 10.5 * output_tokens / 1e6 + # "gemini-1.5-pro" + # Cost is $7 per million input tokens and $21 per million output tokens + return 7.0 * input_tokens / 1e6 + 21.0 * output_tokens / 1e6 + + if "1.5-flash" in model_name: + if (input_tokens + output_tokens) <= 128000: + # "gemini-1.5-flash" + # Cost is $0.35 per million input tokens and $1.05 per million output tokens + return 0.35 * input_tokens / 1e6 + 1.05 * output_tokens / 1e6 + # "gemini-1.5-flash" + # When total tokens is less than 128K cost is $0.70 per million input tokens and $2.10 per million output tokens + return 0.70 * input_tokens / 1e6 + 2.10 * output_tokens / 1e6 + + if "gemini-pro" not in model_name and "gemini-1.0-pro" not in model_name: + warnings.warn( + f"Cost calculation is not implemented for model {model_name}. Using Gemini-1.0-Pro.", UserWarning + ) - return concatenated_parts + # Cost is $0.5 per million input tokens and $1.5 per million output tokens + return 0.5 * input_tokens / 1e6 + 1.5 * output_tokens / 1e6 def _oai_messages_to_gemini_messages(self, messages: list[Dict[str, Any]]) -> list[dict[str, Any]]: """Convert messages from OAI format to Gemini format. @@ -376,38 +452,154 @@ def _oai_messages_to_gemini_messages(self, messages: list[Dict[str, Any]]) -> li """ prev_role = None rst = [] - curr_parts = [] + + def append_parts(parts, role): + if self.use_vertexai: + rst.append(VertexAIContent(parts=parts, role=role)) + else: + rst.append(Content(parts=parts, role=role)) + + def append_text_to_last(text): + if self.use_vertexai: + rst[-1] = VertexAIContent(parts=[*rst[-1].parts, VertexAIPart.from_text(text)], role=rst[-1].role) + else: + rst[-1] = Content(parts=[*rst[-1].parts, Part(text=text)], role=rst[-1].role) + + def is_function_call(parts): + return self.use_vertexai and parts[0].function_call or not self.use_vertexai and "function_call" in parts[0] + for i, message in enumerate(messages): - parts = self._oai_content_to_gemini_content(message["content"]) + + # Since the tool call message does not have the "name" field, we need to find the corresponding tool message. + if message["role"] == "tool": + message["name"] = [ + m["tool_calls"][i]["function"]["name"] + for m in messages + if "tool_calls" in m + for i, tc in enumerate(m["tool_calls"]) + if tc["id"] == message["tool_call_id"] + ][0] + + parts = self._oai_content_to_gemini_content(message) role = "user" if message["role"] in ["user", "system"] else "model" - if (prev_role is None) or (role == prev_role): - curr_parts += parts - elif role != prev_role: - if self.use_vertexai: - rst.append(VertexAIContent(parts=curr_parts, role=prev_role)) - else: - rst.append(Content(parts=curr_parts, role=prev_role)) - curr_parts = parts - prev_role = role - # handle the last message - if self.use_vertexai: - rst.append(VertexAIContent(parts=curr_parts, role=role)) - else: - rst.append(Content(parts=curr_parts, role=role)) + # In Gemini if the current message is a function call then previous message should not be a model message. + if is_function_call(parts): + # If the previous message is a model message then add a dummy "continue" user message before the function call + if prev_role == "model": + append_parts(self._oai_content_to_gemini_content({"content": "continue"}), "user") + append_parts(parts, role) + # In Gemini if the current message is a function response then next message should be a model message. + elif role == "function": + append_parts(parts, "function") + # If the next message is not a model message then add a dummy "continue" model message after the function response + if len(messages) > (i + 1) and messages[i + 1]["role"] in ["user", "system"]: + append_parts(self._oai_content_to_gemini_content({"content": "continue"}), "model") + # If the role is the same as the previous role and both are text messages then concatenate the text + elif role == prev_role: + append_text_to_last(parts[0].text) + # If this is first message or the role is different from the previous role then append the parts + else: + # If the previous text message is empty then update the text to "empty" as Gemini does not support empty messages + if ( + (len(rst) > 0) + and hasattr(rst[-1].parts[0], "_raw_part") + and hasattr(rst[-1].parts[0]._raw_part, "text") + and (rst[-1].parts[0]._raw_part.text == "") + ): + append_text_to_last("empty") + append_parts(parts, role) + + prev_role = role # The Gemini is restrict on order of roles, such that # 1. The messages should be interleaved between user and model. # 2. The last message must be from the user role. # We add a dummy message "continue" if the last role is not the user. - if rst[-1].role != "user": + if rst[-1].role != "user" and rst[-1].role != "function": if self.use_vertexai: - rst.append(VertexAIContent(parts=self._oai_content_to_gemini_content("continue"), role="user")) + rst.append( + VertexAIContent(parts=self._oai_content_to_gemini_content({"content": "continue"}), role="user") + ) else: - rst.append(Content(parts=self._oai_content_to_gemini_content("continue"), role="user")) - + rst.append(Content(parts=self._oai_content_to_gemini_content({"content": "continue"}), role="user")) return rst + def _oai_tools_to_gemini_tools(self, tools: List[Dict[str, Any]]) -> List[Tool]: + """Convert tools from OAI format to Gemini format.""" + if len(tools) == 0: + return None + function_declarations = [] + for tool in tools: + if self.use_vertexai: + function_declaration = VertexAIFunctionDeclaration( + name=tool["function"]["name"], + description=tool["function"]["description"], + parameters=tool["function"]["parameters"], + ) + else: + function_declaration = FunctionDeclaration( + name=tool["function"]["name"], + description=tool["function"]["description"], + parameters=self._oai_function_parameters_to_gemini_function_parameters( + copy.deepcopy(tool["function"]["parameters"]) + ), + ) + function_declarations.append(function_declaration) + if self.use_vertexai: + return [VertexAITool(function_declarations=function_declarations)] + else: + return [Tool(function_declarations=function_declarations)] + + def _oai_function_parameters_to_gemini_function_parameters( + self, function_definition: dict[str, any] + ) -> dict[str, any]: + """ + Convert OpenAPI function definition parameters to Gemini function parameters definition. + The type key is renamed to type_ and the value is capitalized. + """ + assert "anyOf" not in function_definition, "Union types are not supported for function parameter in Gemini." + # Delete the default key as it is not supported in Gemini + if "default" in function_definition: + del function_definition["default"] + + function_definition["type_"] = function_definition["type"].upper() + del function_definition["type"] + if "properties" in function_definition: + for key in function_definition["properties"]: + function_definition["properties"][key] = self._oai_function_parameters_to_gemini_function_parameters( + function_definition["properties"][key] + ) + if "items" in function_definition: + function_definition["items"] = self._oai_function_parameters_to_gemini_function_parameters( + function_definition["items"] + ) + return function_definition + + def _gemini_content_to_oai_choices(self, response: Union[Content, VertexAIContent]) -> List[Choice]: + """Convert response from Gemini format to OAI format.""" + text = None + tool_calls = [] + for part in response.parts: + if part.function_call: + if self.use_vertexai: + arguments = VertexAIPart.to_dict(part)["function_call"]["args"] + else: + arguments = Part.to_dict(part)["function_call"]["args"] + tool_calls.append( + ChatCompletionMessageToolCall( + id=str(random.randint(0, 1000)), + type="function", + function=Function(name=part.function_call.name, arguments=json.dumps(arguments)), + ) + ) + elif part.text: + text = part.text + message = ChatCompletionMessage( + role="assistant", content=text, function_call=None, tool_calls=tool_calls if len(tool_calls) > 0 else None + ) + return [Choice(finish_reason="tool_calls" if tool_calls else "stop", index=0, message=message)] + @staticmethod def _to_vertexai_safety_settings(safety_settings): """Convert safety settings to VertexAI format if needed, @@ -437,6 +629,49 @@ def _to_vertexai_safety_settings(safety_settings): else: return safety_settings + @staticmethod + def _to_vertexai_tool_config(tool_config, tools): + """Convert tool config to VertexAI format, + like when specifying them in the OAI_CONFIG_LIST + """ + if ( + isinstance(tool_config, dict) + and (len(tool_config) > 0) + and all([isinstance(tool_config[tool_config_entry], dict) for tool_config_entry in tool_config]) + ): + if ( + tool_config["function_calling_config"]["mode"] + not in VertexAIToolConfig.FunctionCallingConfig.Mode.__members__ + ): + invalid_mode = tool_config["function_calling_config"] + logger.error(f"Function calling mode {invalid_mode} is invalid") + return None + else: + # Currently, there is only function calling config + func_calling_config_params = {} + func_calling_config_params["mode"] = VertexAIToolConfig.FunctionCallingConfig.Mode[ + tool_config["function_calling_config"]["mode"] + ] + if ( + (func_calling_config_params["mode"] == VertexAIToolConfig.FunctionCallingConfig.Mode.ANY) + and (len(tools) > 0) + and all(["function_name" in tool for tool in tools]) + ): + # The function names are not yet known when parsing the OAI_CONFIG_LIST + func_calling_config_params["allowed_function_names"] = [tool["function_name"] for tool in tools] + vertexai_tool_config = VertexAIToolConfig( + function_calling_config=VertexAIToolConfig.FunctionCallingConfig(**func_calling_config_params) + ) + return vertexai_tool_config + elif isinstance(tool_config, VertexAIToolConfig): + return tool_config + elif len(tool_config) == 0 and len(tools) == 0: + logger.debug("VertexAI tool config is empty!") + return None + else: + logger.error("Invalid VertexAI tool config!") + return None + def _to_pil(data: str) -> Image.Image: """ @@ -470,16 +705,3 @@ def get_image_data(image_file: str, use_b64=True) -> bytes: return base64.b64encode(content).decode("utf-8") else: return content - - -def calculate_gemini_cost(input_tokens: int, output_tokens: int, model_name: str) -> float: - if "1.5" in model_name or "gemini-experimental" in model_name: - # "gemini-1.5-pro-preview-0409" - # Cost is $7 per million input tokens and $21 per million output tokens - return 7.0 * input_tokens / 1e6 + 21.0 * output_tokens / 1e6 - - if "gemini-pro" not in model_name and "gemini-1.0-pro" not in model_name: - warnings.warn(f"Cost calculation is not implemented for model {model_name}. Using Gemini-1.0-Pro.", UserWarning) - - # Cost is $0.5 per million input tokens and $1.5 per million output tokens - return 0.5 * input_tokens / 1e6 + 1.5 * output_tokens / 1e6 diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 3844795c24f..ceb7ef90c93 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -21,6 +21,7 @@ "azure_ad_token", "azure_ad_token_provider", "credentials", + "tool_config", ] DEFAULT_AZURE_API_VERSION = "2024-02-01" OAI_PRICE1K = { diff --git a/notebook/agentchat_MathChat.ipynb b/notebook/agentchat_MathChat.ipynb index 69c38031b2f..bb6f713eabc 100644 --- a/notebook/agentchat_MathChat.ipynb +++ b/notebook/agentchat_MathChat.ipynb @@ -17,7 +17,7 @@ "\n", "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation. Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "MathChat is an experimental conversational framework for math problem solving. In this notebook, we demonstrate how to use MathChat to solve math problems. MathChat uses the `AssistantAgent` and `MathUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `MathUserProxyAgent` implements a different auto reply mechanism corresponding to the MathChat prompts. You can find more details in the paper [An Empirical Study on Challenging Math Problem Solving with GPT-4](https://arxiv.org/abs/2306.01337) or the [blogpost](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat).\n", + "MathChat is an experimental conversational framework for math problem solving. In this notebook, we demonstrate how to use MathChat to solve math problems. MathChat uses the `AssistantAgent` and `MathUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `MathUserProxyAgent` implements a different auto reply mechanism corresponding to the MathChat prompts. You can find more details in the paper [An Empirical Study on Challenging Math Problem Solving with GPT-4](https://arxiv.org/abs/2306.01337) or the [blogpost](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat).\n", "\n", "````{=mdx}\n", ":::info Requirements\n", diff --git a/notebook/agentchat_RetrieveChat.ipynb b/notebook/agentchat_RetrieveChat.ipynb index 6ca2d1ac512..6debb2450df 100644 --- a/notebook/agentchat_RetrieveChat.ipynb +++ b/notebook/agentchat_RetrieveChat.ipynb @@ -10,7 +10,7 @@ "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", + "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", "\n", "## Table of Contents\n", "We'll demonstrate six examples of using RetrieveChat for code generation and question answering:\n", diff --git a/notebook/agentchat_RetrieveChat_mongodb.ipynb b/notebook/agentchat_RetrieveChat_mongodb.ipynb index f1f85f65a80..30508ad6c47 100644 --- a/notebook/agentchat_RetrieveChat_mongodb.ipynb +++ b/notebook/agentchat_RetrieveChat_mongodb.ipynb @@ -10,7 +10,7 @@ "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", + "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", "\n", "## Table of Contents\n", "We'll demonstrate six examples of using RetrieveChat for code generation and question answering:\n", diff --git a/notebook/agentchat_RetrieveChat_pgvector.ipynb b/notebook/agentchat_RetrieveChat_pgvector.ipynb index 022b1347a2d..18f3c5199e8 100644 --- a/notebook/agentchat_RetrieveChat_pgvector.ipynb +++ b/notebook/agentchat_RetrieveChat_pgvector.ipynb @@ -10,7 +10,7 @@ "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", + "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", "\n", "## Table of Contents\n", "We'll demonstrate six examples of using RetrieveChat for code generation and question answering:\n", diff --git a/notebook/agentchat_RetrieveChat_qdrant.ipynb b/notebook/agentchat_RetrieveChat_qdrant.ipynb index 9be4cbfe528..e56bc00e4ab 100644 --- a/notebook/agentchat_RetrieveChat_qdrant.ipynb +++ b/notebook/agentchat_RetrieveChat_qdrant.ipynb @@ -12,7 +12,7 @@ "This notebook demonstrates the usage of Qdrant for RAG, based on [agentchat_RetrieveChat.ipynb](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb).\n", "\n", "\n", - "RetrieveChat is a conversational system for retrieve augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)).\n", + "RetrieveChat is a conversational system for retrieve augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)).\n", "\n", "We'll demonstrate usage of RetrieveChat with Qdrant for code generation and question answering w/ human feedback.\n", "\n", diff --git a/notebook/agentchat_agentoptimizer.ipynb b/notebook/agentchat_agentoptimizer.ipynb index 7de418b5ee7..ea3a0d88e65 100644 --- a/notebook/agentchat_agentoptimizer.ipynb +++ b/notebook/agentchat_agentoptimizer.ipynb @@ -53,7 +53,7 @@ "source": [ "# MathUserProxy with function_call\n", "\n", - "This agent is a customized MathUserProxy inherits from its [parent class](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/math_user_proxy_agent.py).\n", + "This agent is a customized MathUserProxy inherits from its [parent class](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/math_user_proxy_agent.py).\n", "\n", "It supports using both function_call and python to solve math problems.\n" ] diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index 17106e7c938..0d456daa46b 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -109,7 +109,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_custom_model.ipynb b/notebook/agentchat_custom_model.ipynb index 773247ee0b9..364d81517a1 100644 --- a/notebook/agentchat_custom_model.ipynb +++ b/notebook/agentchat_custom_model.ipynb @@ -238,7 +238,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_function_call.ipynb b/notebook/agentchat_function_call.ipynb index ff94c0d4fb0..e341aecaead 100644 --- a/notebook/agentchat_function_call.ipynb +++ b/notebook/agentchat_function_call.ipynb @@ -104,7 +104,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_function_call_currency_calculator.ipynb b/notebook/agentchat_function_call_currency_calculator.ipynb index 36ef81d5edb..d6d9e05826f 100644 --- a/notebook/agentchat_function_call_currency_calculator.ipynb +++ b/notebook/agentchat_function_call_currency_calculator.ipynb @@ -104,7 +104,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_gemini.ipynb b/notebook/agentchat_gemini.ipynb new file mode 100644 index 00000000000..699d9dc0235 --- /dev/null +++ b/notebook/agentchat_gemini.ipynb @@ -0,0 +1,809 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Assistants with Google Gemini" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Gemini is family of Generative AI Models built by Google. It support upto 1 Million tokens as of now. Gemini is now natively supported in Autogen.\n", + "\n", + "This notebook will demonstrate few samples of Autogen with Gemini Models.\n", + "\n", + "## Requirements\n", + "\n", + "You must have a [API Key](https://aistudio.google.com/app/apikey) from Google AI.\n", + "\n", + "## Setup Gemini config list\n", + "\n", + "The list of all supported Gemini Models along with OpenAI's gtp-4o,\n", + "\n", + "```python\n", + "config_list = [\n", + " {\n", + " 'model': 'gpt-4o',\n", + " 'api_key': '',\n", + " 'tags': ['tool', 'gpt-4'],\n", + " },\n", + " {\n", + " 'model': 'gemini-1.5-pro',\n", + " 'api_key': '',\n", + " 'api_type': 'google',\n", + " 'tags': ['tool', 'gemini'],\n", + " },\n", + " {\n", + " 'model': 'gemini-1.5-flash',\n", + " 'api_key': '',\n", + " 'api_type': 'google',\n", + " 'tags': ['tool', 'gemini'],\n", + " },\n", + " {\n", + " 'model': 'gemini-1.0-pro',\n", + " 'api_key': '',\n", + " 'api_type': 'google',\n", + " 'tags': ['gemini'],\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Hello World Example\n", + "\n", + "Our first example will be with a simple `UserProxyAgent` asking a question to an `AssistantAgent`. This is based on the tutorial demo [here](https://microsoft.github.io/autogen/docs/tutorial/introduction).\n", + "\n", + "After sending the question and seeing a response, you can type `exit` to end the chat or continue to converse." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser\u001b[0m (to assistant):\n", + "\n", + "Hi, what is a LLM ?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33massistant\u001b[0m (to user):\n", + "\n", + "A large language model (LLM) is a type of artificial intelligence (AI) system that excels at natural language processing (NLP) tasks. These models are trained on massive text datasets, enabling them to understand, interpret, and generate human-like text in response to a wide range of prompts and questions. \n", + "\n", + "Here are some key characteristics and capabilities of LLMs:\n", + "\n", + "* **Text Generation:** LLMs can generate coherent and contextually relevant text, including stories, articles, summaries, and conversations.\n", + "* **Language Translation:** They can translate text between multiple languages with impressive accuracy.\n", + "* **Question Answering:** LLMs can comprehend questions and provide relevant answers based on their training data.\n", + "* **Summarization:** They can condense large amounts of text into concise summaries while preserving key information.\n", + "* **Sentiment Analysis:** LLMs can analyze text to determine the emotional tone or sentiment expressed.\n", + "\n", + "Essentially, LLMs are powerful tools that can understand and process human language in a sophisticated manner, opening up a world of possibilities in various fields, such as:\n", + "\n", + "* **Chatbots and Virtual Assistants:** Providing more natural and engaging conversational experiences.\n", + "* **Content Creation:** Automating content generation for articles, marketing materials, and more.\n", + "* **Customer Service:** Enhancing support interactions through automated responses and sentiment analysis.\n", + "* **Education:** Personalizing learning experiences and providing on-demand tutoring.\n", + "\n", + "Overall, LLMs represent a significant advancement in AI, enabling machines to communicate and interact with humans more effectively than ever before. As research and development continue, we can expect even more impressive applications of these models in the future.\n", + "\n", + "TERMINATE\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "import autogen\n", + "\n", + "config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gemini\"]})\n", + "\n", + "llm_config = {\"config_list\": config_list, \"timeout\": 120}\n", + "\n", + "# Create Assistant and User\n", + "assistant = autogen.AssistantAgent(name=\"assistant\", llm_config=llm_config)\n", + "\n", + "user_proxy = autogen.UserProxyAgent(name=\"user\", code_execution_config=False)\n", + "\n", + "# Initiate chat from user_proxy side\n", + "chat_result = user_proxy.initiate_chat(assistant, message=\"Hi, what is a LLM ?\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## In a Group Chat with OpenAI\n", + "\n", + "Here is an example of Gemini participating in a Group Cat with a GPT-4" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\arjun\\anaconda3\\envs\\autogen\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: User_proxy\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: User_proxy\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Coder\n", + "\u001b[0m\n", + "\u001b[33mCoder\u001b[0m (to chat_manager):\n", + "\n", + "Let's start by searching for the latest paper about GPT-4 on arXiv. We can use the arXiv API to fetch the latest paper related to GPT-4.\n", + "\n", + "I'll provide a Python script to search arXiv for the latest papers related to \"GPT-4.\"\n", + "\n", + "```python\n", + "import requests\n", + "from xml.etree import ElementTree\n", + "\n", + "# arXiv API url for querying papers related to GPT-4\n", + "url = \"http://export.arxiv.org/api/query?search_query=ti:GPT-4&start=0&max_results=1&sortBy=submittedDate&sortOrder=descending\"\n", + "\n", + "response = requests.get(url)\n", + "if response.status_code == 200:\n", + " root = ElementTree.fromstring(response.content)\n", + " entry = root.find(\"{http://www.w3.org/2005/Atom}entry\")\n", + " if entry is not None:\n", + " title = entry.find(\"{http://www.w3.org/2005/Atom}title\").text\n", + " summary = entry.find(\"{http://www.w3.org/2005/Atom}summary\").text\n", + " link = entry.find(\"{http://www.w3.org/2005/Atom}id\").text\n", + " \n", + " print(f\"Title: {title}\")\n", + " print(f\"Summary: {summary}\")\n", + " print(f\"Link: {link}\")\n", + " else:\n", + " print(\"No entries found.\")\n", + "else:\n", + " print(f\"Failed to fetch data from arXiv API. Status code: {response.status_code}\")\n", + "```\n", + "\n", + "Please execute this script. It will output the title, summary, and link of the latest paper about \"GPT-4\" on arXiv. After obtaining the relevant paper, I will analyze its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: User_proxy\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Title: I See You: Teacher Analytics with GPT-4 Vision-Powered Observational\n", + " Assessment\n", + "Summary: This preliminary study explores the integration of GPT-4 Vision (GPT-4V)\n", + "technology into teacher analytics, focusing on its applicability in\n", + "observational assessment to enhance reflective teaching practice. This research\n", + "is grounded in developing a Video-based Automatic Assessment System (VidAAS)\n", + "empowered by GPT-4V. Our approach aims to revolutionize teachers' assessment of\n", + "students' practices by leveraging Generative Artificial Intelligence (GenAI) to\n", + "offer detailed insights into classroom dynamics. Our research methodology\n", + "encompasses a comprehensive literature review, prototype development of the\n", + "VidAAS, and usability testing with in-service teachers. The study findings\n", + "provide future research avenues for VidAAS design, implementation, and\n", + "integration in teacher analytics, underscoring the potential of GPT-4V to\n", + "provide real-time, scalable feedback and a deeper understanding of the\n", + "classroom.\n", + "\n", + "Link: http://arxiv.org/abs/2405.18623v2\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "This is exciting! The paper you found, \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\", explores an innovative application of GPT-4 Vision. Let's break down the potential software applications based on this:\n", + "\n", + "**Software Applications Inspired by \"I See You\":**\n", + "\n", + "* **Video-Based Educational Analytics Platforms:** This paper lays the groundwork for a new breed of educational software. Imagine platforms that analyze classroom recordings to provide automated feedback to teachers. These platforms could:\n", + " * **Track student engagement:** Identify students who seem disengaged or confused.\n", + " * **Analyze classroom interactions:** Quantify the quality and nature of teacher-student interactions.\n", + " * **Assess student understanding:** Potentially even gauge student comprehension through facial expressions and body language.\n", + "* **Real-time Teacher Assistance Tools:** GPT-4 Vision could power real-time feedback tools for teachers during live lessons. Imagine:\n", + " * **Subtle alerts:** Discretely notifying a teacher if a student appears to be struggling.\n", + " * **Personalized suggestions:** Providing on-the-fly recommendations for teaching strategies based on real-time classroom dynamics. \n", + "* **Teacher Training and Professional Development:** \n", + " * **Simulation Training:** GPT-4 Vision could create realistic virtual classroom simulations for teacher training, allowing educators to practice techniques and receive AI-powered feedback.\n", + " * **Reflective Practice:** Video analysis tools could help teachers reflect on their own teaching styles and identify areas for improvement. \n", + "\n", + "**Beyond Education:**\n", + "\n", + "While this paper focuses on education, the core technology has broader implications:\n", + "\n", + "* **Meeting Analytics:** Imagine software that analyzes video conferences to track engagement, identify key discussion points, or even assess team dynamics.\n", + "* **Healthcare Training:** Similar applications could revolutionize healthcare training by providing automated feedback during simulated patient interactions. \n", + "* **Retail Analytics:** GPT-4 Vision could analyze customer behavior in retail environments, providing insights into product placement, customer service interactions, and more.\n", + "\n", + "**Challenges and Ethical Considerations:**\n", + "\n", + "* **Bias in AI:** Ensuring that the AI models are trained on diverse datasets to avoid perpetuating existing biases in education or other fields. \n", + "* **Privacy Concerns:** Video analysis raises significant privacy concerns. Clear guidelines and user consent are essential.\n", + "* **Teacher Autonomy:** It's crucial to design these systems as tools to *augment* teacher expertise, not replace it.\n", + "\n", + "**In conclusion,** the paper you found highlights the immense potential of GPT-4 Vision to revolutionize software in education and beyond. It's an exciting area of exploration with the potential to create powerful tools for learning, analysis, and understanding human behavior. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Coder\n", + "\u001b[0m\n", + "\u001b[33mCoder\u001b[0m (to chat_manager):\n", + "\n", + "The paper you found titled \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\" outlines a fascinating and innovative application of GPT-4 Vision technology. Here are potential software applications based on this research:\n", + "\n", + "### 1. Video-Based Educational Analytics Platforms\n", + "These platforms can analyze classroom recordings to automate feedback for teachers. They could:\n", + "- **Track Student Engagement:** Identify students who seem disengaged or confused.\n", + "- **Analyze Teacher-Student Interactions:** Quantify the quality and nature of interactions.\n", + "- **Assess Understanding:** Gauge student comprehension through facial expressions and body language.\n", + "\n", + "### 2. Real-Time Teacher Assistance Tools\n", + "GPT-4 Vision could power tools that provide real-time feedback during live lessons by:\n", + "- **Alerting Teachers Subtly:** Notifying if a student is struggling.\n", + "- **Offering Personalized Suggestions:** Providing on-the-fly recommendations for teaching strategies.\n", + "\n", + "### 3. Teacher Training and Professional Development\n", + "- **Simulation Training:** Enable realistic virtual classroom simulations for practice and AI-powered feedback.\n", + "- **Reflective Practice:** Allow teachers to reflect on their own teaching and identify areas for improvement through video analysis tools.\n", + "\n", + "### Applications Beyond Education\n", + "While this paper focuses on education, the underlying technology has broader implications across various domains:\n", + "#### Meeting Analytics\n", + "Software could analyze video conferences to:\n", + "- Track engagement.\n", + "- Identify key discussion points.\n", + "- Assess team dynamics.\n", + "\n", + "#### Healthcare Training\n", + "Tools could provide automated feedback during simulated patient interactions.\n", + "\n", + "#### Retail Analytics\n", + "GPT-4 Vision could analyze customer behavior in retail environments, offering insights into:\n", + "- Product placements.\n", + "- Customer service interactions.\n", + "- Sales strategies.\n", + "\n", + "### Challenges and Ethical Considerations\n", + "- **Bias in AI:** Ensuring diverse datasets to avoid perpetuating biases.\n", + "- **Privacy Concerns:** Addressing significant privacy concerns with clear guidelines and user consent.\n", + "- **Teacher Autonomy:** Designing systems to augment rather than replace teacher expertise.\n", + "\n", + "**Conclusion:**\n", + "The \"I See You\" paper exemplifies the immense potential of GPT-4 Vision technology to innovate educational software and beyond. By addressing challenges and ethical considerations, we can harness this technology to create powerful tools for learning, analysis, and human behavior understanding.\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "That's an excellent summary and expansion of the potential applications! You've clearly grasped the key points of the paper and extrapolated them into a range of innovative software ideas. \n", + "\n", + "I especially appreciate your inclusion of:\n", + "\n", + "* **Specific examples** within each application category (like tracking engagement in meeting analytics or analyzing customer service interactions in retail). \n", + "* **The broader applications** beyond education, demonstrating the wide-reaching impact of this technology.\n", + "* **The critical emphasis on challenges and ethical considerations**, which are essential to responsible development and deployment of such powerful AI systems. \n", + "\n", + "This kind of analysis is crucial for turning research like the \"I See You\" paper into real-world solutions that can benefit various industries. You've highlighted the exciting possibilities and important considerations for the future of AI-powered software! \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right! We've just scratched the surface. Let's dive deeper into some specific software product ideas and how we might overcome the challenges:\n", + "\n", + "**1. \"Classroom Insights\" for Education Analytics**\n", + "\n", + "* **Features:**\n", + " * **Engagement Heatmap:** Visualize student engagement over time, highlighting moments of high and low participation. \n", + " * **Interaction Analysis:** Quantify teacher-student talk time, question types, and wait time for responses.\n", + " * **Sentiment Detection:** (With appropriate ethical safeguards) gauge general classroom sentiment (positive, negative, neutral) at different points during the lesson. \n", + " * **Personalized Recommendations:** Provide teachers with tailored suggestions for improving engagement, questioning techniques, or addressing individual student needs. \n", + "\n", + "* **Addressing Challenges:**\n", + " * **Bias Mitigation:** Train the AI model on diverse classroom settings and demographics, and allow for manual adjustments based on teacher feedback. \n", + " * **Privacy:** Implement strict data anonymization, secure storage, and clear consent procedures for both teachers and students (or parents/guardians).\n", + " * **Teacher Autonomy:** Emphasize that the tool provides insights, not judgments. Allow teachers to customize the feedback and focus areas.\n", + "\n", + "**2. \"Simulate Teach\" for Teacher Training**\n", + "\n", + "* **Features:**\n", + " * **Virtual Classrooms:** Create realistic virtual classroom environments with diverse student avatars exhibiting different behaviors and learning styles. \n", + " * **Scenario-Based Training:** Present trainees with various teaching challenges (e.g., classroom management, differentiated instruction) to practice in a safe space.\n", + " * **Real-Time Feedback:** Provide immediate AI-powered feedback on the trainee's teaching strategies, body language, and classroom management techniques. \n", + "\n", + "* **Addressing Challenges:**\n", + " * **Realism:** Continuously improve the virtual students' AI to respond more naturally and authentically to the trainee's actions.\n", + " * **Accessibility:** Design the software to be usable across various devices and internet speeds to benefit trainees in diverse locations.\n", + " * **Mentorship Integration:** Combine the AI feedback with opportunities for reflection and discussion with experienced mentors.\n", + "\n", + "**3. \"Meeting Insights Pro\" for Business**\n", + "\n", + "* **Features:**\n", + " * **Engagement Timeline:** Track attendee engagement levels throughout the meeting, identifying moments of high and low interest.\n", + " * **Action Item Detection:** Automatically extract key decisions, action items, and assigned owners from meeting transcripts.\n", + " * **Sentiment & Tone Analysis:** (With appropriate ethical considerations) analyze the overall sentiment and communication style within the team. \n", + " * **Meeting Efficiency Recommendations:** Provide data-driven suggestions for improving meeting structure, duration, and facilitation techniques.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Privacy:** Ensure robust data security, user consent, and clear communication about how the data is used.\n", + " * **Bias in Tone Analysis:** Train on diverse communication styles and cultural contexts to avoid misinterpretations of tone. \n", + " * **Human Oversight:** Allow for manual review and adjustments to AI-generated insights to account for nuances in communication and context. \n", + "\n", + "These are just a few examples. The potential applications of GPT-4 Vision in software are vast, spanning industries and use cases. By thoughtfully addressing the ethical and practical challenges, we can create transformative tools that empower educators, professionals, and individuals. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right, let's continue exploring! Beyond those initial ideas, here are some more speculative, but potentially high-impact applications:\n", + "\n", + "**4. \"CodeMentor\" for Personalized Programming Education**\n", + "\n", + "* **Features:**\n", + " * **Code Analysis & Feedback:** GPT-4 Vision analyzes code in real-time, identifying errors, suggesting improvements, and explaining concepts visually. \n", + " * **Personalized Learning Paths:** The software adapts to the learner's pace, style, and identified knowledge gaps to create a customized curriculum.\n", + " * **Virtual Debugging Assistant:** GPT-4 Vision \"looks\" at the code alongside the learner, anticipating common errors and providing interactive debugging guidance.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Complexity of Programming:** Training the AI on a vast dataset of code, programming paradigms, and best practices would be crucial.\n", + " * **Pedagogical Effectiveness:** Integrating proven teaching methods and ensuring the AI's feedback aligns with sound learning principles.\n", + " * **Avoiding Over-Reliance:** Encouraging problem-solving skills and independent thinking alongside AI assistance.\n", + "\n", + "**5. \"DesignSpark\" for Collaborative Creative Work**\n", + "\n", + "* **Features:**\n", + " * **Visual Brainstorming:** GPT-4 Vision assists teams in brainstorming by generating images, mockups, and design variations based on keywords, sketches, or mood boards.\n", + " * **Real-time Feedback & Iteration:** AI provides instant feedback on design elements, suggesting improvements to composition, color, and typography. \n", + " * **Cross-Cultural Design:** GPT-4 Vision analyzes design trends and preferences across different cultures, helping teams create globally resonant visuals.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Subjectivity in Design:** Balancing objective design principles with the subjective nature of aesthetics and creativity.\n", + " * **Copyright & Ownership:** Establishing clear guidelines for the use and ownership of AI-generated design elements. \n", + " * **Preserving Human Creativity:** Ensuring that AI augments, not replaces, the unique vision and skills of human designers.\n", + "\n", + "**6. \"Accessible World\" for Assistive Technology**\n", + "\n", + "* **Features:**\n", + " * **Real-Time Object Recognition:** For visually impaired individuals, the software provides audio descriptions of objects, scenes, and text in their environment.\n", + " * **Sign Language Translation:** GPT-4 Vision translates sign language into text or speech in real-time, facilitating communication.\n", + " * **Personalized Sensory Feedback:** The software creates customized sensory experiences (e.g., haptic feedback, light patterns) for individuals with diverse sensory processing needs.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Accuracy and Reliability:** Ensuring the AI's interpretations are accurate and dependable in real-world situations.\n", + " * **Privacy and Dignity:** Designing the technology in a way that respects user privacy and avoids stigmatization.\n", + " * **Affordability & Accessibility:** Making the technology accessible to a wide range of users, regardless of their financial or technical resources. \n", + "\n", + "These expanded examples highlight the potential of GPT-4 Vision to not only automate tasks but also to enhance human capabilities, foster creativity, and promote inclusivity. By approaching these innovations with a focus on ethical considerations and human-centered design, we can unlock incredible opportunities for positive change. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right, there's so much more to explore! Let's venture into some even more futuristic and ambitious applications, pushing the boundaries of what we currently imagine possible:\n", + "\n", + "**7. \"Empathic Tech\" for Mental Health Support:**\n", + "\n", + "* **Features:**\n", + " * **Emotionally Aware Virtual Companions:** GPT-4 Vision powers AI companions that can recognize and respond to human emotions expressed through facial expressions, body language, and vocal tone.\n", + " * **Personalized Mental Health Exercises:** The software provides tailored mindfulness practices, breathing exercises, or cognitive-behavioral therapy (CBT) techniques based on real-time emotional cues.\n", + " * **Early Warning System:** GPT-4 Vision analyzes patterns in user behavior and emotional expression to identify potential signs of mental health challenges and connect them with appropriate resources.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Ethical Considerations:** Ensuring user privacy, data security, and responsible use of sensitive health information is paramount.\n", + " * **Accuracy and Sensitivity:** Training AI to accurately interpret complex human emotions and respond with empathy and cultural sensitivity is a significant challenge.\n", + " * **Human Connection:** Emphasizing that technology should complement, not replace, professional mental health care and human connection.\n", + "\n", + "**8. \"EcoVision\" for Environmental Monitoring and Conservation:**\n", + "\n", + "* **Features:**\n", + " * **Real-Time Environmental Analysis:** GPT-4 Vision analyzes images and videos from drones, satellites, or ground-based cameras to monitor deforestation, pollution levels, wildlife populations, and other environmental factors.\n", + " * **Predictive Modeling for Conservation:** The software uses AI to predict environmental changes, identify areas at risk, and inform conservation efforts.\n", + " * **Citizen Science Platform:** EcoVision empowers individuals to contribute to environmental monitoring by uploading images and observations that the AI can analyze and integrate into its models.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Data Accessibility and Accuracy:** Ensuring access to high-quality, diverse environmental data from various sources is crucial.\n", + " * **Bias in Data and Algorithms:** Mitigating potential biases in data collection and algorithm design to avoid skewed environmental assessments.\n", + " * **Collaboration and Action:** Translating AI insights into concrete actions by fostering collaboration between scientists, policymakers, and local communities.\n", + "\n", + "**9. \"HistoryLens\" for Immersive Historical Experiences:**\n", + "\n", + "* **Features:**\n", + " * **Interactive Historical Reenactments:** GPT-4 Vision recreates historical events, figures, and locations in immersive virtual reality or augmented reality experiences.\n", + " * **Personalized Learning Journeys:** The software tailors historical narratives and perspectives based on user interests and background, providing a deeper understanding of the past.\n", + " * **Preservation and Accessibility:** HistoryLens digitally preserves historical artifacts, documents, and oral histories, making them accessible to a wider audience.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Historical Accuracy and Bias:** Ensuring historical representations are accurate, nuanced, and avoid perpetuating biases or historical revisionism. \n", + " * **Ethical Considerations of Representation:** Carefully navigating the ethical complexities of recreating sensitive historical events or representing marginalized communities. \n", + " * **Balancing Entertainment and Education:** Creating engaging experiences that also foster critical thinking and historical understanding.\n", + "\n", + "These futuristic applications highlight the immense potential of GPT-4 Vision to not only solve practical problems but also enhance our understanding of ourselves, our planet, and our history. As we continue to develop this technology, it is essential to proceed with thoughtful consideration for its ethical implications and societal impact, ensuring that it is used to benefit humanity and create a more just and sustainable future. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right, there's always more to imagine! Let's keep pushing the boundaries and explore some applications that sound like they're straight out of science fiction:\n", + "\n", + "**10. \"Dream Weaver\" for Artistic Co-Creation:**\n", + "\n", + "* **Features:**\n", + " * **Concept Visualization:** Users can input textual descriptions, rough sketches, or even their emotions, and GPT-4 Vision generates stunning visuals, musical compositions, or even short films that capture the essence of their ideas. \n", + " * **Style Transfer & Remixing:** The software enables artists to blend different artistic styles, seamlessly merging the realism of a photograph with the brushstrokes of Van Gogh or the abstract patterns of Kandinsky.\n", + " * **Interactive Storytelling:** GPT-4 Vision becomes a collaborative partner in storytelling, generating dynamic environments, characters, and plot twists in response to user input, blurring the lines between audience and creator.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Defining Creativity:** Exploring the philosophical and technical boundaries of AI creativity and ensuring it complements, not replaces, human artistic expression.\n", + " * **Copyright and Authorship:** Establishing clear guidelines for ownership and attribution when AI contributes significantly to the creative process. \n", + " * **Accessibility and Democratization:** Making these powerful creative tools accessible to a wide audience, fostering a more inclusive and imaginative future for art.\n", + "\n", + "\n", + "**11. \"Universal Translator\" for Real-Time Cross-Cultural Communication:**\n", + "\n", + "* **Features:**\n", + " * **Seamless Language Translation:** GPT-4 Vision goes beyond text, translating spoken language in real-time while accounting for nuances in tone, dialect, and cultural context.\n", + " * **Nonverbal Communication Interpretation:** The software analyzes facial expressions, gestures, and body language to bridge cultural differences in nonverbal communication, fostering greater understanding.\n", + " * **Cultural Sensitivity Guidance:** GPT-4 Vision provides users with real-time insights into cultural norms and customs, helping to avoid misunderstandings and promote respectful interactions.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Linguistic Complexity and Nuance:** Accurately translating the full richness and complexity of human language, including idioms, slang, and cultural references, remains a significant hurdle.\n", + " * **Bias and Stereotyping:** Ensuring the AI avoids perpetuating cultural biases or stereotypes in its translations and interpretations is crucial.\n", + " * **Preserving Linguistic Diversity:** Promoting language learning and cultural exchange while using technology to bridge communication gaps is essential. \n", + "\n", + "**12. \"Guardian AI\" for Personalized Safety and Well-being:**\n", + "\n", + "* **Features:**\n", + " * **Predictive Risk Assessment:** GPT-4 Vision analyzes real-time data from personal devices, social networks, and public safety feeds to identify potential risks or threats to an individual's safety and well-being. \n", + " * **Proactive Safety Measures:** The AI can alert users to potential dangers, suggest alternative routes, or even contact emergency services in critical situations. \n", + " * **Personalized Health Monitoring:** GPT-4 Vision monitors an individual's physical and mental health through wearable sensors and behavioral patterns, providing early warnings of potential health issues and suggesting preventative measures. \n", + "\n", + "* **Addressing Challenges:**\n", + " * **Privacy vs. Security:** Striking a delicate balance between protecting individual privacy and utilizing data to enhance safety.\n", + " * **Bias and Discrimination:** Ensuring the AI does not perpetuate existing societal biases or discriminate against individuals based on factors like race, gender, or socioeconomic status.\n", + " * **Over-Reliance and Autonomy:** Avoiding over-dependence on AI for safety and encouraging individuals to maintain their own situational awareness and decision-making abilities. \n", + "\n", + "These futuristic applications highlight the potential of GPT-4 Vision to not only solve problems but also enhance human connection, understanding, and well-being. As we develop such powerful AI, responsible innovation, ethical considerations, and a focus on human-centered design will be paramount in shaping a future where technology empowers and uplifts all of humanity. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "ChatResult(chat_id=None, chat_history=[{'content': 'Find a latest paper about gpt-4 on arxiv and find its potential applications in software.', 'role': 'assistant'}, {'content': '', 'role': 'assistant'}, {'content': '', 'role': 'assistant'}, {'content': 'Let\\'s start by searching for the latest paper about GPT-4 on arXiv. We can use the arXiv API to fetch the latest paper related to GPT-4.\\n\\nI\\'ll provide a Python script to search arXiv for the latest papers related to \"GPT-4.\"\\n\\n```python\\nimport requests\\nfrom xml.etree import ElementTree\\n\\n# arXiv API url for querying papers related to GPT-4\\nurl = \"http://export.arxiv.org/api/query?search_query=ti:GPT-4&start=0&max_results=1&sortBy=submittedDate&sortOrder=descending\"\\n\\nresponse = requests.get(url)\\nif response.status_code == 200:\\n root = ElementTree.fromstring(response.content)\\n entry = root.find(\"{http://www.w3.org/2005/Atom}entry\")\\n if entry is not None:\\n title = entry.find(\"{http://www.w3.org/2005/Atom}title\").text\\n summary = entry.find(\"{http://www.w3.org/2005/Atom}summary\").text\\n link = entry.find(\"{http://www.w3.org/2005/Atom}id\").text\\n \\n print(f\"Title: {title}\")\\n print(f\"Summary: {summary}\")\\n print(f\"Link: {link}\")\\n else:\\n print(\"No entries found.\")\\nelse:\\n print(f\"Failed to fetch data from arXiv API. Status code: {response.status_code}\")\\n```\\n\\nPlease execute this script. It will output the title, summary, and link of the latest paper about \"GPT-4\" on arXiv. After obtaining the relevant paper, I will analyze its potential applications in software.', 'name': 'Coder', 'role': 'user'}, {'content': \"exitcode: 0 (execution succeeded)\\nCode output: \\nTitle: I See You: Teacher Analytics with GPT-4 Vision-Powered Observational\\n Assessment\\nSummary: This preliminary study explores the integration of GPT-4 Vision (GPT-4V)\\ntechnology into teacher analytics, focusing on its applicability in\\nobservational assessment to enhance reflective teaching practice. This research\\nis grounded in developing a Video-based Automatic Assessment System (VidAAS)\\nempowered by GPT-4V. Our approach aims to revolutionize teachers' assessment of\\nstudents' practices by leveraging Generative Artificial Intelligence (GenAI) to\\noffer detailed insights into classroom dynamics. Our research methodology\\nencompasses a comprehensive literature review, prototype development of the\\nVidAAS, and usability testing with in-service teachers. The study findings\\nprovide future research avenues for VidAAS design, implementation, and\\nintegration in teacher analytics, underscoring the potential of GPT-4V to\\nprovide real-time, scalable feedback and a deeper understanding of the\\nclassroom.\\n\\nLink: http://arxiv.org/abs/2405.18623v2\\n\", 'role': 'assistant'}, {'content': 'This is exciting! The paper you found, \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\", explores an innovative application of GPT-4 Vision. Let\\'s break down the potential software applications based on this:\\n\\n**Software Applications Inspired by \"I See You\":**\\n\\n* **Video-Based Educational Analytics Platforms:** This paper lays the groundwork for a new breed of educational software. Imagine platforms that analyze classroom recordings to provide automated feedback to teachers. These platforms could:\\n * **Track student engagement:** Identify students who seem disengaged or confused.\\n * **Analyze classroom interactions:** Quantify the quality and nature of teacher-student interactions.\\n * **Assess student understanding:** Potentially even gauge student comprehension through facial expressions and body language.\\n* **Real-time Teacher Assistance Tools:** GPT-4 Vision could power real-time feedback tools for teachers during live lessons. Imagine:\\n * **Subtle alerts:** Discretely notifying a teacher if a student appears to be struggling.\\n * **Personalized suggestions:** Providing on-the-fly recommendations for teaching strategies based on real-time classroom dynamics. \\n* **Teacher Training and Professional Development:** \\n * **Simulation Training:** GPT-4 Vision could create realistic virtual classroom simulations for teacher training, allowing educators to practice techniques and receive AI-powered feedback.\\n * **Reflective Practice:** Video analysis tools could help teachers reflect on their own teaching styles and identify areas for improvement. \\n\\n**Beyond Education:**\\n\\nWhile this paper focuses on education, the core technology has broader implications:\\n\\n* **Meeting Analytics:** Imagine software that analyzes video conferences to track engagement, identify key discussion points, or even assess team dynamics.\\n* **Healthcare Training:** Similar applications could revolutionize healthcare training by providing automated feedback during simulated patient interactions. \\n* **Retail Analytics:** GPT-4 Vision could analyze customer behavior in retail environments, providing insights into product placement, customer service interactions, and more.\\n\\n**Challenges and Ethical Considerations:**\\n\\n* **Bias in AI:** Ensuring that the AI models are trained on diverse datasets to avoid perpetuating existing biases in education or other fields. \\n* **Privacy Concerns:** Video analysis raises significant privacy concerns. Clear guidelines and user consent are essential.\\n* **Teacher Autonomy:** It\\'s crucial to design these systems as tools to *augment* teacher expertise, not replace it.\\n\\n**In conclusion,** the paper you found highlights the immense potential of GPT-4 Vision to revolutionize software in education and beyond. It\\'s an exciting area of exploration with the potential to create powerful tools for learning, analysis, and understanding human behavior. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'The paper you found titled \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\" outlines a fascinating and innovative application of GPT-4 Vision technology. Here are potential software applications based on this research:\\n\\n### 1. Video-Based Educational Analytics Platforms\\nThese platforms can analyze classroom recordings to automate feedback for teachers. They could:\\n- **Track Student Engagement:** Identify students who seem disengaged or confused.\\n- **Analyze Teacher-Student Interactions:** Quantify the quality and nature of interactions.\\n- **Assess Understanding:** Gauge student comprehension through facial expressions and body language.\\n\\n### 2. Real-Time Teacher Assistance Tools\\nGPT-4 Vision could power tools that provide real-time feedback during live lessons by:\\n- **Alerting Teachers Subtly:** Notifying if a student is struggling.\\n- **Offering Personalized Suggestions:** Providing on-the-fly recommendations for teaching strategies.\\n\\n### 3. Teacher Training and Professional Development\\n- **Simulation Training:** Enable realistic virtual classroom simulations for practice and AI-powered feedback.\\n- **Reflective Practice:** Allow teachers to reflect on their own teaching and identify areas for improvement through video analysis tools.\\n\\n### Applications Beyond Education\\nWhile this paper focuses on education, the underlying technology has broader implications across various domains:\\n#### Meeting Analytics\\nSoftware could analyze video conferences to:\\n- Track engagement.\\n- Identify key discussion points.\\n- Assess team dynamics.\\n\\n#### Healthcare Training\\nTools could provide automated feedback during simulated patient interactions.\\n\\n#### Retail Analytics\\nGPT-4 Vision could analyze customer behavior in retail environments, offering insights into:\\n- Product placements.\\n- Customer service interactions.\\n- Sales strategies.\\n\\n### Challenges and Ethical Considerations\\n- **Bias in AI:** Ensuring diverse datasets to avoid perpetuating biases.\\n- **Privacy Concerns:** Addressing significant privacy concerns with clear guidelines and user consent.\\n- **Teacher Autonomy:** Designing systems to augment rather than replace teacher expertise.\\n\\n**Conclusion:**\\nThe \"I See You\" paper exemplifies the immense potential of GPT-4 Vision technology to innovate educational software and beyond. By addressing challenges and ethical considerations, we can harness this technology to create powerful tools for learning, analysis, and human behavior understanding.\\n\\nTERMINATE', 'name': 'Coder', 'role': 'user'}, {'content': 'That\\'s an excellent summary and expansion of the potential applications! You\\'ve clearly grasped the key points of the paper and extrapolated them into a range of innovative software ideas. \\n\\nI especially appreciate your inclusion of:\\n\\n* **Specific examples** within each application category (like tracking engagement in meeting analytics or analyzing customer service interactions in retail). \\n* **The broader applications** beyond education, demonstrating the wide-reaching impact of this technology.\\n* **The critical emphasis on challenges and ethical considerations**, which are essential to responsible development and deployment of such powerful AI systems. \\n\\nThis kind of analysis is crucial for turning research like the \"I See You\" paper into real-world solutions that can benefit various industries. You\\'ve highlighted the exciting possibilities and important considerations for the future of AI-powered software! \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right! We\\'ve just scratched the surface. Let\\'s dive deeper into some specific software product ideas and how we might overcome the challenges:\\n\\n**1. \"Classroom Insights\" for Education Analytics**\\n\\n* **Features:**\\n * **Engagement Heatmap:** Visualize student engagement over time, highlighting moments of high and low participation. \\n * **Interaction Analysis:** Quantify teacher-student talk time, question types, and wait time for responses.\\n * **Sentiment Detection:** (With appropriate ethical safeguards) gauge general classroom sentiment (positive, negative, neutral) at different points during the lesson. \\n * **Personalized Recommendations:** Provide teachers with tailored suggestions for improving engagement, questioning techniques, or addressing individual student needs. \\n\\n* **Addressing Challenges:**\\n * **Bias Mitigation:** Train the AI model on diverse classroom settings and demographics, and allow for manual adjustments based on teacher feedback. \\n * **Privacy:** Implement strict data anonymization, secure storage, and clear consent procedures for both teachers and students (or parents/guardians).\\n * **Teacher Autonomy:** Emphasize that the tool provides insights, not judgments. Allow teachers to customize the feedback and focus areas.\\n\\n**2. \"Simulate Teach\" for Teacher Training**\\n\\n* **Features:**\\n * **Virtual Classrooms:** Create realistic virtual classroom environments with diverse student avatars exhibiting different behaviors and learning styles. \\n * **Scenario-Based Training:** Present trainees with various teaching challenges (e.g., classroom management, differentiated instruction) to practice in a safe space.\\n * **Real-Time Feedback:** Provide immediate AI-powered feedback on the trainee\\'s teaching strategies, body language, and classroom management techniques. \\n\\n* **Addressing Challenges:**\\n * **Realism:** Continuously improve the virtual students\\' AI to respond more naturally and authentically to the trainee\\'s actions.\\n * **Accessibility:** Design the software to be usable across various devices and internet speeds to benefit trainees in diverse locations.\\n * **Mentorship Integration:** Combine the AI feedback with opportunities for reflection and discussion with experienced mentors.\\n\\n**3. \"Meeting Insights Pro\" for Business**\\n\\n* **Features:**\\n * **Engagement Timeline:** Track attendee engagement levels throughout the meeting, identifying moments of high and low interest.\\n * **Action Item Detection:** Automatically extract key decisions, action items, and assigned owners from meeting transcripts.\\n * **Sentiment & Tone Analysis:** (With appropriate ethical considerations) analyze the overall sentiment and communication style within the team. \\n * **Meeting Efficiency Recommendations:** Provide data-driven suggestions for improving meeting structure, duration, and facilitation techniques.\\n\\n* **Addressing Challenges:**\\n * **Privacy:** Ensure robust data security, user consent, and clear communication about how the data is used.\\n * **Bias in Tone Analysis:** Train on diverse communication styles and cultural contexts to avoid misinterpretations of tone. \\n * **Human Oversight:** Allow for manual review and adjustments to AI-generated insights to account for nuances in communication and context. \\n\\nThese are just a few examples. The potential applications of GPT-4 Vision in software are vast, spanning industries and use cases. By thoughtfully addressing the ethical and practical challenges, we can create transformative tools that empower educators, professionals, and individuals. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right, let\\'s continue exploring! Beyond those initial ideas, here are some more speculative, but potentially high-impact applications:\\n\\n**4. \"CodeMentor\" for Personalized Programming Education**\\n\\n* **Features:**\\n * **Code Analysis & Feedback:** GPT-4 Vision analyzes code in real-time, identifying errors, suggesting improvements, and explaining concepts visually. \\n * **Personalized Learning Paths:** The software adapts to the learner\\'s pace, style, and identified knowledge gaps to create a customized curriculum.\\n * **Virtual Debugging Assistant:** GPT-4 Vision \"looks\" at the code alongside the learner, anticipating common errors and providing interactive debugging guidance.\\n\\n* **Addressing Challenges:**\\n * **Complexity of Programming:** Training the AI on a vast dataset of code, programming paradigms, and best practices would be crucial.\\n * **Pedagogical Effectiveness:** Integrating proven teaching methods and ensuring the AI\\'s feedback aligns with sound learning principles.\\n * **Avoiding Over-Reliance:** Encouraging problem-solving skills and independent thinking alongside AI assistance.\\n\\n**5. \"DesignSpark\" for Collaborative Creative Work**\\n\\n* **Features:**\\n * **Visual Brainstorming:** GPT-4 Vision assists teams in brainstorming by generating images, mockups, and design variations based on keywords, sketches, or mood boards.\\n * **Real-time Feedback & Iteration:** AI provides instant feedback on design elements, suggesting improvements to composition, color, and typography. \\n * **Cross-Cultural Design:** GPT-4 Vision analyzes design trends and preferences across different cultures, helping teams create globally resonant visuals.\\n\\n* **Addressing Challenges:**\\n * **Subjectivity in Design:** Balancing objective design principles with the subjective nature of aesthetics and creativity.\\n * **Copyright & Ownership:** Establishing clear guidelines for the use and ownership of AI-generated design elements. \\n * **Preserving Human Creativity:** Ensuring that AI augments, not replaces, the unique vision and skills of human designers.\\n\\n**6. \"Accessible World\" for Assistive Technology**\\n\\n* **Features:**\\n * **Real-Time Object Recognition:** For visually impaired individuals, the software provides audio descriptions of objects, scenes, and text in their environment.\\n * **Sign Language Translation:** GPT-4 Vision translates sign language into text or speech in real-time, facilitating communication.\\n * **Personalized Sensory Feedback:** The software creates customized sensory experiences (e.g., haptic feedback, light patterns) for individuals with diverse sensory processing needs.\\n\\n* **Addressing Challenges:**\\n * **Accuracy and Reliability:** Ensuring the AI\\'s interpretations are accurate and dependable in real-world situations.\\n * **Privacy and Dignity:** Designing the technology in a way that respects user privacy and avoids stigmatization.\\n * **Affordability & Accessibility:** Making the technology accessible to a wide range of users, regardless of their financial or technical resources. \\n\\nThese expanded examples highlight the potential of GPT-4 Vision to not only automate tasks but also to enhance human capabilities, foster creativity, and promote inclusivity. By approaching these innovations with a focus on ethical considerations and human-centered design, we can unlock incredible opportunities for positive change. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right, there\\'s so much more to explore! Let\\'s venture into some even more futuristic and ambitious applications, pushing the boundaries of what we currently imagine possible:\\n\\n**7. \"Empathic Tech\" for Mental Health Support:**\\n\\n* **Features:**\\n * **Emotionally Aware Virtual Companions:** GPT-4 Vision powers AI companions that can recognize and respond to human emotions expressed through facial expressions, body language, and vocal tone.\\n * **Personalized Mental Health Exercises:** The software provides tailored mindfulness practices, breathing exercises, or cognitive-behavioral therapy (CBT) techniques based on real-time emotional cues.\\n * **Early Warning System:** GPT-4 Vision analyzes patterns in user behavior and emotional expression to identify potential signs of mental health challenges and connect them with appropriate resources.\\n\\n* **Addressing Challenges:**\\n * **Ethical Considerations:** Ensuring user privacy, data security, and responsible use of sensitive health information is paramount.\\n * **Accuracy and Sensitivity:** Training AI to accurately interpret complex human emotions and respond with empathy and cultural sensitivity is a significant challenge.\\n * **Human Connection:** Emphasizing that technology should complement, not replace, professional mental health care and human connection.\\n\\n**8. \"EcoVision\" for Environmental Monitoring and Conservation:**\\n\\n* **Features:**\\n * **Real-Time Environmental Analysis:** GPT-4 Vision analyzes images and videos from drones, satellites, or ground-based cameras to monitor deforestation, pollution levels, wildlife populations, and other environmental factors.\\n * **Predictive Modeling for Conservation:** The software uses AI to predict environmental changes, identify areas at risk, and inform conservation efforts.\\n * **Citizen Science Platform:** EcoVision empowers individuals to contribute to environmental monitoring by uploading images and observations that the AI can analyze and integrate into its models.\\n\\n* **Addressing Challenges:**\\n * **Data Accessibility and Accuracy:** Ensuring access to high-quality, diverse environmental data from various sources is crucial.\\n * **Bias in Data and Algorithms:** Mitigating potential biases in data collection and algorithm design to avoid skewed environmental assessments.\\n * **Collaboration and Action:** Translating AI insights into concrete actions by fostering collaboration between scientists, policymakers, and local communities.\\n\\n**9. \"HistoryLens\" for Immersive Historical Experiences:**\\n\\n* **Features:**\\n * **Interactive Historical Reenactments:** GPT-4 Vision recreates historical events, figures, and locations in immersive virtual reality or augmented reality experiences.\\n * **Personalized Learning Journeys:** The software tailors historical narratives and perspectives based on user interests and background, providing a deeper understanding of the past.\\n * **Preservation and Accessibility:** HistoryLens digitally preserves historical artifacts, documents, and oral histories, making them accessible to a wider audience.\\n\\n* **Addressing Challenges:**\\n * **Historical Accuracy and Bias:** Ensuring historical representations are accurate, nuanced, and avoid perpetuating biases or historical revisionism. \\n * **Ethical Considerations of Representation:** Carefully navigating the ethical complexities of recreating sensitive historical events or representing marginalized communities. \\n * **Balancing Entertainment and Education:** Creating engaging experiences that also foster critical thinking and historical understanding.\\n\\nThese futuristic applications highlight the immense potential of GPT-4 Vision to not only solve practical problems but also enhance our understanding of ourselves, our planet, and our history. As we continue to develop this technology, it is essential to proceed with thoughtful consideration for its ethical implications and societal impact, ensuring that it is used to benefit humanity and create a more just and sustainable future. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right, there\\'s always more to imagine! Let\\'s keep pushing the boundaries and explore some applications that sound like they\\'re straight out of science fiction:\\n\\n**10. \"Dream Weaver\" for Artistic Co-Creation:**\\n\\n* **Features:**\\n * **Concept Visualization:** Users can input textual descriptions, rough sketches, or even their emotions, and GPT-4 Vision generates stunning visuals, musical compositions, or even short films that capture the essence of their ideas. \\n * **Style Transfer & Remixing:** The software enables artists to blend different artistic styles, seamlessly merging the realism of a photograph with the brushstrokes of Van Gogh or the abstract patterns of Kandinsky.\\n * **Interactive Storytelling:** GPT-4 Vision becomes a collaborative partner in storytelling, generating dynamic environments, characters, and plot twists in response to user input, blurring the lines between audience and creator.\\n\\n* **Addressing Challenges:**\\n * **Defining Creativity:** Exploring the philosophical and technical boundaries of AI creativity and ensuring it complements, not replaces, human artistic expression.\\n * **Copyright and Authorship:** Establishing clear guidelines for ownership and attribution when AI contributes significantly to the creative process. \\n * **Accessibility and Democratization:** Making these powerful creative tools accessible to a wide audience, fostering a more inclusive and imaginative future for art.\\n\\n\\n**11. \"Universal Translator\" for Real-Time Cross-Cultural Communication:**\\n\\n* **Features:**\\n * **Seamless Language Translation:** GPT-4 Vision goes beyond text, translating spoken language in real-time while accounting for nuances in tone, dialect, and cultural context.\\n * **Nonverbal Communication Interpretation:** The software analyzes facial expressions, gestures, and body language to bridge cultural differences in nonverbal communication, fostering greater understanding.\\n * **Cultural Sensitivity Guidance:** GPT-4 Vision provides users with real-time insights into cultural norms and customs, helping to avoid misunderstandings and promote respectful interactions.\\n\\n* **Addressing Challenges:**\\n * **Linguistic Complexity and Nuance:** Accurately translating the full richness and complexity of human language, including idioms, slang, and cultural references, remains a significant hurdle.\\n * **Bias and Stereotyping:** Ensuring the AI avoids perpetuating cultural biases or stereotypes in its translations and interpretations is crucial.\\n * **Preserving Linguistic Diversity:** Promoting language learning and cultural exchange while using technology to bridge communication gaps is essential. \\n\\n**12. \"Guardian AI\" for Personalized Safety and Well-being:**\\n\\n* **Features:**\\n * **Predictive Risk Assessment:** GPT-4 Vision analyzes real-time data from personal devices, social networks, and public safety feeds to identify potential risks or threats to an individual\\'s safety and well-being. \\n * **Proactive Safety Measures:** The AI can alert users to potential dangers, suggest alternative routes, or even contact emergency services in critical situations. \\n * **Personalized Health Monitoring:** GPT-4 Vision monitors an individual\\'s physical and mental health through wearable sensors and behavioral patterns, providing early warnings of potential health issues and suggesting preventative measures. \\n\\n* **Addressing Challenges:**\\n * **Privacy vs. Security:** Striking a delicate balance between protecting individual privacy and utilizing data to enhance safety.\\n * **Bias and Discrimination:** Ensuring the AI does not perpetuate existing societal biases or discriminate against individuals based on factors like race, gender, or socioeconomic status.\\n * **Over-Reliance and Autonomy:** Avoiding over-dependence on AI for safety and encouraging individuals to maintain their own situational awareness and decision-making abilities. \\n\\nThese futuristic applications highlight the potential of GPT-4 Vision to not only solve problems but also enhance human connection, understanding, and well-being. As we develop such powerful AI, responsible innovation, ethical considerations, and a focus on human-centered design will be paramount in shaping a future where technology empowers and uplifts all of humanity. \\n', 'name': 'Product_manager', 'role': 'user'}], summary='You\\'re right, there\\'s always more to imagine! Let\\'s keep pushing the boundaries and explore some applications that sound like they\\'re straight out of science fiction:\\n\\n**10. \"Dream Weaver\" for Artistic Co-Creation:**\\n\\n* **Features:**\\n * **Concept Visualization:** Users can input textual descriptions, rough sketches, or even their emotions, and GPT-4 Vision generates stunning visuals, musical compositions, or even short films that capture the essence of their ideas. \\n * **Style Transfer & Remixing:** The software enables artists to blend different artistic styles, seamlessly merging the realism of a photograph with the brushstrokes of Van Gogh or the abstract patterns of Kandinsky.\\n * **Interactive Storytelling:** GPT-4 Vision becomes a collaborative partner in storytelling, generating dynamic environments, characters, and plot twists in response to user input, blurring the lines between audience and creator.\\n\\n* **Addressing Challenges:**\\n * **Defining Creativity:** Exploring the philosophical and technical boundaries of AI creativity and ensuring it complements, not replaces, human artistic expression.\\n * **Copyright and Authorship:** Establishing clear guidelines for ownership and attribution when AI contributes significantly to the creative process. \\n * **Accessibility and Democratization:** Making these powerful creative tools accessible to a wide audience, fostering a more inclusive and imaginative future for art.\\n\\n\\n**11. \"Universal Translator\" for Real-Time Cross-Cultural Communication:**\\n\\n* **Features:**\\n * **Seamless Language Translation:** GPT-4 Vision goes beyond text, translating spoken language in real-time while accounting for nuances in tone, dialect, and cultural context.\\n * **Nonverbal Communication Interpretation:** The software analyzes facial expressions, gestures, and body language to bridge cultural differences in nonverbal communication, fostering greater understanding.\\n * **Cultural Sensitivity Guidance:** GPT-4 Vision provides users with real-time insights into cultural norms and customs, helping to avoid misunderstandings and promote respectful interactions.\\n\\n* **Addressing Challenges:**\\n * **Linguistic Complexity and Nuance:** Accurately translating the full richness and complexity of human language, including idioms, slang, and cultural references, remains a significant hurdle.\\n * **Bias and Stereotyping:** Ensuring the AI avoids perpetuating cultural biases or stereotypes in its translations and interpretations is crucial.\\n * **Preserving Linguistic Diversity:** Promoting language learning and cultural exchange while using technology to bridge communication gaps is essential. \\n\\n**12. \"Guardian AI\" for Personalized Safety and Well-being:**\\n\\n* **Features:**\\n * **Predictive Risk Assessment:** GPT-4 Vision analyzes real-time data from personal devices, social networks, and public safety feeds to identify potential risks or threats to an individual\\'s safety and well-being. \\n * **Proactive Safety Measures:** The AI can alert users to potential dangers, suggest alternative routes, or even contact emergency services in critical situations. \\n * **Personalized Health Monitoring:** GPT-4 Vision monitors an individual\\'s physical and mental health through wearable sensors and behavioral patterns, providing early warnings of potential health issues and suggesting preventative measures. \\n\\n* **Addressing Challenges:**\\n * **Privacy vs. Security:** Striking a delicate balance between protecting individual privacy and utilizing data to enhance safety.\\n * **Bias and Discrimination:** Ensuring the AI does not perpetuate existing societal biases or discriminate against individuals based on factors like race, gender, or socioeconomic status.\\n * **Over-Reliance and Autonomy:** Avoiding over-dependence on AI for safety and encouraging individuals to maintain their own situational awareness and decision-making abilities. \\n\\nThese futuristic applications highlight the potential of GPT-4 Vision to not only solve problems but also enhance human connection, understanding, and well-being. As we develop such powerful AI, responsible innovation, ethical considerations, and a focus on human-centered design will be paramount in shaping a future where technology empowers and uplifts all of humanity. \\n', cost={'usage_including_cached_inference': {'total_cost': 0}, 'usage_excluding_cached_inference': {'total_cost': 0}}, human_input=[])" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import autogen\n", + "\n", + "gpt_config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gpt-4\"]})\n", + "\n", + "gpt_llm_config = {\"config_list\": gpt_config_list, \"timeout\": 120}\n", + "\n", + "gemini_config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gemini\"]})\n", + "\n", + "gemini_llm_config = {\"config_list\": gemini_config_list, \"timeout\": 120}\n", + "\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"User_proxy\",\n", + " system_message=\"A human admin.\",\n", + " code_execution_config={\n", + " \"last_n_messages\": 2,\n", + " \"work_dir\": \"groupchat\",\n", + " \"use_docker\": False,\n", + " }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly.\n", + " human_input_mode=\"TERMINATE\",\n", + ")\n", + "coder = autogen.AssistantAgent(\n", + " name=\"Coder\",\n", + " llm_config=gpt_llm_config,\n", + ")\n", + "pm = autogen.AssistantAgent(\n", + " name=\"Product_manager\",\n", + " system_message=\"Creative in software product ideas.\",\n", + " llm_config=gemini_llm_config,\n", + ")\n", + "groupchat = autogen.GroupChat(agents=[user_proxy, coder, pm], messages=[], max_round=12)\n", + "manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=gpt_llm_config)\n", + "user_proxy.initiate_chat(\n", + " manager, message=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function Calling with Gemini\n", + "\n", + "Here is an example of Gemini with Function Calling," + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "Draw two agents chatting with each other with an example dialog. Don't add plt.show().\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_l7Rz8YLE4F2y8nGLCaroD6XL): python *****\u001b[0m\n", + "Arguments: \n", + "{\n", + " \"cell\": `\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Create figure and axes\n", + "fig, ax = plt.subplots()\n", + "\n", + "# Define agent coordinates\n", + "agent1_x, agent1_y = 1, 1\n", + "agent2_x, agent2_y = 3, 1\n", + "\n", + "# Draw agents as circles\n", + "agent1 = plt.Circle((agent1_x, agent1_y), 0.1, color='blue')\n", + "agent2 = plt.Circle((agent2_x, agent2_y), 0.1, color='green')\n", + "ax.add_patch(agent1)\n", + "ax.add_patch(agent2)\n", + "\n", + "# Add example dialog\n", + "dialog1 = \"Hi, how are you?\"\n", + "dialog2 = \"I'm good, thanks!\"\n", + "\n", + "# Add text labels for the dialog\n", + "plt.text(agent1_x, agent1_y + 0.3, dialog1, fontsize=12, ha='center')\n", + "plt.text(agent2_x, agent2_y + 0.3, dialog2, fontsize=12, ha='center')\n", + "\n", + "# Set limits and turn off the axes\n", + "ax.set_xlim(0, 4)\n", + "ax.set_ylim(0, 2)\n", + "ax.axis('off')\n", + "`\n", + "}\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_l7Rz8YLE4F2y8nGLCaroD6XL) *****\u001b[0m\n", + "Error: Expecting value: line 1 column 12 (char 11)\n", + " You argument should follow json format.\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_VjzGKf34UMKMlSHCrQpJ6ICK): python *****\u001b[0m\n", + "Arguments: \n", + "{\n", + " \"cell\": \"import matplotlib.pyplot as plt\\n\\n# Create figure and axes\\nfig, ax = plt.subplots()\\n\\n# Define agent coordinates\\nagent1_x, agent1_y = 1, 1\\nagent2_x, agent2_y = 3, 1\\n\\n# Draw agents as circles\\nagent1 = plt.Circle((agent1_x, agent1_y), 0.1, color='blue')\\nagent2 = plt.Circle((agent2_x, agent2_y), 0.1, color='green')\\nax.add_patch(agent1)\\nax.add_patch(agent2)\\n\\n# Add example dialog\\ndialog1 = \\\"Hi, how are you?\\\"\\ndialog2 = \\\"I'm good, thanks!\\\"\\n\\n# Add text labels for the dialog\\nplt.text(agent1_x, agent1_y + 0.3, dialog1, fontsize=12, ha='center')\\nplt.text(agent2_x, agent2_y + 0.3, dialog2, fontsize=12, ha='center')\\n\\n# Set limits and turn off the axes\\nax.set_xlim(0, 4)\\nax.set_ylim(0, 2)\\nax.axis('off')\"\n", + "}\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION python...\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "(0.0, 4.0, 0.0, 2.0)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGFCAYAAABg2vAPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgJklEQVR4nO3deXRU9d3H8c9kshGihGBIgGLYgsqmKKIgAQQUBEGQTUUhWAEVBS1aly5Cq3CktQdl0YCURdzKUhSNFmSRFBdEcAOMIItiyk4AC4Ek83v+yJMpQxIIYUIw3/frnBzNnTv33lzmN3nPnXsnHuecEwAAMCukvDcAAACUL2IAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YOAt16tRRSkpKqe7bvn17NWnSJLgbBABFaN++vdq3bx/UZW7btk0ej0d//etfg7rckvJ4PHrggQfKZd0VETHw/2bOnCmPx6M1a9YUeTu/vG346KOP1LdvX1188cWKjo5W69at9eGHH5b3ZuEMFTWeR48erTp16pTfRv1CpaWlafTo0eW9GWWq4PGybdu28t6UckMMnIWMjAxNmzatvDcDQTRgwADt27dPjzzyiJ555hnt3btXXbp00bffflvemwaUi7S0NI0ZM6a8NwNlLLS8N+CXLCIiorw34byTm5srn8+n8PDw8t6UUnnjjTd0zTXX+L+/6aabdMkll2j+/Pn63e9+V45bBgBlhyMDZ+FszhkosGHDBl1//fWKiopSrVq1NH78+ELz7N69W7/+9a8VHx+vyMhIXX755Zo1a1bAPFdeeaVuvfXWgGlNmzaVx+PRV1995Z/25ptvyuPxaOPGjcVu0/Hjx/XHP/5RV111lapUqaLKlSsrOTlZy5cvD5jvxPcMJ0yYoPr16ysiIkIbNmyQJH377bfq06ePYmNjFRkZqRYtWujtt98+5f5wzqlOnTq65ZZbCt2WnZ2tKlWqaNiwYWe0b1asWCGPx6MVK1YUuf0zZ870TzsxBCQpMjLSv09Q8RS87zx37lw1atRIlSpVUqtWrfT1119LklJTU9WgQQNFRkaqffv2JT6MvGLFCrVo0UKRkZGqX7++UlNTNXr0aHk8noD5cnNz9ec//9k/durUqaMnn3xSx44dK7TMKVOmqHHjxoqIiFDNmjU1fPhwZWVlFZpv6tSpql+/vipVqqSWLVsqPT39jPdLgZSUFE2ePFlS/r4q+CpunREREbr66qv12WefBdz+1VdfKSUlRfXq1VNkZKQSEhJ09913a9++fQHzFeyjzZs3KyUlRTExMapSpYoGDx6sI0eOnHZ7n376aYWEhGjixIn+aRMnTlTjxo0VFRWlqlWrqkWLFnrttddKszsqNI4MnOTgwYPau3dvoek5OTlBX9eBAwfUpUsX3XrrrerXr5/mzZunxx57TE2bNtVNN90kSTp69Kjat2+vzZs364EHHlDdunU1d+5cpaSkKCsrSyNHjpQkJScn6/XXX/cve//+/Vq/fr1CQkKUnp6uZs2aSZLS09MVFxenyy67rNjtOnTokF5++WXdfvvtGjJkiA4fPqzp06erc+fOWr16ta644oqA+WfMmKHs7GwNHTpUERERio2N1fr163XdddepVq1aevzxx1W5cmX94x//UM+ePTV//nz16tWryHV7PB7deeedGj9+vPbv36/Y2Fj/bYsWLdKhQ4d05513ntG+KS2fz6dRo0YpIiJCAwYMOKtl4fyVnp6ut99+W8OHD5ckjRs3TjfffLN++9vfasqUKbr//vt14MABjR8/XnfffbeWLVt2yuWtW7dOXbp0UY0aNTRmzBjl5eXpT3/6k+Li4grNe88992jWrFnq06ePRo0apU8//VTjxo3Txo0b9c9//tM/3+jRozVmzBh16tRJ9913nzIyMvTiiy/qs88+06pVqxQWFiZJmj59uoYNG6bWrVvroYce0pYtW9SjRw/Fxsaqdu3aZ7xvhg0bpszMTC1ZskSvvPJKkfO89tprOnz4sIYNGyaPx6Px48fr1ltv1ZYtW/zbtWTJEm3ZskWDBw9WQkKC1q9fr6lTp2r9+vX65JNPCgVGv379VLduXY0bN05r167Vyy+/rOrVq+vZZ58tdlt///vfa+zYsUpNTdWQIUMkSdOmTdOIESPUp08fjRw5UtnZ2frqq6/06aef6o477jjj/VGhOTjnnJsxY4aTdMqvxo0bB9wnMTHRDRo0qFTra9eunZPkZs+e7Z927Ngxl5CQ4Hr37u2fNmHCBCfJzZkzxz/t+PHjrlWrVi46OtodOnTIOefc3LlznSS3YcMG55xzb7/9touIiHA9evRw/fv399+3WbNmrlevXqfcttzcXHfs2LGAaQcOHHDx8fHu7rvv9k/bunWrk+QuvPBCt3v37oD5O3bs6Jo2beqys7P903w+n2vdurVLSko65fozMjKcJPfiiy8GTO/Ro4erU6eO8/l8Z7Rvli9f7iS55cuXByyvYPtnzJhR5HYMHTrUeTwe99prr51ye3H+KRjPn3322Snnk+QiIiLc1q1b/dNSU1OdJJeQkOB/DDnn3BNPPOEkBcxblO7du7uoqCj3008/+adt2rTJhYaGuhOfcr/44gsnyd1zzz0B93/kkUecJLds2TLnnHO7d+924eHh7sYbb3R5eXn++SZNmuQkub///e/OufzHfvXq1d0VV1wRMH6nTp3qJLl27dqdcruLM3z4cFfUr4qC8VOtWjW3f/9+//S33nrLSXKLFi3yTzty5Eih+7/++utOklu5cqV/2lNPPeUkBTzPOOdcr169XLVq1QKmSXLDhw93zjk3atQoFxIS4mbOnBkwzy233FLoebsoBY+X0/3bVmS8TXCSyZMna8mSJYW+Cl5ZB1N0dLT/Va4khYeHq2XLltqyZYt/WlpamhISEnT77bf7p4WFhWnEiBH6+eef/We6JycnS5JWrlwpKf/VztVXX60bbrjBf5gwKytL33zzjX/e4ni9Xv97/j6fT/v371dubq5atGihtWvXFpq/d+/eAa969u/fr2XLlqlfv346fPiw9u7dq71792rfvn3q3LmzNm3apJ9++qnY9Tds2FDXXHONXn311YBlvvfeexowYID/VURJ901pTJ8+XVOnTtVzzz0XsHxUPB07dgy4yqDgraLevXvrggsuKDT9xPF5sry8PH3wwQfq2bOnatas6Z/eoEED/9G+AmlpaZKk3/zmNwHTR40aJUl69913JUkffPCBjh8/roceekghIf97yh4yZIguvPBC/3xr1qzR7t27de+99wacs5OSkqIqVaqcZi+UXv/+/VW1alX/9wXPLyfup0qVKvn/Pzs7W3v37tW1114rSUU+p9x7770B3ycnJ2vfvn06dOhQwHTnnB544AE9//zzmjNnjgYNGhRwe0xMjHbs2FHobQsURgycpGXLlurUqVOhrxMf7MHyq1/9qtDhsapVq+rAgQP+77dv366kpKSAJwFJ/sP827dvlyTFx8crKSnJ/4s/PT1dycnJatu2rTIzM7VlyxatWrVKPp/vtDEgSbNmzVKzZs0UGRmpatWqKS4uTu+++64OHjxYaN66desGfL9582Y55/SHP/xBcXFxAV9PPfWUpPz3+k9l4MCBWrVqlf/nmzt3rnJycnTXXXed8b4pjVdeeUUNGzbUww8/XOpl4Jfh4osvDvi+4BfnyYfVC6afOD5Ptnv3bh09elQNGjQodNvJ07Zv366QkJBC0xMSEhQTE+N//Bb895JLLgmYLzw8XPXq1Ss0X1JSUsB8YWFhqlevXrHbfLZO3n8Fz5Un7qf9+/dr5MiRio+PV6VKlRQXF+d/3ijqOaUky5Sk2bNna/LkyZo4cWKR0f7YY48pOjpaLVu2VFJSkoYPH65Vq1aV4qes+IiBcuT1eouc7pwr1fLatGmj9PR0HT16VJ9//rmSk5PVpEkTxcTEKD09Xenp6YqOjlbz5s1PuZw5c+YoJSVF9evX1/Tp0/X+++9ryZIl6tChg3w+X6H5T6x+Sf55HnnkkSKPsixZsqTIJ8sT3XbbbQoLC/MfHZgzZ45atGhR6AmxJIo64UnKfxVXnH379qlGjRpnvC788hQ3DoM9PotT3OPzl6Ik+6lfv36aNm2a7r33Xi1YsECLFy/W+++/L0lFPqeUdN9fd911io+P16RJk7R///5C81922WXKyMjQG2+8oTZt2mj+/Plq06aN/0UJ/ocYOM8lJiZq06ZNhQZMwXXviYmJ/mnJycn64Ycf9MYbbygvL0+tW7dWSEiIPxLS09PVunXrYgdagXnz5qlevXpasGCB7rrrLnXu3FmdOnVSdnZ2iba54FVIWFhYkUdZOnXqFHD4tSixsbHq1q2bXn31VW3fvl2rVq0KOCpwJvum4FXFyWden+rIwe23317sSY5AcapXr67IyEht3ry50G0nT0tMTJTP59OmTZsCpu/atUtZWVn+x2/BfzMyMgLmO378uLZu3VpovpOXl5OTo61bt5b6ZzrbWDlw4ICWLl2qxx9/XGPGjFGvXr10ww03BOVoRYMGDbR48WJlZmaqS5cuOnz4cKF5KleurP79+2vGjBn64Ycf1K1bNz3zzDMBz2cpKSn+K5msIgbOc127dtXOnTv15ptv+qfl5uZq4sSJio6OVrt27fzTCw7/P/vss2rWrJn/sGZycrKWLl2qNWvWlOgtgoJYOLHCP/30U3388ccl2ubq1aurffv2Sk1N1X/+859Ct+/Zs6dEy7nrrru0YcMGPfroo/J6vbrtttsCbi/pvklMTJTX6/WfT1FgypQpxa67f//+xADOmNfrVadOnbRw4UJlZmb6p2/evFnvvfdewLxdu3aVJE2YMCFg+t/+9jdJUrdu3SRJnTp1Unh4uF544YWAMTl9+nQdPHjQP1+LFi0UFxenl156KeBS2JkzZxZ5CWJJVa5cWVLhmC6pop5PpMI/d2k1a9ZMaWlp2rhxo7p3766jR4/6bzv50sXw8HA1atRIzrmAK8T27t2rb7/9tkyuGvul4NLCICsoy2B9rOXQoUOVmpqqlJQUff7556pTp47mzZunVatWacKECQGvsBs0aKCEhARlZGTowQcf9E9v27atHnvsMUkqUQzcfPPNWrBggXr16qVu3bpp69ateumll9SoUSP9/PPPJdruyZMnq02bNmratKmGDBmievXqadeuXfr444+1Y8cOffnll6ddRrdu3VStWjXNnTtXN910k6pXr16qfVOlShX17dtXEydOlMfjUf369fXOO++c8ryFgpPKTv5sAuB0Ro8ercWLF+u6667Tfffdp7y8PE2aNElNmjTRF1984Z/v8ssv16BBgzR16lRlZWWpXbt2Wr16tWbNmqWePXvq+uuvlyTFxcXpiSee0JgxY9SlSxf16NFDGRkZmjJliq6++mr/SchhYWF6+umnNWzYMHXo0EH9+/fX1q1bNWPGjCJfhbdv314ffvjhad/2uOqqqyRJI0aMUOfOnYsM81O58MIL1bZtW40fP145OTmqVauWFi9efFZHK0527bXX6q233lLXrl3Vp08fLVy4UGFhYbrxxhuVkJDgfzth48aNmjRpkrp16xbw3Dlp0iSNGTNGW7dutXt0oHwuYjj/nO5SpHbt2pXo0sKLLrrIXXvttaddX1HLc865QYMGucTExIBpu3btcoMHD3YXXXSRCw8Pd02bNi32cri+ffs6Se7NN9/0Tzt+/LiLiopy4eHh7ujRo6fdNp/P58aOHesSExNdRESEa968uXvnnXcKbVvBpUV/+ctfilzO999/7wYOHOgSEhJcWFiYq1Wrlrv55pvdvHnzTrsNBe6//34nqdjL+0q6b/bs2eN69+7toqKiXNWqVd2wYcPcN998U+ylhYmJiaW+FAvl70wuLSy4PK1AcY/rgktU586de9r1L1261DVv3tyFh4e7+vXru5dfftmNGjXKRUZGBsyXk5PjxowZ4+rWrevCwsJc7dq13RNPPBFwSW6BSZMmuUsvvdSFhYW5+Ph4d99997kDBw4Umm/KlCmubt26LiIiwrVo0cKtXLnStWvXrtDj+aqrrnIJCQmn/Vlyc3Pdgw8+6OLi4pzH4/FfZniq8S/JPfXUU/7vd+zY4Xr16uViYmJclSpVXN++fV1mZmah+QouLdyzZ0/A8oq69K+of7u33nrLhYaGuv79+7u8vDyXmprq2rZt66pVq+YiIiJc/fr13aOPPuoOHjwYcL+C9Vq+tNDjXJDPhjFsw4YNaty4sd555x3/oTucnYcffljTp0/Xzp07FRUVVd6bA5Raz549tX79+kLv6ZeHw4cPKzY2VhMmTPB/2BJs45yBIFq+fLlatWpFCARJdna25syZo969exMC+EU58X1rKf+kvrS0tKD/GeHSWrlypWrVquX/pD6AIwM47+zevVsffPCB5s2bp4ULF2rt2rWFPgIZOJ/VqFHD/1n827dv14svvqhjx45p3bp1hT4HADgfcAIhzjsbNmzQgAEDVL16db3wwguEAH5xunTpotdff107d+5URESEWrVqpbFjxxICOG9xZAAAAOM4ZwAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwLrS8NwDlz+eTli6VFi3K/9q5M396QoLUvXv+V8eOUgjpCPzi+JxPS7cs1aLvFmnRd4u08+f8AZ4QnaDuDbure8Pu6livo0I8DHDLPM45V94bgfLz5ZfS0KHS6tVSaKiUmxt4e8G0li2ladOkZs3KZzsBnLkvd36poYuGanXmaoWGhCrXFzjAC6a1rNVS07pPU7N4BrhVpKBhzz0nXXmltHZt/vcnh8CJ09aulZo3z78PgPPfcx89pyunXqm1O/MH+MkhcOK0tZlr1Ty1uZ77iAFuFUcGjBo3TnryydLf9/HHg7s9AIJnXPo4PbmsdAN8XMdxerwNA9waYsCgF16QRo48u2U8/7w0YkRwtgdA8Lzw6Qsa+f7ZDfDnuzyvEdcwwC0hBoxZty7//f+i3hI4E6Gh+ecZNG8enO0CcPbW/WedWr7cssi3BM5EaEioVt+zWs1rMMCt4JwBQ3JzpYEDpWDkn3PSoEFnHxUAgiPXl6uBCwcqGK/vnHMatHDQWUcFfjmIAUNmz5a++UbKyzv7ZeXlSV9/nb9MAOVv9pez9c3ub5Tnzn6A57k8fb37a83+kgFuBW8TGOGcdOml0qZNwTkyIEkej9SwobRxY/7/AygfzjldOvlSbdq3SU7BGeAeedSwWkNtHL5RHgZ4hceRASP+9S/pu++CFwJS/rIyMqTFi4O3TABn7l/f/0vf7fsuaCEgSU5OGfsytPh7BrgFxIARqamS1xv85Xq90ksvBX+5AEoudU2qvJ7gD3Cvx6uX1jDALeBtAgNycqSYGOnIkbJZflSUlJUlhYWVzfIBFC8nL0cxz8boSE7ZDPCosChlPZalMC8DvCLjyIABH39cdiEg5S/7k0/KbvkAivfxjo/LLAQk6UjOEX2ygwFe0REDBixZkv+5AGUlNJTzBoDysuT7JQoNKbsBHhoSynkDBhADBqxfH5zLCYvj80kbNpTd8gEUb/2e9crzld0A9zmfNuxhgFd0xIABGRnBvYrgZD5f/joAnHsZ+zKCehXByXzOp4x9DPCKjhio4JyTtm0r+/Vs21a2wQGgMOectmVtK/P1bMvaFpRPNsT5ixio4A4fLtuTBwv897/Szz+X/XoA/M/h44fL9OTBAv/N+a9+Ps4Ar8iIgQru2LGKuS4A0rHcczfojuUxwCsyYqCC8/kq5roA5L+fXxHXhXOPGKjgwsMr5roASOHeczfozuW6cO4RAxXcBRdIIefgX9nrlaKjy349AP7ngogLFOIp+wHu9XgVHc4Ar8iIgQouNFSqVavs11OzZtl+sBGAwkJDQlXrgrIf4DUvqFmmH2yE8kcMGJCUVPbraNiw7NcBoLCkamU/wBtWY4BXdMSAAUlJZftHhMLCzk1wACgsKTZJYSFlN8DDQsKUFMsAr+iIAQOSk/P/cmFZycnJXweAcy/54mTl+MpugOf4cpScyACv6PgTxgbs2SPFx5fdJwR6PNKuXVJcXNksH0Dx9vx3j+L/Gl9mH0nskUe7HtmluMoM8IqMIwMGxMVJzZqV3fIvv5wQAMpLXOU4NYsvuwF+ecLlhIABxIARAwaUzSWGHo90xx3BXy6AkhvQdECZXGLokUd3NGGAW8DbBEZkZUk1akjZ2cFdbqVKUmamFBMT3OUCKLms7CzVeK6GsnODO8ArhVZS5qhMxUTGBHW5OP9wZMCImBhp6ND8DwcKFq83f5mEAFC+YiJjNPTKofJ6gjfAvR6vhl41lBAwgiMDhvz4Y/7nAQTr6EBkpPTdd1Lt2sFZHoDS+/Hgj2o4qWHQjg5Ehkbquwe+U+0qDHALODJgSO3a0tixwVveuHGEAHC+qF2ltsZ2CN4AH9dxHCFgCEcGjPH5pA4dpH//W8rLK90yvN78zxVYuvTc/N0DACXjcz51mNVB//7h38pzpRvgXo9XyYnJWjpw6Tn5uwc4P/AvbUxIiLRggdSoUenOH/B6pSZNpPnzCQHgfBPiCdGC/gvUKK5Rqc4f8Hq8alK9ieb3m08IGMO/tkGxsdKKFfm/1M/kF3pISP59li3LXwaA809spVitSFmhJtWbnNEv9BBPiJpUb6Jlg5YpthID3BpiwKjYWCk9XRoxIv+zAk51lMDrzZ9nxIj8+xACwPkttlKs0gena0TLEfLIc8qjBF6PVx55NKLlCKUPTicEjOKcAWjduvwTC9PSpCNHAm+LipK6dpWefFJq3rx8tg9A6a37zzqNTR+rtM1pOpITOMCjwqLUtUFXPZn8pJrXYIBbRgzA7/hx6aOP8j9ESJJq1pRat5bCw8t3uwCcveN5x/XRjx8p83D+AK95QU21rt1a4V4GOIgBAADM45wBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADDu/wC+Ka8LXmqY8wAAAABJRU5ErkJggg==", + "text/plain": [ + "

" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_VjzGKf34UMKMlSHCrQpJ6ICK) *****\u001b[0m\n", + "(0.0, 4.0, 0.0, 2.0)\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "from IPython import get_ipython\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "from autogen.cache import Cache\n", + "\n", + "config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gemini\", \"tool\"]})\n", + "\n", + "llm_config = {\n", + " \"config_list\": config_list,\n", + " \"timeout\": 120,\n", + "}\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"For coding tasks, only use the functions you have been provided with. Reply TERMINATE when the task is done.\",\n", + " llm_config=llm_config,\n", + ")\n", + "\n", + "# create a UserProxyAgent instance named \"user_proxy\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and x.get(\"content\", \"\").rstrip().endswith(\"TERMINATE\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=10,\n", + " code_execution_config={\n", + " \"work_dir\": \"coding\",\n", + " \"use_docker\": False,\n", + " }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly.\n", + ")\n", + "\n", + "\n", + "# define functions according to the function description\n", + "\n", + "\n", + "# one way of registering functions is to use the register_for_llm and register_for_execution decorators\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(name=\"python\", description=\"run cell in ipython and return the execution result.\")\n", + "def exec_python(cell: Annotated[str, \"Valid Python cell to execute.\"]) -> str:\n", + " ipython = get_ipython()\n", + " result = ipython.run_cell(cell)\n", + " log = str(result.result)\n", + " if result.error_before_exec is not None:\n", + " log += f\"\\n{result.error_before_exec}\"\n", + " if result.error_in_exec is not None:\n", + " log += f\"\\n{result.error_in_exec}\"\n", + " return log\n", + "\n", + "\n", + "# another way of registering functions is to use the register_function\n", + "def exec_sh(script: Annotated[str, \"Valid Python cell to execute.\"]) -> str:\n", + " return user_proxy.execute_code_blocks([(\"sh\", script)])\n", + "\n", + "\n", + "autogen.agentchat.register_function(\n", + " exec_python,\n", + " caller=chatbot,\n", + " executor=user_proxy,\n", + " name=\"sh\",\n", + " description=\"run a shell script and return the execution result.\",\n", + ")\n", + "\n", + "with Cache.disk() as cache:\n", + " # start the conversation\n", + " user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"Draw two agents chatting with each other with an example dialog. Don't add plt.show().\",\n", + " cache=cache,\n", + " max_turns=3,\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebook/agentchat_huggingface_langchain.ipynb b/notebook/agentchat_huggingface_langchain.ipynb new file mode 100644 index 00000000000..248ad18f19c --- /dev/null +++ b/notebook/agentchat_huggingface_langchain.ipynb @@ -0,0 +1,866 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Using AutoGen AgentChat with LangChain-based Custom Client and Hugging Face Models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "This notebook demonstrates how you can use LangChain's extensive support for LLMs to enable flexible use of various Language Models (LLMs) in agent-based conversations in AutoGen.\n", + "\n", + "What we'll cover:\n", + "\n", + "1. Creating a custom model client that uses LangChain to load and interact with LLMs\n", + "2. Configuring AutoGen to use our custom LangChain-based model\n", + "3. Setting up AutoGen agents with the custom model\n", + "4. Demonstrating a simple conversation using this setup\n", + "\n", + "While we used a Hugging Face model in this example, the same approach can be applied to any LLM supported by LangChain, including models from OpenAI, Anthropic, or custom models. This integration opens up a wide range of possibilities for creating sophisticated, multi-model conversational agents using AutoGen\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Requirements\n", + "\n", + "````{=mdx}\n", + ":::info Requirements\n", + "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", + "\n", + "```bash\n", + "pip install pyautogen torch transformers sentencepiece langchain-huggingface \n", + "```\n", + "\n", + "For more information, please refer to the [installation guide](/docs/installation/).\n", + ":::\n", + "````\n", + "\n", + "**NOTE: Depending on what model you use, you may need to play with the default prompts of the Agent's**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup and Imports\n", + "\n", + "First, let's import the necessary libraries and define our custom model client." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from types import SimpleNamespace\n", + "\n", + "from langchain_core.messages import AIMessage, HumanMessage, SystemMessage\n", + "from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent, config_list_from_json" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create and configure the custom model\n", + "\n", + "A custom model class can be created in many ways, but needs to adhere to the `ModelClient` protocol and response structure which is defined in client.py and shown below.\n", + "\n", + "The response protocol has some minimum requirements, but can be extended to include any additional information that is needed.\n", + "Message retrieval therefore can be customized, but needs to return a list of strings or a list of `ModelClientResponseProtocol.Choice.Message` objects.\n", + "\n", + "\n", + "```python\n", + "class ModelClient(Protocol):\n", + " \"\"\"\n", + " A client class must implement the following methods:\n", + " - create must return a response object that implements the ModelClientResponseProtocol\n", + " - cost must return the cost of the response\n", + " - get_usage must return a dict with the following keys:\n", + " - prompt_tokens\n", + " - completion_tokens\n", + " - total_tokens\n", + " - cost\n", + " - model\n", + "\n", + " This class is used to create a client that can be used by OpenAIWrapper.\n", + " The response returned from create must adhere to the ModelClientResponseProtocol but can be extended however needed.\n", + " The message_retrieval method must be implemented to return a list of str or a list of messages from the response.\n", + " \"\"\"\n", + "\n", + " RESPONSE_USAGE_KEYS = [\"prompt_tokens\", \"completion_tokens\", \"total_tokens\", \"cost\", \"model\"]\n", + "\n", + " class ModelClientResponseProtocol(Protocol):\n", + " class Choice(Protocol):\n", + " class Message(Protocol):\n", + " content: Optional[str]\n", + "\n", + " message: Message\n", + "\n", + " choices: List[Choice]\n", + " model: str\n", + "\n", + " def create(self, params) -> ModelClientResponseProtocol:\n", + " ...\n", + "\n", + " def message_retrieval(\n", + " self, response: ModelClientResponseProtocol\n", + " ) -> Union[List[str], List[ModelClient.ModelClientResponseProtocol.Choice.Message]]:\n", + " \"\"\"\n", + " Retrieve and return a list of strings or a list of Choice.Message from the response.\n", + "\n", + " NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object,\n", + " since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used.\n", + " \"\"\"\n", + " ...\n", + "\n", + " def cost(self, response: ModelClientResponseProtocol) -> float:\n", + " ...\n", + "\n", + " @staticmethod\n", + " def get_usage(response: ModelClientResponseProtocol) -> Dict:\n", + " \"\"\"Return usage summary of the response using RESPONSE_USAGE_KEYS.\"\"\"\n", + " ...\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example of simple custom client\n", + "\n", + "Following the huggingface example for using [Mistral's Open-Orca](https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca)\n", + "\n", + "For the response object, python's `SimpleNamespace` is used to create a simple object that can be used to store the response data, but any object that follows the `ClientResponseProtocol` can be used.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# custom client with custom model loader\n", + "\n", + "\n", + "class CustomModelClient:\n", + " \"\"\"Custom model client implementation for LangChain integration with AutoGen.\"\"\"\n", + "\n", + " def __init__(self, config, **kwargs):\n", + " \"\"\"Initialize the CustomModelClient.\"\"\"\n", + " print(f\"CustomModelClient config: {config}\")\n", + " self.device = config.get(\"device\", \"cpu\")\n", + "\n", + " gen_config_params = config.get(\"params\", {})\n", + " self.model_name = config[\"model\"]\n", + " pipeline = HuggingFacePipeline.from_model_id(\n", + " model_id=self.model_name,\n", + " task=\"text-generation\",\n", + " pipeline_kwargs=gen_config_params,\n", + " device=self.device,\n", + " )\n", + " self.model = ChatHuggingFace(llm=pipeline)\n", + " print(f\"Loaded model {config['model']} to {self.device}\")\n", + "\n", + " def _to_chatml_format(self, message):\n", + " \"\"\"Convert message to ChatML format.\"\"\"\n", + " if message[\"role\"] == \"system\":\n", + " return SystemMessage(content=message[\"content\"])\n", + " if message[\"role\"] == \"assistant\":\n", + " return AIMessage(content=message[\"content\"])\n", + " if message[\"role\"] == \"user\":\n", + " return HumanMessage(content=message[\"content\"])\n", + " raise ValueError(f\"Unknown message type: {type(message)}\")\n", + "\n", + " def create(self, params):\n", + " \"\"\"Create a response using the model.\"\"\"\n", + " if params.get(\"stream\", False) and \"messages\" in params:\n", + " raise NotImplementedError(\"Local models do not support streaming.\")\n", + "\n", + " num_of_responses = params.get(\"n\", 1)\n", + " response = SimpleNamespace()\n", + " inputs = [self._to_chatml_format(m) for m in params[\"messages\"]]\n", + " response.choices = []\n", + " response.model = self.model_name\n", + "\n", + " for _ in range(num_of_responses):\n", + " outputs = self.model.invoke(inputs)\n", + " text = outputs.content\n", + " choice = SimpleNamespace()\n", + " choice.message = SimpleNamespace()\n", + " choice.message.content = text\n", + " choice.message.function_call = None\n", + " response.choices.append(choice)\n", + "\n", + " return response\n", + "\n", + " def message_retrieval(self, response):\n", + " \"\"\"Retrieve messages from the response.\"\"\"\n", + " return [choice.message.content for choice in response.choices]\n", + "\n", + " def cost(self, response) -> float:\n", + " \"\"\"Calculate the cost of the response.\"\"\"\n", + " response.cost = 0\n", + " return 0\n", + "\n", + " @staticmethod\n", + " def get_usage(response):\n", + " \"\"\"Get usage statistics.\"\"\"\n", + " return {}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set your API Endpoint\n", + "\n", + "The [`config_list_from_json`](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils#config_list_from_json) function loads a list of configurations from an environment variable or a json file.\n", + "\n", + "It first looks for an environment variable of a specified name (\"OAI_CONFIG_LIST\" in this example), which needs to be a valid json string. If that variable is not found, it looks for a json file with the same name. It filters the configs by models (you can filter by other keys as well).\n", + "\n", + "The json looks like the following:\n", + "```json\n", + "[\n", + " {\n", + " \"model\": \"gpt-4\",\n", + " \"api_key\": \"\"\n", + " },\n", + " {\n", + " \"model\": \"gpt-4\",\n", + " \"api_key\": \"\",\n", + " \"base_url\": \"\",\n", + " \"api_type\": \"azure\",\n", + " \"api_version\": \"2024-02-01\"\n", + " },\n", + " {\n", + " \"model\": \"gpt-4-32k\",\n", + " \"api_key\": \"\",\n", + " \"base_url\": \"\",\n", + " \"api_type\": \"azure\",\n", + " \"api_version\": \"2024-02-01\"\n", + " }\n", + "]\n", + "```\n", + "\n", + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set the config for the custom model\n", + "\n", + "You can add any paramteres that are needed for the custom model loading in the same configuration list.\n", + "\n", + "It is important to add the `model_client_cls` field and set it to a string that corresponds to the class name: `\"CustomModelClient\"`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"OAI_CONFIG_LIST\"] = json.dumps(\n", + " [\n", + " {\n", + " \"model\": \"mistralai/Mistral-7B-Instruct-v0.2\",\n", + " \"model_client_cls\": \"CustomModelClient\",\n", + " \"device\": 0,\n", + " \"n\": 1,\n", + " \"params\": {\n", + " \"max_new_tokens\": 500,\n", + " \"top_k\": 50,\n", + " \"temperature\": 0.1,\n", + " \"do_sample\": True,\n", + " \"return_full_text\": False,\n", + " },\n", + " }\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "config_list_custom = config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\"model_client_cls\": [\"CustomModelClient\"]},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "from huggingface_hub import login\n", + "\n", + "# The Mistral-7B-Instruct-v0.2 is a gated model which requires API token to access\n", + "login(token=getpass.getpass(\"Enter your HuggingFace API Token\"))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Construct Agents\n", + "\n", + "Consturct a simple conversation between a User proxy and an Assistent agent" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[autogen.oai.client: 09-01 12:53:51] {484} INFO - Detected custom model client in config: CustomModelClient, model client can not be used until register_model_client is called.\n" + ] + } + ], + "source": [ + "assistant = AssistantAgent(\"assistant\", llm_config={\"config_list\": config_list_custom})\n", + "user_proxy = UserProxyAgent(\"user_proxy\", code_execution_config=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Register the custom client class to the assistant agent" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CustomModelClient config: {'model': 'microsoft/Phi-3.5-mini-instruct', 'model_client_cls': 'CustomModelClient', 'device': 0, 'n': 1, 'params': {'max_new_tokens': 100, 'top_k': 50, 'temperature': 0.1, 'do_sample': True, 'return_full_text': False}}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:07<00:00, 3.51s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded model microsoft/Phi-3.5-mini-instruct to 0\n" + ] + } + ], + "source": [ + "assistant.register_model_client(model_client_cls=CustomModelClient)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\n", + "Write python code to print Hello World!\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "You are not running the flash-attention implementation, expect numerical differences.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\n", + " ```python\n", + "# filename: hello_world.py\n", + "\n", + "print(\"Hello World!\")\n", + "```\n", + "\n", + "To execute this code, save it in a file named `hello_world.py`. Then, open your terminal or command prompt, navigate to the directory containing the file, and run the following command:\n", + "\n", + "```\n", + "python hello_world.py\n", + "```\n", + "\n", + "The output should be:\n", + "\n", + "```\n", + "Hello World!\n", + "```\n", + "\n", + "If you encounter any errors,\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "ChatResult(chat_id=None, chat_history=[{'content': 'Write python code to print Hello World!', 'role': 'assistant', 'name': 'user_proxy'}, {'content': ' ```python\\n# filename: hello_world.py\\n\\nprint(\"Hello World!\")\\n```\\n\\nTo execute this code, save it in a file named `hello_world.py`. Then, open your terminal or command prompt, navigate to the directory containing the file, and run the following command:\\n\\n```\\npython hello_world.py\\n```\\n\\nThe output should be:\\n\\n```\\nHello World!\\n```\\n\\nIf you encounter any errors,', 'role': 'user', 'name': 'assistant'}], summary=' ```python\\n# filename: hello_world.py\\n\\nprint(\"Hello World!\")\\n```\\n\\nTo execute this code, save it in a file named `hello_world.py`. Then, open your terminal or command prompt, navigate to the directory containing the file, and run the following command:\\n\\n```\\npython hello_world.py\\n```\\n\\nThe output should be:\\n\\n```\\nHello World!\\n```\\n\\nIf you encounter any errors,', cost={'usage_including_cached_inference': {'total_cost': 0}, 'usage_excluding_cached_inference': {'total_cost': 0}}, human_input=['exit'])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "user_proxy.initiate_chat(assistant, message=\"Write python code to print Hello World!\")" + ] + } + ], + "metadata": { + "front_matter": { + "description": "Define and laod a custom model", + "tags": [ + "custom model" + ] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "vscode": { + "interpreter": { + "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1" + } + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2d910cfd2d2a4fc49fc30fbbdc5576a7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "454146d0f7224f038689031002906e6f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e4ae2b6f5a974fd4bafb6abb9d12ff26", + "IPY_MODEL_577e1e3cc4db4942b0883577b3b52755", + "IPY_MODEL_b40bdfb1ac1d4cffb7cefcb870c64d45" + ], + "layout": "IPY_MODEL_dc83c7bff2f241309537a8119dfc7555", + "tabbable": null, + "tooltip": null + } + }, + "577e1e3cc4db4942b0883577b3b52755": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2d910cfd2d2a4fc49fc30fbbdc5576a7", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_74a6ba0c3cbc4051be0a83e152fe1e62", + "tabbable": null, + "tooltip": null, + "value": 1 + } + }, + "6086462a12d54bafa59d3c4566f06cb2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "74a6ba0c3cbc4051be0a83e152fe1e62": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7d3f3d9e15894d05a4d188ff4f466554": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "b40bdfb1ac1d4cffb7cefcb870c64d45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f1355871cc6f4dd4b50d9df5af20e5c8", + "placeholder": "​", + "style": "IPY_MODEL_ca245376fd9f4354af6b2befe4af4466", + "tabbable": null, + "tooltip": null, + "value": " 1/1 [00:00<00:00, 44.69it/s]" + } + }, + "ca245376fd9f4354af6b2befe4af4466": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "dc83c7bff2f241309537a8119dfc7555": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e4ae2b6f5a974fd4bafb6abb9d12ff26": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_6086462a12d54bafa59d3c4566f06cb2", + "placeholder": "​", + "style": "IPY_MODEL_7d3f3d9e15894d05a4d188ff4f466554", + "tabbable": null, + "tooltip": null, + "value": "100%" + } + }, + "f1355871cc6f4dd4b50d9df5af20e5c8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebook/agentchat_human_feedback.ipynb b/notebook/agentchat_human_feedback.ipynb index 3c21c7c9f4f..77b09fe0e8d 100644 --- a/notebook/agentchat_human_feedback.ipynb +++ b/notebook/agentchat_human_feedback.ipynb @@ -102,7 +102,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_langchain.ipynb b/notebook/agentchat_langchain.ipynb index 1d5fea19e06..bb4805cb355 100644 --- a/notebook/agentchat_langchain.ipynb +++ b/notebook/agentchat_langchain.ipynb @@ -139,7 +139,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_oai_assistant_retrieval.ipynb b/notebook/agentchat_oai_assistant_retrieval.ipynb index ec7e41a00ba..fec9605161a 100644 --- a/notebook/agentchat_oai_assistant_retrieval.ipynb +++ b/notebook/agentchat_oai_assistant_retrieval.ipynb @@ -6,7 +6,7 @@ "source": [ "## RAG OpenAI Assistants in AutoGen\n", "\n", - "This notebook shows an example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43) with retrieval augmented generation. `GPTAssistantAgent` is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", + "This notebook shows an example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43) with retrieval augmented generation. `GPTAssistantAgent` is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", "`UserProxyAgent` in AutoGen." ] }, diff --git a/notebook/agentchat_oai_assistant_twoagents_basic.ipynb b/notebook/agentchat_oai_assistant_twoagents_basic.ipynb index 94da2358c98..295784bea2c 100644 --- a/notebook/agentchat_oai_assistant_twoagents_basic.ipynb +++ b/notebook/agentchat_oai_assistant_twoagents_basic.ipynb @@ -6,7 +6,7 @@ "source": [ "## OpenAI Assistants in AutoGen\n", "\n", - "This notebook shows a very basic example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43), which is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", + "This notebook shows a very basic example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43), which is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", "`UserProxyAgent` in AutoGen." ] }, diff --git a/notebook/agentchat_planning.ipynb b/notebook/agentchat_planning.ipynb index 9953bf3b57f..14fed1c29e0 100644 --- a/notebook/agentchat_planning.ipynb +++ b/notebook/agentchat_planning.ipynb @@ -105,7 +105,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n", + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n", "\n", "## Construct Agents\n", "\n", diff --git a/notebook/agentchat_stream.ipynb b/notebook/agentchat_stream.ipynb index 5536cef96aa..d3989582727 100644 --- a/notebook/agentchat_stream.ipynb +++ b/notebook/agentchat_stream.ipynb @@ -102,7 +102,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_surfer.ipynb b/notebook/agentchat_surfer.ipynb index 09c9cb3d4f8..18be350a41f 100644 --- a/notebook/agentchat_surfer.ipynb +++ b/notebook/agentchat_surfer.ipynb @@ -479,7 +479,7 @@ "#### Multi-Agent Conversation Framework[​](#multi-agent-conversation-framework \"Direct link to Multi-Agent Conversation Framework\")\n", "\n", "Autogen enables the next-gen LLM applications with a generic multi-agent conversation framework. It offers customizable and conversable agents which integrate LLMs, tools, and humans.\n", - "By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py),\n", + "By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py),\n", "\n", "The figure below shows an example conversation flow with AutoGen.\n", "\n", diff --git a/notebook/agentchat_teachability.ipynb b/notebook/agentchat_teachability.ipynb index 4be1e135dab..dd41727a02e 100644 --- a/notebook/agentchat_teachability.ipynb +++ b/notebook/agentchat_teachability.ipynb @@ -13,7 +13,7 @@ "\n", "In making decisions about memo storage and retrieval, `Teachability` calls an instance of `TextAnalyzerAgent` to analyze pieces of text in several different ways. This adds extra LLM calls involving a relatively small number of tokens. These calls can add a few seconds to the time a user waits for a response.\n", "\n", - "This notebook demonstrates how `Teachability` can be added to an agent so that it can learn facts, preferences, and skills from users. To chat with a teachable agent yourself, run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/main/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py).\n", + "This notebook demonstrates how `Teachability` can be added to an agent so that it can learn facts, preferences, and skills from users. To chat with a teachable agent yourself, run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/0.2/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py).\n", "\n", "## Requirements\n", "\n", diff --git a/notebook/agentchat_teachable_oai_assistants.ipynb b/notebook/agentchat_teachable_oai_assistants.ipynb index 75744d9a397..be37b0d7b5c 100644 --- a/notebook/agentchat_teachable_oai_assistants.ipynb +++ b/notebook/agentchat_teachable_oai_assistants.ipynb @@ -860,7 +860,7 @@ "metadata": {}, "source": [ "### Create the OpenAI Assistant with retrieval as a tool\n", - "For this example, first upload the [conversable_agent.py](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/conversable_agent.py) file to your OpenAI API account. This can be done manually through the [website](https://platform.openai.com/assistants). Then find the uploaded File ID on the [Files page](https://platform.openai.com/files), and paste that ID into the `file_ids` list in the code below." + "For this example, first upload the [conversable_agent.py](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/conversable_agent.py) file to your OpenAI API account. This can be done manually through the [website](https://platform.openai.com/assistants). Then find the uploaded File ID on the [Files page](https://platform.openai.com/files), and paste that ID into the `file_ids` list in the code below." ] }, { diff --git a/notebook/agentchat_two_users.ipynb b/notebook/agentchat_two_users.ipynb index 14b95b53701..c4fc85861f5 100644 --- a/notebook/agentchat_two_users.ipynb +++ b/notebook/agentchat_two_users.ipynb @@ -82,7 +82,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_video_transcript_translate_with_whisper.ipynb b/notebook/agentchat_video_transcript_translate_with_whisper.ipynb index e19e61419ad..51ad4c6268a 100644 --- a/notebook/agentchat_video_transcript_translate_with_whisper.ipynb +++ b/notebook/agentchat_video_transcript_translate_with_whisper.ipynb @@ -8,7 +8,7 @@ "# Translating Video audio using Whisper and GPT-3.5-turbo\n", "\n", "In this notebook, we demonstrate how to use whisper and GPT-3.5-turbo with `AssistantAgent` and `UserProxyAgent` to recognize and translate\n", - "the speech sound from a video file and add the timestamp like a subtitle file based on [agentchat_function_call.ipynb](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_function_call.ipynb)\n" + "the speech sound from a video file and add the timestamp like a subtitle file based on [agentchat_function_call.ipynb](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_function_call.ipynb)\n" ] }, { diff --git a/notebook/agentchat_web_info.ipynb b/notebook/agentchat_web_info.ipynb index e2962e9049f..47660502fa7 100644 --- a/notebook/agentchat_web_info.ipynb +++ b/notebook/agentchat_web_info.ipynb @@ -116,7 +116,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agenteval_cq_math.ipynb b/notebook/agenteval_cq_math.ipynb index 199967a9108..a67b0ae992c 100644 --- a/notebook/agenteval_cq_math.ipynb +++ b/notebook/agenteval_cq_math.ipynb @@ -65,7 +65,7 @@ "## Set your API Endpoint\n", "* The [`config_list_from_json`](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils#config_list_from_json) function loads a list of configurations from an environment variable or a json file. It first looks for an environment variable with a specified name. The value of the environment variable needs to be a valid json string. If that variable is not found, it looks for a json file with the same name. It filters the configs by filter_dict.\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n" + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n" ] }, { diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index 76ebf52e4bb..202724230b5 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -4,11 +4,11 @@ import io import json import logging +from types import SimpleNamespace from typing import Any, Dict, List, Optional -from unittest import TestCase, mock +from unittest import mock import pytest -from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST import autogen from autogen import Agent, AssistantAgent, GroupChat, GroupChatManager @@ -2062,6 +2062,60 @@ def test_manager_resume_messages(): return_agent, return_message = manager.resume(messages="Let's get this conversation started.") +def test_custom_model_client(): + class CustomModelClient: + def __init__(self, config, **kwargs): + print(f"CustomModelClient config: {config}") + + def create(self, params): + num_of_responses = params.get("n", 1) + + response = SimpleNamespace() + response.choices = [] + response.model = "test_model_name" + + for _ in range(num_of_responses): + text = "this is a dummy text response" + choice = SimpleNamespace() + choice.message = SimpleNamespace() + choice.message.content = text + choice.message.function_call = None + response.choices.append(choice) + return response + + def message_retrieval(self, response): + choices = response.choices + return [choice.message.content for choice in choices] + + def cost(self, response) -> float: + response.cost = 0 + return 0 + + @staticmethod + def get_usage(response): + return {} + + llm_config = {"config_list": [{"model": "test_model_name", "model_client_cls": "CustomModelClient"}]} + + group_chat = autogen.GroupChat( + agents=[], + messages=[], + max_round=3, + select_speaker_auto_llm_config=llm_config, + select_speaker_auto_model_client_cls=CustomModelClient, + ) + + checking_agent, speaker_selection_agent = group_chat._create_internal_agents( + agents=[], messages=[], max_attempts=3, validate_speaker_name=(True, "test") + ) + + # Check that the custom model client is assigned to the speaker selection agent + assert isinstance(speaker_selection_agent.client._clients[0], CustomModelClient) + + # Check that the LLM Config is assigned + assert speaker_selection_agent.client._config_list == llm_config["config_list"] + + def test_select_speaker_transform_messages(): """Tests adding transform messages to a GroupChat for speaker selection when in 'auto' mode""" @@ -2132,5 +2186,6 @@ def test_select_speaker_transform_messages(): # test_manager_resume_functions() # test_manager_resume_returns() # test_manager_resume_messages() + # test_custom_model_client() # test_select_speaker_transform_messages() pass diff --git a/test/oai/test_gemini.py b/test/oai/test_gemini.py index 61fdbe6d735..e6ff3e02672 100644 --- a/test/oai/test_gemini.py +++ b/test/oai/test_gemini.py @@ -1,3 +1,4 @@ +import json import os from unittest.mock import MagicMock, patch @@ -10,7 +11,9 @@ from google.cloud.aiplatform.initializer import global_config as vertexai_global_config from vertexai.generative_models import HarmBlockThreshold as VertexAIHarmBlockThreshold from vertexai.generative_models import HarmCategory as VertexAIHarmCategory + from vertexai.generative_models import Part as VertexAIPart from vertexai.generative_models import SafetySetting as VertexAISafetySetting + from vertexai.generative_models import ToolConfig as VertexAIToolConfig from autogen.oai.gemini import GeminiClient @@ -20,6 +23,8 @@ VertexAIHarmBlockThreshold = object VertexAIHarmCategory = object VertexAISafetySetting = object + VertexAIPart = object + VertexAIToolConfig = object vertexai_global_config = object InternalServerError = object skip = True @@ -234,8 +239,6 @@ def test_vertexai_safety_setting_list(gemini_client): for category in harm_categories ] - print(safety_settings) - converted_safety_settings = GeminiClient._to_vertexai_safety_settings(safety_settings) def compare_safety_settings(converted_safety_settings, expected_safety_settings): @@ -250,6 +253,59 @@ def compare_safety_settings(converted_safety_settings, expected_safety_settings) assert all(settings_comparison), "Converted safety settings are incorrect" +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +def test_vertexai_tool_config(gemini_client): + + tools = [{"function_name": "calculator"}] + + tool_config = {"function_calling_config": {"mode": "ANY"}} + + expected_tool_config = VertexAIToolConfig( + function_calling_config=VertexAIToolConfig.FunctionCallingConfig( + mode=VertexAIToolConfig.FunctionCallingConfig.Mode.ANY, + allowed_function_names=["calculator"], + ) + ) + + converted_tool_config = GeminiClient._to_vertexai_tool_config(tool_config, tools) + + converted_mode = converted_tool_config._gapic_tool_config.function_calling_config.mode + expected_mode = expected_tool_config._gapic_tool_config.function_calling_config.mode + converted_allowed_func = converted_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + expected_allowed_func = expected_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + + assert converted_mode == expected_mode, "Function calling mode is not converted correctly" + assert ( + converted_allowed_func == expected_allowed_func + ), "Function calling allowed function names is not converted correctly" + + +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +def test_vertexai_tool_config_no_functions(gemini_client): + + tools = [] + + tool_config = {"function_calling_config": {"mode": "ANY"}} + + expected_tool_config = VertexAIToolConfig( + function_calling_config=VertexAIToolConfig.FunctionCallingConfig( + mode=VertexAIToolConfig.FunctionCallingConfig.Mode.ANY, + ) + ) + + converted_tool_config = GeminiClient._to_vertexai_tool_config(tool_config, tools) + + converted_mode = converted_tool_config._gapic_tool_config.function_calling_config.mode + expected_mode = expected_tool_config._gapic_tool_config.function_calling_config.mode + converted_allowed_func = converted_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + expected_allowed_func = expected_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + + assert converted_mode == expected_mode, "Function calling mode is not converted correctly" + assert ( + converted_allowed_func == expected_allowed_func + ), "Function calling allowed function names is not converted correctly" + + # Test error handling @patch("autogen.oai.gemini.genai") @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") @@ -279,9 +335,10 @@ def test_cost_calculation(gemini_client, mock_response): @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +@patch("autogen.oai.gemini.Content") @patch("autogen.oai.gemini.genai.GenerativeModel") @patch("autogen.oai.gemini.genai.configure") -def test_create_response(mock_configure, mock_generative_model, gemini_client): +def test_create_response(mock_configure, mock_generative_model, mock_content, gemini_client): # Mock the genai model configuration and creation process mock_chat = MagicMock() mock_model = MagicMock() @@ -292,6 +349,8 @@ def test_create_response(mock_configure, mock_generative_model, gemini_client): # Set up a mock for the chat history item access and the text attribute return mock_history_part = MagicMock() mock_history_part.text = "Example response" + mock_history_part.function_call = None + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) mock_chat.history.__getitem__.return_value.parts.__getitem__.return_value = mock_history_part # Setup the mock to return a mocked chat response @@ -306,6 +365,55 @@ def test_create_response(mock_configure, mock_generative_model, gemini_client): assert response.choices[0].message.content == "Example response", "Response content should match expected output" +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +@patch("autogen.oai.gemini.Part") +@patch("autogen.oai.gemini.Content") +@patch("autogen.oai.gemini.genai.GenerativeModel") +@patch("autogen.oai.gemini.genai.configure") +def test_create_function_call_response(mock_configure, mock_generative_model, mock_content, mock_part, gemini_client): + # Mock the genai model configuration and creation process + mock_chat = MagicMock() + mock_model = MagicMock() + mock_configure.return_value = None + mock_generative_model.return_value = mock_model + mock_model.start_chat.return_value = mock_chat + + mock_part.to_dict.return_value = { + "function_call": {"name": "function_name", "args": {"arg1": "value1", "arg2": "value2"}} + } + + # Set up a mock for the chat history item access and the text attribute return + mock_history_part = MagicMock() + mock_history_part.text = None + mock_history_part.function_call.name = "function_name" + mock_history_part.function_call.args = {"arg1": "value1", "arg2": "value2"} + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) + + # Setup the mock to return a mocked chat response + mock_chat.send_message.return_value = MagicMock( + history=[ + MagicMock( + parts=[ + MagicMock( + function_call=MagicMock(name="function_name", arguments='{"arg1": "value1", "arg2": "value2"}') + ) + ] + ) + ] + ) + + # Call the create method + response = gemini_client.create( + {"model": "gemini-pro", "messages": [{"content": "Hello", "role": "user"}], "stream": False} + ) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.tool_calls[0].function.name == "function_name" + and json.loads(response.choices[0].message.tool_calls[0].function.arguments)["arg1"] == "value1" + ), "Response content should match expected output" + + @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") @patch("autogen.oai.gemini.GenerativeModel") @patch("autogen.oai.gemini.vertexai.init") @@ -320,7 +428,9 @@ def test_vertexai_create_response(mock_init, mock_generative_model, gemini_clien # Set up a mock for the chat history item access and the text attribute return mock_history_part = MagicMock() mock_history_part.text = "Example response" - mock_chat.history.__getitem__.return_value.parts.__getitem__.return_value = mock_history_part + mock_history_part.function_call = None + mock_history_part.role = "model" + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) # Setup the mock to return a mocked chat response mock_chat.send_message.return_value = MagicMock(history=[MagicMock(parts=[MagicMock(text="Example response")])]) @@ -330,10 +440,60 @@ def test_vertexai_create_response(mock_init, mock_generative_model, gemini_clien {"model": "gemini-pro", "messages": [{"content": "Hello", "role": "user"}], "stream": False} ) - # Assertions to check if response is structured as expected assert response.choices[0].message.content == "Example response", "Response content should match expected output" +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +@patch("autogen.oai.gemini.VertexAIPart") +@patch("autogen.oai.gemini.VertexAIContent") +@patch("autogen.oai.gemini.GenerativeModel") +@patch("autogen.oai.gemini.vertexai.init") +def test_vertexai_create_function_call_response( + mock_init, mock_generative_model, mock_content, mock_part, gemini_client_with_credentials +): + # Mock the genai model configuration and creation process + mock_chat = MagicMock() + mock_model = MagicMock() + mock_init.return_value = None + mock_generative_model.return_value = mock_model + mock_model.start_chat.return_value = mock_chat + + mock_part.to_dict.return_value = { + "function_call": {"name": "function_name", "args": {"arg1": "value1", "arg2": "value2"}} + } + + # Set up a mock for the chat history item access and the text attribute return + mock_history_part = MagicMock() + mock_history_part.text = None + mock_history_part.function_call.name = "function_name" + mock_history_part.function_call.args = {"arg1": "value1", "arg2": "value2"} + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) + + # Setup the mock to return a mocked chat response + mock_chat.send_message.return_value = MagicMock( + history=[ + MagicMock( + parts=[ + MagicMock( + function_call=MagicMock(name="function_name", arguments='{"arg1": "value1", "arg2": "value2"}') + ) + ] + ) + ] + ) + + # Call the create method + response = gemini_client_with_credentials.create( + {"model": "gemini-pro", "messages": [{"content": "Hello", "role": "user"}], "stream": False} + ) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.tool_calls[0].function.name == "function_name" + and json.loads(response.choices[0].message.tool_calls[0].function.arguments)["arg1"] == "value1" + ), "Response content should match expected output" + + @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") @patch("autogen.oai.gemini.GenerativeModel") @patch("autogen.oai.gemini.vertexai.init") @@ -348,6 +508,8 @@ def test_vertexai_default_auth_create_response(mock_init, mock_generative_model, # Set up a mock for the chat history item access and the text attribute return mock_history_part = MagicMock() mock_history_part.text = "Example response" + mock_history_part.function_call = None + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) mock_chat.history.__getitem__.return_value.parts.__getitem__.return_value = mock_history_part # Setup the mock to return a mocked chat response @@ -373,11 +535,11 @@ def test_create_vision_model_response(mock_configure, mock_generative_model, gem # Set up a mock to simulate the vision model behavior mock_vision_response = MagicMock() - mock_vision_part = MagicMock(text="Vision model output") + mock_vision_part = MagicMock(text="Vision model output", function_call=None) # Setting up the chain of return values for vision model response - mock_vision_response._result.candidates.__getitem__.return_value.content.parts.__getitem__.return_value = ( - mock_vision_part + mock_vision_response._result.candidates.__getitem__.return_value.content.parts.__iter__.return_value = iter( + [mock_vision_part] ) mock_model.generate_content.return_value = mock_vision_response @@ -420,10 +582,12 @@ def test_vertexai_create_vision_model_response(mock_init, mock_generative_model, # Set up a mock to simulate the vision model behavior mock_vision_response = MagicMock() - mock_vision_part = MagicMock(text="Vision model output") + mock_vision_part = MagicMock(text="Vision model output", function_call=None) # Setting up the chain of return values for vision model response - mock_vision_response.candidates.__getitem__.return_value.content.parts.__getitem__.return_value = mock_vision_part + mock_vision_response.candidates.__getitem__.return_value.content.parts.__iter__.return_value = iter( + [mock_vision_part] + ) mock_model.generate_content.return_value = mock_vision_response diff --git a/website/blog/2023-06-28-MathChat/index.mdx b/website/blog/2023-06-28-MathChat/index.mdx index be2423de9ee..2521065a625 100644 --- a/website/blog/2023-06-28-MathChat/index.mdx +++ b/website/blog/2023-06-28-MathChat/index.mdx @@ -76,7 +76,7 @@ We found that compared to basic prompting, which demonstrates the innate capabil For categories like Algebra and Prealgebra, PoT and PS showed little improvement, and in some instances, even led to a decrease in accuracy. However, MathChat was able to enhance total accuracy by around 6% compared to PoT and PS, showing competitive performance across all categories. Remarkably, MathChat improved accuracy in the Algebra category by about 15% over other methods. Note that categories like Intermediate Algebra and Precalculus remained challenging for all methods, with only about 20% of problems solved accurately. The code for experiments can be found at this [repository](https://github.com/yiranwu0/FLAML/tree/gpt_math_solver/flaml/autogen/math). -We now provide an implementation of MathChat using the interactive agents in AutoGen. See this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_MathChat.ipynb) for example usage. +We now provide an implementation of MathChat using the interactive agents in AutoGen. See this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_MathChat.ipynb) for example usage. ## Future Directions diff --git a/website/blog/2023-10-18-RetrieveChat/index.mdx b/website/blog/2023-10-18-RetrieveChat/index.mdx index 4bad582eb5d..9a6c2378723 100644 --- a/website/blog/2023-10-18-RetrieveChat/index.mdx +++ b/website/blog/2023-10-18-RetrieveChat/index.mdx @@ -206,7 +206,7 @@ We are using chromadb as the default vector database, you can also use mongodb, by simply set `vector_db` to `mongodb`, `pgvector`, `qdrant` and `couchbase` in `retrieve_config`, respectively. To plugin any other dbs, you can also extend class `agentchat.contrib.vectordb.base`, -check out the code [here](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/vectordb/base.py). +check out the code [here](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/vectordb/base.py). ## Advanced Usage of RAG Agents @@ -401,8 +401,8 @@ The online app and the source code are hosted in [HuggingFace](https://huggingfa ## Read More You can check out more example notebooks for RAG use cases: -- [Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) -- [Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) -- [Using RetrieveChat with Qdrant for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) -- [Using RetrieveChat Powered by PGVector for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) -- [Using RetrieveChat Powered by MongoDB Atlas for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_mongodb.ipynb) +- [Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat.ipynb) +- [Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_groupchat_RAG.ipynb) +- [Using RetrieveChat with Qdrant for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_qdrant.ipynb) +- [Using RetrieveChat Powered by PGVector for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- [Using RetrieveChat Powered by MongoDB Atlas for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_mongodb.ipynb) diff --git a/website/blog/2023-10-26-TeachableAgent/index.mdx b/website/blog/2023-10-26-TeachableAgent/index.mdx index f097acc7372..9ac57fa9995 100644 --- a/website/blog/2023-10-26-TeachableAgent/index.mdx +++ b/website/blog/2023-10-26-TeachableAgent/index.mdx @@ -24,13 +24,13 @@ In order to make effective decisions about memo storage and retrieval, the `Teac AutoGen contains four code examples that use `Teachability`. -1. Run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/main/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py) to converse with a teachable agent. +1. Run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/0.2/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py) to converse with a teachable agent. -2. Run [test_teachable_agent.py](https://github.com/microsoft/autogen/blob/main/test/agentchat/contrib/capabilities/test_teachable_agent.py) for quick unit testing of a teachable agent. +2. Run [test_teachable_agent.py](https://github.com/microsoft/autogen/blob/0.2/test/agentchat/contrib/capabilities/test_teachable_agent.py) for quick unit testing of a teachable agent. -3. Use the Jupyter notebook [agentchat_teachability.ipynb](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachability.ipynb) to step through examples discussed below. +3. Use the Jupyter notebook [agentchat_teachability.ipynb](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachability.ipynb) to step through examples discussed below. -4. Use the Jupyter notebook [agentchat_teachable_oai_assistants.ipynb](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachable_oai_assistants.ipynb) to make arbitrary OpenAI Assistants teachable through `GPTAssistantAgent`. +4. Use the Jupyter notebook [agentchat_teachable_oai_assistants.ipynb](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachable_oai_assistants.ipynb) to make arbitrary OpenAI Assistants teachable through `GPTAssistantAgent`. ## Basic Usage of Teachability diff --git a/website/blog/2023-11-06-LMM-Agent/index.mdx b/website/blog/2023-11-06-LMM-Agent/index.mdx index 0ab92c4dbb9..6613f794ee3 100644 --- a/website/blog/2023-11-06-LMM-Agent/index.mdx +++ b/website/blog/2023-11-06-LMM-Agent/index.mdx @@ -9,8 +9,8 @@ tags: [LMM, multimodal] **In Brief:** * Introducing the **Multimodal Conversable Agent** and the **LLaVA Agent** to enhance LMM functionalities. * Users can input text and images simultaneously using the `` tag to specify image loading. -* Demonstrated through the [GPT-4V notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb). -* Demonstrated through the [LLaVA notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb). +* Demonstrated through the [GPT-4V notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_gpt-4v.ipynb). +* Demonstrated through the [LLaVA notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb). ## Introduction Large multimodal models (LMMs) augment large language models (LLMs) with the ability to process multi-sensory data. @@ -62,7 +62,7 @@ The `MultimodalConversableAgent` interprets the input prompt, extracting images ## Advanced Usage Similar to other AutoGen agents, multimodal agents support multi-round dialogues with other agents, code generation, factual queries, and management via a GroupChat interface. -For example, the `FigureCreator` in our [GPT-4V notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) and [LLaVA notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) integrates two agents: a coder (an AssistantAgent) and critics (a multimodal agent). +For example, the `FigureCreator` in our [GPT-4V notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_gpt-4v.ipynb) and [LLaVA notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb) integrates two agents: a coder (an AssistantAgent) and critics (a multimodal agent). The coder drafts Python code for visualizations, while the critics provide insights for enhancement. Collaboratively, these agents aim to refine visual outputs. With `human_input_mode=ALWAYS`, you can also contribute suggestions for better visualizations. diff --git a/website/blog/2023-11-13-OAI-assistants/index.mdx b/website/blog/2023-11-13-OAI-assistants/index.mdx index 2fc9bee359b..ae8073285a4 100644 --- a/website/blog/2023-11-13-OAI-assistants/index.mdx +++ b/website/blog/2023-11-13-OAI-assistants/index.mdx @@ -9,12 +9,12 @@ tags: [openai-assistant] ## TL;DR -OpenAI assistants are now integrated into AutoGen via [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/gpt_assistant_agent.py). +OpenAI assistants are now integrated into AutoGen via [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/gpt_assistant_agent.py). This enables multiple OpenAI assistants, which form the backend of the now popular GPTs, to collaborate and tackle complex tasks. Checkout example notebooks for reference: -* [Basic example](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) -* [Code interpreter](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_code_interpreter.ipynb) -* [Function calls](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_function_call.ipynb) +* [Basic example](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) +* [Code interpreter](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_code_interpreter.ipynb) +* [Function calls](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_function_call.ipynb) ## Introduction diff --git a/website/blog/2023-11-20-AgentEval/index.mdx b/website/blog/2023-11-20-AgentEval/index.mdx index 1abb9e6c9f8..ac7fab7dc8f 100644 --- a/website/blog/2023-11-20-AgentEval/index.mdx +++ b/website/blog/2023-11-20-AgentEval/index.mdx @@ -14,7 +14,7 @@ tags: [LLM, GPT, evaluation, task utility] **TL;DR:** * As a developer of an LLM-powered application, how can you assess the utility it brings to end users while helping them with their tasks? * To shed light on the question above, we introduce `AgentEval` — the first version of the framework to assess the utility of any LLM-powered application crafted to assist users in specific tasks. AgentEval aims to simplify the evaluation process by automatically proposing a set of criteria tailored to the unique purpose of your application. This allows for a comprehensive assessment, quantifying the utility of your application against the suggested criteria. -* We demonstrate how `AgentEval` work using [math problems dataset](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat) as an example in the [following notebook](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb). Any feedback would be useful for future development. Please contact us on our [Discord](http://aka.ms/autogen-dc). +* We demonstrate how `AgentEval` work using [math problems dataset](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat) as an example in the [following notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb). Any feedback would be useful for future development. Please contact us on our [Discord](http://aka.ms/autogen-dc). ## Introduction @@ -54,7 +54,7 @@ critic = autogen.AssistantAgent( ) ``` -Next, the critic is given successful and failed examples of the task execution; then, it is able to return a list of criteria (Fig. 1). For reference, use the [following notebook](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb). +Next, the critic is given successful and failed examples of the task execution; then, it is able to return a list of criteria (Fig. 1). For reference, use the [following notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb). * The goal of `QuantifierAgent` is to quantify each of the suggested criteria (Fig. 1), providing us with an idea of the utility of this system for the given task. Here is an example of how it can be defined: diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index a6d0025699d..c7329a4b192 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -14,8 +14,8 @@ user prompt required, powered by a new designed class **AgentBuilder**. AgentBui leveraging [vLLM](https://docs.vllm.ai/en/latest/index.html) and [FastChat](https://github.com/lm-sys/FastChat). Checkout example notebooks and source code for reference: -- [AutoBuild Examples](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) -- [AgentBuilder](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/agent_builder.py) +- [AutoBuild Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_basic.ipynb) +- [AgentBuilder](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/agent_builder.py) ## Introduction In this blog, we introduce **AutoBuild**, a pipeline that can automatically build multi-agent systems for complex tasks. diff --git a/website/blog/2023-12-01-AutoGenStudio/index.mdx b/website/blog/2023-12-01-AutoGenStudio/index.mdx index a2558acb01d..b379e452045 100644 --- a/website/blog/2023-12-01-AutoGenStudio/index.mdx +++ b/website/blog/2023-12-01-AutoGenStudio/index.mdx @@ -139,7 +139,7 @@ This section focuses on defining the properties of agents and agent workflows. I

-**Agents**: This provides an interface to declaratively specify properties for an AutoGen agent (mirrors most of the members of a base [AutoGen conversable agent](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/conversable_agent.py) class). +**Agents**: This provides an interface to declaratively specify properties for an AutoGen agent (mirrors most of the members of a base [AutoGen conversable agent](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/conversable_agent.py) class). **Agent Workflows**: An agent workflow is a specification of a set of agents that can work together to accomplish a task. The simplest version of this is a setup with two agents – a user proxy agent (that represents a user i.e. it compiles code and prints result) and an assistant that can address task requests (e.g., generating plans, writing code, evaluating responses, proposing error recovery steps, etc.). A more complex flow could be a group chat where even more agents work towards a solution. diff --git a/website/blog/2023-12-23-AgentOptimizer/index.mdx b/website/blog/2023-12-23-AgentOptimizer/index.mdx index ce92952cfb6..8c709e931f3 100644 --- a/website/blog/2023-12-23-AgentOptimizer/index.mdx +++ b/website/blog/2023-12-23-AgentOptimizer/index.mdx @@ -17,7 +17,7 @@ More information could be found in: **Paper**: https://arxiv.org/abs/2402.11359. -**Notebook**: https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentoptimizer.ipynb. +**Notebook**: https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_agentoptimizer.ipynb. ## Introduction In the traditional ML pipeline, we train a model by updating its weights according to the loss on the training set, while in the era of LLM agents, how should we train an agent? diff --git a/website/blog/2024-01-25-AutoGenBench/index.mdx b/website/blog/2024-01-25-AutoGenBench/index.mdx index 3f4b2d4f216..cd58ecee643 100644 --- a/website/blog/2024-01-25-AutoGenBench/index.mdx +++ b/website/blog/2024-01-25-AutoGenBench/index.mdx @@ -21,8 +21,8 @@ Today we are releasing AutoGenBench - a tool for evaluating AutoGen agents and w AutoGenBench is a standalone command line tool, installable from PyPI, which handles downloading, configuring, running, and reporting supported benchmarks. AutoGenBench works best when run alongside Docker, since it uses Docker to isolate tests from one another. -- See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/README.md) for information on installation and running benchmarks. -- See the [AutoGenBench CONTRIBUTING guide](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/CONTRIBUTING.md) for information on developing or contributing benchmark datasets. +- See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/README.md) for information on installation and running benchmarks. +- See the [AutoGenBench CONTRIBUTING guide](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/CONTRIBUTING.md) for information on developing or contributing benchmark datasets. ### Quick Start @@ -125,7 +125,7 @@ Please do not cite these values in academic work without first inspecting and ve From this output we can see the results of the three separate repetitions of each task, and final summary statistics of each run. In this case, the results were generated via GPT-4 (as defined in the OAI_CONFIG_LIST that was provided), and used the `TwoAgents` template. **It is important to remember that AutoGenBench evaluates _specific_ end-to-end configurations of agents (as opposed to evaluating a model or cognitive framework more generally).** -Finally, complete execution traces and logs can be found in the `Results` folder. See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/README.md) for more details about command-line options and output formats. Each of these commands also offers extensive in-line help via: +Finally, complete execution traces and logs can be found in the `Results` folder. See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/README.md) for more details about command-line options and output formats. Each of these commands also offers extensive in-line help via: - `autogenbench --help` - `autogenbench clone --help` @@ -145,4 +145,4 @@ For an up to date tracking of our work items on this project, please see [AutoGe ## Call for Participation -Finally, we want to end this blog post with an open call for contributions. AutoGenBench is still nascent, and has much opportunity for improvement. New benchmarks are constantly being published, and will need to be added. Everyone may have their own distinct set of metrics that they care most about optimizing, and these metrics should be onboarded. To this end, we welcome any and all contributions to this corner of the AutoGen project. If contributing is something that interests you, please see the [contributor’s guide](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/CONTRIBUTING.md) and join our [Discord](https://aka.ms/autogen-dc) discussion in the [#autogenbench](https://discord.com/channels/1153072414184452236/1199851779328847902) channel! +Finally, we want to end this blog post with an open call for contributions. AutoGenBench is still nascent, and has much opportunity for improvement. New benchmarks are constantly being published, and will need to be added. Everyone may have their own distinct set of metrics that they care most about optimizing, and these metrics should be onboarded. To this end, we welcome any and all contributions to this corner of the AutoGen project. If contributing is something that interests you, please see the [contributor’s guide](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/CONTRIBUTING.md) and join our [Discord](https://aka.ms/autogen-dc) discussion in the [#autogenbench](https://discord.com/channels/1153072414184452236/1199851779328847902) channel! diff --git a/website/blog/2024-01-26-Custom-Models/index.mdx b/website/blog/2024-01-26-Custom-Models/index.mdx index 81a9ad383d3..e0d43c2ac0a 100644 --- a/website/blog/2024-01-26-Custom-Models/index.mdx +++ b/website/blog/2024-01-26-Custom-Models/index.mdx @@ -13,7 +13,7 @@ AutoGen now supports custom models! This feature empowers users to define and lo ## Quickstart -An interactive and easy way to get started is by following the notebook [here](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_custom_model.ipynb) which loads a local model from HuggingFace into AutoGen and uses it for inference, and making changes to the class provided. +An interactive and easy way to get started is by following the notebook [here](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_custom_model.ipynb) which loads a local model from HuggingFace into AutoGen and uses it for inference, and making changes to the class provided. ### Step 1: Create the custom model client class diff --git a/website/blog/2024-03-03-AutoGen-Update/index.mdx b/website/blog/2024-03-03-AutoGen-Update/index.mdx index 7458ec74e05..f1128b52368 100644 --- a/website/blog/2024-03-03-AutoGen-Update/index.mdx +++ b/website/blog/2024-03-03-AutoGen-Update/index.mdx @@ -138,7 +138,7 @@ The community has been working hard to address them in several dimensions: We are working on agent-based evaluation tools and benchmarking tools. For example: -- [AgentEval](/blog/2023/11/20/AgentEval). Our [research](https://arxiv.org/abs/2402.09015) finds that LLM agents built with AutoGen can be used to automatically identify evaluation criteria and assess the performance from task descriptions and execution logs. It is demonstrated as a [notebook example](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb). Feedback and help are welcome for building it into the library. +- [AgentEval](/blog/2023/11/20/AgentEval). Our [research](https://arxiv.org/abs/2402.09015) finds that LLM agents built with AutoGen can be used to automatically identify evaluation criteria and assess the performance from task descriptions and execution logs. It is demonstrated as a [notebook example](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb). Feedback and help are welcome for building it into the library. - [AutoGenBench](/blog/2024/01/25/AutoGenBench). AutoGenBench is a commandline tool for downloading, configuring, running an agentic benchmark, and reporting results. It is designed to allow repetition, isolation and instrumentation, leveraging the new [runtime logging](/docs/notebooks/agentchat_logging) feature. These tools have been used for improving the AutoGen library as well as applications. For example, the new state-of-the-art performance achieved by a multi-agent solution to the [GAIA](https://huggingface.co/spaces/gaia-benchmark/leaderboard) benchmark has benefited from these evaluation tools. @@ -147,7 +147,7 @@ These tools have been used for improving the AutoGen library as well as applicat We are making rapid progress in further improving the interface to make it even easier to build agent applications. For example: -- [AutoBuild](/blog/2023/11/26/Agent-AutoBuild). AutoBuild is an ongoing area of research to automatically create or select a group of agents for a given task and objective. If successful, it will greatly reduce the effort from users or developers when using the multi-agent technology. It also paves the way for agentic decomposition to handle complex tasks. It is available as an experimental feature and demonstrated in two modes: free-form [creation](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) and [selection](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb) from a library. +- [AutoBuild](/blog/2023/11/26/Agent-AutoBuild). AutoBuild is an ongoing area of research to automatically create or select a group of agents for a given task and objective. If successful, it will greatly reduce the effort from users or developers when using the multi-agent technology. It also paves the way for agentic decomposition to handle complex tasks. It is available as an experimental feature and demonstrated in two modes: free-form [creation](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_basic.ipynb) and [selection](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_agent_library.ipynb) from a library. - [AutoGen Studio](/blog/2023/12/01/AutoGenStudio). AutoGen Studio is a no-code UI for fast experimentation with the multi-agent conversations. It lowers the barrier of entrance to the AutoGen technology. Models, agents, and workflows can all be configured without writing code. And chatting with multiple agents in a playground is immediately available after the configuration. Although only a subset of `autogen-agentchat` features are available in this sample app, it demonstrates a promising experience. It has generated tremendous excitement in the community. - Conversation Programming+. The [AutoGen paper](https://arxiv.org/abs/2308.08155) introduced a key concept of _Conversation Programming_, which can be used to program diverse conversation patterns such as 1-1 chat, group chat, hierarchical chat, nested chat etc. While we offered dynamic group chat as an example of high-level orchestration, it made other patterns relatively less discoverable. Therefore, we have added more convenient conversation programming features which enables easier definition of other types of complex workflow, such as [finite state machine based group chat](/blog/2024/02/11/FSM-GroupChat), [sequential chats](/docs/notebooks/agentchats_sequential_chats), and [nested chats](/docs/notebooks/agentchat_nestedchat). Many users have found them useful in implementing specific patterns, which have been always possible but more obvious with the added features. I will write another blog post for a deep dive. @@ -173,8 +173,8 @@ The extensible design of AutoGen makes it easy to integrate with new technologie The above only covers a subset of new features and roadmap. There are many other interesting new features, integration examples or sample apps: -- new features like stateful code execution, [tool decorators](/docs/Use-Cases/agent_chat#tool-calling), [long context handling](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_capability_long_context_handling.ipynb), [web agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_surfer.ipynb). -- integration examples like using [guidance](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_guidance.ipynb) to generate structured response. +- new features like stateful code execution, [tool decorators](/docs/Use-Cases/agent_chat#tool-calling), [long context handling](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_capability_long_context_handling.ipynb), [web agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_surfer.ipynb). +- integration examples like using [guidance](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_guidance.ipynb) to generate structured response. - sample apps like [AutoAnny](/blog/2024/02/02/AutoAnny). ## Call for Help diff --git a/website/blog/2024-10-02-new-autogen-architecture-preview/index.mdx b/website/blog/2024-10-02-new-autogen-architecture-preview/index.mdx index 3b13ac514f8..42a873d38bb 100644 --- a/website/blog/2024-10-02-new-autogen-architecture-preview/index.mdx +++ b/website/blog/2024-10-02-new-autogen-architecture-preview/index.mdx @@ -71,7 +71,7 @@ Today, we are delighted to share our progress and invite everyone to collaborate with us and provide feedback to evolve AutoGen and help shape the future of multi-agent systems. -As the first step, we are opening a [pull request](#) into the main branch with the +As the first step, we are opening a [pull request](https://github.com/microsoft/autogen/pull/3600) into the main branch with the current state of development of 0.4. After approximately a week, we plan to merge this into main and continue development. There's still a lot left to do before 0.4 is ready for release though, so keep in mind this is a work in @@ -82,12 +82,12 @@ Starting in AutoGen 0.4, the project will have three main libraries: - **Core** - the building blocks for an event-driven agentic system. - **AgentChat** - a task-driven, high-level API built with core, including group chat, code execution, pre-built agents, and more. This is the most similar API - to AutoGen 0.2 and will be the easiest API to migrate to. + to AutoGen [0.2](https://github.com/microsoft/autogen/tree/0.2) and will be the easiest API to migrate to. - **Extensions** - implementations of core interfaces and third-party integrations (e.g., Azure code executor and OpenAI model client). -AutoGen 0.2 is still available, developed and maintained out of the [0.2 branch](https://github.com/microsoft/autogen/tree/0.2). -For everyone looking for a stable version, we recommend continuing to use 0.2 +AutoGen [0.2](https://github.com/microsoft/autogen/tree/0.2) is still available, developed and maintained out of the [0.2 branch](https://github.com/microsoft/autogen/tree/0.2). +For everyone looking for a stable version, we recommend continuing to use [0.2](https://github.com/microsoft/autogen/tree/0.2) for the time being. It can be installed using: ```sh diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 5efd71748f9..550c2604659 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -18,7 +18,7 @@ Links to notebook examples: - Automated Task Solving by Group Chat (with 3 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat) - Automated Data Visualization by Group Chat (with 3 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_vis) - Automated Complex Task Solving by Group Chat (with 6 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_research) -- Automated Task Solving with Coding & Planning Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_planning.ipynb) +- Automated Task Solving with Coding & Planning Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_planning.ipynb) - Automated Task Solving with transition paths specified in a graph - [View Notebook](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat_finite_state_machine) - Running a group chat as an inner-monolgue via the SocietyOfMindAgent - [View Notebook](/docs/notebooks/agentchat_society_of_mind) - Running a group chat with custom speaker selection function - [View Notebook](/docs/notebooks/agentchat_groupchat_customized) @@ -38,73 +38,73 @@ Links to notebook examples: ### Applications -- Automated Continual Learning from New Data - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_stream.ipynb) +- Automated Continual Learning from New Data - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_stream.ipynb) - [OptiGuide](https://github.com/microsoft/optiguide) - Coding, Tool Using, Safeguarding & Question Answering for Supply Chain Optimization - [AutoAnny](https://github.com/microsoft/autogen/tree/main/samples/apps/auto-anny) - A Discord bot built using AutoGen ### Tool Use -- **Web Search**: Solve Tasks Requiring Web Info - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_web_info.ipynb) +- **Web Search**: Solve Tasks Requiring Web Info - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_web_info.ipynb) - Use Provided Tools as Functions - [View Notebook](/docs/notebooks/agentchat_function_call_currency_calculator) - Use Tools via Sync and Async Function Calling - [View Notebook](/docs/notebooks/agentchat_function_call_async) -- Task Solving with Langchain Provided Tools as Functions - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_langchain.ipynb) +- Task Solving with Langchain Provided Tools as Functions - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_langchain.ipynb) - **RAG**: Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_RAG) -- Function Inception: Enable AutoGen agents to update/remove functions during conversations. - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_inception_function.ipynb) +- Function Inception: Enable AutoGen agents to update/remove functions during conversations. - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_inception_function.ipynb) - Agent Chat with Whisper - [View Notebook](/docs/notebooks/agentchat_video_transcript_translate_with_whisper) -- Constrained Responses via Guidance - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_guidance.ipynb) -- Browse the Web with Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_surfer.ipynb) -- **SQL**: Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_sql_spider.ipynb) +- Constrained Responses via Guidance - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_guidance.ipynb) +- Browse the Web with Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_surfer.ipynb) +- **SQL**: Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_sql_spider.ipynb) - **Web Scraping**: Web Scraping with Apify - [View Notebook](/docs/notebooks/agentchat_webscraping_with_apify) -- **Write a software app, task by task, with specially designed functions.** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_function_call_code_writing.ipynb). +- **Write a software app, task by task, with specially designed functions.** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_function_call_code_writing.ipynb). ### Human Involvement -- Simple example in ChatGPT style [View example](https://github.com/microsoft/autogen/blob/main/samples/simple_chat.py) -- Auto Code Generation, Execution, Debugging and **Human Feedback** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_human_feedback.ipynb) -- Automated Task Solving with GPT-4 + **Multiple Human Users** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_two_users.ipynb) -- Agent Chat with **Async Human Inputs** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/Async_human_input.ipynb) +- Simple example in ChatGPT style [View example](https://github.com/microsoft/autogen/blob/0.2/samples/simple_chat.py) +- Auto Code Generation, Execution, Debugging and **Human Feedback** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_human_feedback.ipynb) +- Automated Task Solving with GPT-4 + **Multiple Human Users** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_two_users.ipynb) +- Agent Chat with **Async Human Inputs** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/Async_human_input.ipynb) ### Agent Teaching and Learning - Teach Agents New Skills & Reuse via Automated Chat - [View Notebook](/docs/notebooks/agentchat_teaching) - Teach Agents New Facts, User Preferences and Skills Beyond Coding - [View Notebook](/docs/notebooks/agentchat_teachability) -- Teach OpenAI Assistants Through GPTAssistantAgent - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachable_oai_assistants.ipynb) -- Agent Optimizer: Train Agents in an Agentic Way - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentoptimizer.ipynb) +- Teach OpenAI Assistants Through GPTAssistantAgent - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachable_oai_assistants.ipynb) +- Agent Optimizer: Train Agents in an Agentic Way - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_agentoptimizer.ipynb) ### Multi-Agent Chat with OpenAI Assistants in the loop -- Hello-World Chat with OpenAi Assistant in AutoGen - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) -- Chat with OpenAI Assistant using Function Call - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_function_call.ipynb) -- Chat with OpenAI Assistant with Code Interpreter - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_code_interpreter.ipynb) -- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_retrieval.ipynb) -- OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_groupchat.ipynb) -- GPTAssistantAgent based Multi-Agent Tool Use - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/gpt_assistant_agent_function_call.ipynb) +- Hello-World Chat with OpenAi Assistant in AutoGen - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) +- Chat with OpenAI Assistant using Function Call - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_function_call.ipynb) +- Chat with OpenAI Assistant with Code Interpreter - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_code_interpreter.ipynb) +- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_retrieval.ipynb) +- OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_groupchat.ipynb) +- GPTAssistantAgent based Multi-Agent Tool Use - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/gpt_assistant_agent_function_call.ipynb) ### Non-OpenAI Models - Conversational Chess using non-OpenAI Models - [View Notebook](/docs/notebooks/agentchat_nested_chats_chess_altmodels) ### Multimodal Agent -- Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) -- Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) -- Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) +- Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_dalle_and_gpt4v.ipynb) +- Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb) +- Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_gpt-4v.ipynb) ### Long Context Handling - + - Long Context Handling as A Capability - [View Notebook](/docs/notebooks/agentchat_transform_messages) ### Evaluation and Assessment -- AgentEval: A Multi-Agent System for Assess Utility of LLM-powered Applications - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb) +- AgentEval: A Multi-Agent System for Assess Utility of LLM-powered Applications - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb) ### Automatic Agent Building -- Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) -- Automatically Build Multi-agent System from Agent Library - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb) +- Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_basic.ipynb) +- Automatically Build Multi-agent System from Agent Library - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_agent_library.ipynb) ### Observability -- Track LLM calls, tool usage, actions and errors using AgentOps - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentops.ipynb) +- Track LLM calls, tool usage, actions and errors using AgentOps - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_agentops.ipynb) ## Enhanced Inferences @@ -112,7 +112,7 @@ Links to notebook examples: - API Unification - [View Documentation with Code Example](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference/#api-unification) - Utility Functions to Help Managing API configurations effectively - [View Notebook](/docs/topics/llm_configuration) -- Cost Calculation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_cost_token_tracking.ipynb) +- Cost Calculation - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_cost_token_tracking.ipynb) ### Inference Hyperparameters Tuning @@ -120,5 +120,5 @@ AutoGen offers a cost-effective hyperparameter optimization technique [EcoOptiGe Please find documentation about this feature [here](/docs/Use-Cases/enhanced_inference). Links to notebook examples: -* [Optimize for Code Generation](https://github.com/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) -* [Optimize for Math](https://github.com/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) +* [Optimize for Code Generation](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_completion.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) +* [Optimize for Math](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_chatgpt_gpt4.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) diff --git a/website/docs/Getting-Started.mdx b/website/docs/Getting-Started.mdx index 4a2bbf63fff..761ed6f3d34 100644 --- a/website/docs/Getting-Started.mdx +++ b/website/docs/Getting-Started.mdx @@ -118,7 +118,7 @@ Learn more about configuring LLMs for agents [here](/docs/topics/llm_configurati #### Multi-Agent Conversation Framework Autogen enables the next-gen LLM applications with a generic multi-agent conversation framework. It offers customizable and conversable agents which integrate LLMs, tools, and humans. -By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py), +By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py), The figure below shows an example conversation flow with AutoGen. diff --git a/website/docs/Migration-Guide.md b/website/docs/Migration-Guide.md index f996a157a29..97e16de0770 100644 --- a/website/docs/Migration-Guide.md +++ b/website/docs/Migration-Guide.md @@ -26,7 +26,7 @@ autogen.runtime_logging.start() # Stop logging autogen.runtime_logging.stop() ``` -Checkout [Logging documentation](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference#logging) and [Logging example notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_logging.ipynb) to learn more. +Checkout [Logging documentation](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference#logging) and [Logging example notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_logging.ipynb) to learn more. Inference parameter tuning can be done via [`flaml.tune`](https://microsoft.github.io/FLAML/docs/Use-Cases/Tune-User-Defined-Function). - `seed` in autogen is renamed into `cache_seed` to accommodate the newly added `seed` param in openai chat completion api. `use_cache` is removed as a kwarg in `OpenAIWrapper.create()` for being automatically decided by `cache_seed`: int | None. The difference between autogen's `cache_seed` and openai's `seed` is that: diff --git a/website/docs/Use-Cases/agent_chat.md b/website/docs/Use-Cases/agent_chat.md index 76f9959841b..fa85691810c 100644 --- a/website/docs/Use-Cases/agent_chat.md +++ b/website/docs/Use-Cases/agent_chat.md @@ -81,13 +81,13 @@ AutoGen, by integrating conversation-driven control utilizing both programming a With the pluggable auto-reply function, one can choose to invoke conversations with other agents depending on the content of the current message and context. For example: - Hierarchical chat like in [OptiGuide](https://github.com/microsoft/optiguide). -- [Dynamic Group Chat](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat.ipynb) which is a special form of hierarchical chat. In the system, we register a reply function in the group chat manager, which broadcasts messages and decides who the next speaker will be in a group chat setting. +- [Dynamic Group Chat](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_groupchat.ipynb) which is a special form of hierarchical chat. In the system, we register a reply function in the group chat manager, which broadcasts messages and decides who the next speaker will be in a group chat setting. - [Finite State Machine graphs to set speaker transition constraints](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat_finite_state_machine) which is a special form of dynamic group chat. In this approach, a directed transition matrix is fed into group chat. Users can specify legal transitions or specify disallowed transitions. -- Nested chat like in [conversational chess](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_nested_chats_chess.ipynb). +- Nested chat like in [conversational chess](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_nested_chats_chess.ipynb). 2. LLM-Based Function Call -Another approach involves LLM-based function calls, where LLM decides if a specific function should be invoked based on the conversation's status during each inference. This approach enables dynamic multi-agent conversations, as seen in scenarios like [multi-user math problem solving scenario](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_two_users.ipynb), where a student assistant automatically seeks expertise via function calls. +Another approach involves LLM-based function calls, where LLM decides if a specific function should be invoked based on the conversation's status during each inference. This approach enables dynamic multi-agent conversations, as seen in scenarios like [multi-user math problem solving scenario](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_two_users.ipynb), where a student assistant automatically seeks expertise via function calls. ### Diverse Applications Implemented with AutoGen diff --git a/website/docs/Use-Cases/enhanced_inference.md b/website/docs/Use-Cases/enhanced_inference.md index 14723391e8c..825f46ff071 100644 --- a/website/docs/Use-Cases/enhanced_inference.md +++ b/website/docs/Use-Cases/enhanced_inference.md @@ -140,7 +140,7 @@ The `OpenAIWrapper` from `autogen` tracks token counts and costs of your API cal - `mode='actual'`: only print non-cached usage. - `mode='total'`: only print all usage (including cache). -Reset your session's usage data with `clear_usage_summary()` when needed. [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_client_cost.ipynb) +Reset your session's usage data with `clear_usage_summary()` when needed. [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_client_cost.ipynb) Example usage: ```python @@ -307,7 +307,7 @@ When debugging or diagnosing an LLM-based system, it is often convenient to log ### For openai >= 1 -Logging example: [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_logging.ipynb) +Logging example: [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_logging.ipynb) #### Start logging: ```python diff --git a/website/docs/contributor-guide/docker.md b/website/docs/contributor-guide/docker.md index 8ea54ed7c22..b09a53aa35f 100644 --- a/website/docs/contributor-guide/docker.md +++ b/website/docs/contributor-guide/docker.md @@ -48,4 +48,4 @@ docker run -it -p 8081:3000 -v /home/AutoGenDeveloper/autogen-newcode:newstuff/ ## Develop in Remote Container If you use vscode, you can open the autogen folder in a [Container](https://code.visualstudio.com/docs/remote/containers). -We have provided the configuration in [devcontainer](https://github.com/microsoft/autogen/blob/main/.devcontainer). They can be used in GitHub codespace too. Developing AutoGen in dev containers is recommended. +We have provided the configuration in [devcontainer](https://github.com/microsoft/autogen/blob/0.2/.devcontainer). They can be used in GitHub codespace too. Developing AutoGen in dev containers is recommended. diff --git a/website/docs/contributor-guide/documentation.md b/website/docs/contributor-guide/documentation.md index 70a79b70069..3380ff52928 100644 --- a/website/docs/contributor-guide/documentation.md +++ b/website/docs/contributor-guide/documentation.md @@ -2,7 +2,7 @@ ## How to get a notebook rendered on the website -See [here](https://github.com/microsoft/autogen/blob/main/notebook/contributing.md#how-to-get-a-notebook-displayed-on-the-website) for instructions on how to get a notebook in the `notebook` directory rendered on the website. +See [here](https://github.com/microsoft/autogen/blob/0.2/notebook/contributing.md#how-to-get-a-notebook-displayed-on-the-website) for instructions on how to get a notebook in the `notebook` directory rendered on the website. ## Build documentation locally diff --git a/website/docs/contributor-guide/tests.md b/website/docs/contributor-guide/tests.md index c5eabb90732..a4b03b6379d 100644 --- a/website/docs/contributor-guide/tests.md +++ b/website/docs/contributor-guide/tests.md @@ -2,12 +2,12 @@ Tests are automatically run via GitHub actions. There are two workflows: -1. [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) -1. [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml) +1. [build.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/build.yml) +1. [openai.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/openai.yml) -The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml). +The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/openai.yml). -Make sure all tests pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) checks to pass +Make sure all tests pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/build.yml) checks to pass ## Running tests locally @@ -27,7 +27,7 @@ Tests for the `autogen.agentchat.contrib` module may be skipped automatically if required dependencies are not installed. Please consult the documentation for each contrib module to see what dependencies are required. -See [here](https://github.com/microsoft/autogen/blob/main/notebook/contributing.md#testing) for how to run notebook tests. +See [here](https://github.com/microsoft/autogen/blob/0.2/notebook/contributing.md#testing) for how to run notebook tests. ## Skip flags for tests diff --git a/website/docs/ecosystem/llamaindex.md b/website/docs/ecosystem/llamaindex.md index 4e9cffeedc7..6561880b4d2 100644 --- a/website/docs/ecosystem/llamaindex.md +++ b/website/docs/ecosystem/llamaindex.md @@ -4,4 +4,4 @@ [Llamaindex](https://www.llamaindex.ai/) allows the users to create Llamaindex agents and integrate them in autogen conversation patterns. -- [Llamaindex + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb) +- [Llamaindex + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb) diff --git a/website/docs/ecosystem/microsoft-fabric.md b/website/docs/ecosystem/microsoft-fabric.md index d0593a4a7d0..5ef625c7176 100644 --- a/website/docs/ecosystem/microsoft-fabric.md +++ b/website/docs/ecosystem/microsoft-fabric.md @@ -4,4 +4,4 @@ [Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/get-started/microsoft-fabric-overview) is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place. In this notenook, we give a simple example for using AutoGen in Microsoft Fabric. -- [Microsoft Fabric + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_microsoft_fabric.ipynb) +- [Microsoft Fabric + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_microsoft_fabric.ipynb) diff --git a/website/docs/ecosystem/pgvector.md b/website/docs/ecosystem/pgvector.md index 99afa676e7e..637941e6135 100644 --- a/website/docs/ecosystem/pgvector.md +++ b/website/docs/ecosystem/pgvector.md @@ -2,4 +2,4 @@ [PGVector](https://github.com/pgvector/pgvector) is an open-source vector similarity search for Postgres. -- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_pgvector.ipynb) diff --git a/website/docs/installation/Docker.md b/website/docs/installation/Docker.md index a7fa6bd829b..a4506f9c10a 100644 --- a/website/docs/installation/Docker.md +++ b/website/docs/installation/Docker.md @@ -41,7 +41,7 @@ Here's how you can run an application built with AutoGen, using the Docker image Here, `$(pwd)/myapp` is your local directory, and `/home/autogen/autogen/myapp` is the path in the Docker container where your code will be located. -2. **Mount your code:** Now suppose you have your application built with AutoGen in a main script named `twoagent.py` ([example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py)) in a folder named `myapp`. With the command line below, you can mount your folder and run the application in Docker. +2. **Mount your code:** Now suppose you have your application built with AutoGen in a main script named `twoagent.py` ([example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py)) in a folder named `myapp`. With the command line below, you can mount your folder and run the application in Docker. ```python # Mount the local folder `myapp` into docker image and run the script named "twoagent.py" in the docker. @@ -82,6 +82,6 @@ docker run -it -p {WorkstationPortNum}:{DockerPortNum} -v {WorkStation_Dir}:{Doc ## Additional Resources -- Details on all the Dockerfile options can be found in the [Dockerfile](https://github.com/microsoft/autogen/blob/main/.devcontainer/README.md) README. +- Details on all the Dockerfile options can be found in the [Dockerfile](https://github.com/microsoft/autogen/blob/0.2/.devcontainer/README.md) README. - For more information on Docker usage and best practices, refer to the [official Docker documentation](https://docs.docker.com). - Details on how to use the Dockerfile dev version can be found on the [Contributor Guide](/docs/contributor-guide/docker). diff --git a/website/docs/installation/Optional-Dependencies.md b/website/docs/installation/Optional-Dependencies.md index 3f8164a667e..89013d023ea 100644 --- a/website/docs/installation/Optional-Dependencies.md +++ b/website/docs/installation/Optional-Dependencies.md @@ -38,9 +38,9 @@ pip install "pyautogen[blendsearch]<0.2" Example notebooks: -[Optimize for Code Generation](https://github.com/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) +[Optimize for Code Generation](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_completion.ipynb) -[Optimize for Math](https://github.com/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) +[Optimize for Math](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_chatgpt_gpt4.ipynb) ## retrievechat @@ -72,11 +72,11 @@ You can find a list of all supported document types by using `autogen.retrieve_u Example notebooks: -[Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) +[Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat.ipynb) -[Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) +[Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_groupchat_RAG.ipynb) -[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) +[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_qdrant.ipynb) ## Teachability @@ -86,7 +86,7 @@ To use Teachability, please install AutoGen with the [teachable] option. pip install "autogen-agentchat[teachable]~=0.2" ``` -Example notebook: [Chatting with a teachable agent](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachability.ipynb) +Example notebook: [Chatting with a teachable agent](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachability.ipynb) ## Large Multimodal Model (LMM) Agents @@ -98,7 +98,7 @@ pip install "autogen-agentchat[lmm]~=0.2" Example notebooks: -[LLaVA Agent](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) +[LLaVA Agent](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb) ## mathchat @@ -110,7 +110,7 @@ pip install "pyautogen[mathchat]<0.2" Example notebooks: -[Using MathChat to Solve Math Problems](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_MathChat.ipynb) +[Using MathChat to Solve Math Problems](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_MathChat.ipynb) ## Graph diff --git a/website/docs/topics/groupchat/using_custom_models.md b/website/docs/topics/groupchat/using_custom_models.md new file mode 100644 index 00000000000..4948d8ab9f6 --- /dev/null +++ b/website/docs/topics/groupchat/using_custom_models.md @@ -0,0 +1,79 @@ +# Using Custom Models + +When using `GroupChatManager` we need to pass a `GroupChat` object in the constructor, a dataclass responsible for +gathering agents, preparing messages from prompt templates and selecting speakers +(eventually using `speaker_selection_method` as described [here](customized_speaker_selection)). + +To do so GroupChat internally initializes two instances of ConversableAgent. +In order to control the model clients used by the agents instantiated within the GroupChat, which already receives the +`llm_config` passed to GroupChatManager, the optional `model_client_cls` attribute can be set. + + +## Example +First we need to define an `llm_config` and define some agents that will partake in the group chat: +```python +from autogen import GroupChat, ConversableAgent, GroupChatManager, UserProxyAgent +from somewhere import MyModelClient + + +# Define the custom model configuration +llm_config = { + "config_list": [ + { + "model": "gpt-3.5-turbo", + "model_client_cls": "MyModelClient" + } + ] +} + +# Initialize the agents with the custom model +agent1 = ConversableAgent( + name="Agent 1", + llm_config=llm_config +) +agent1.register_model_client(model_client_cls=MyModelClient) + +agent2 = ConversableAgent( + name="Agent 2", + llm_config=llm_config +) +agent2.register_model_client(model_client_cls=MyModelClient) + +agent3 = ConversableAgent( + name="Agent 2", + llm_config=llm_config +) +agent3.register_model_client(model_client_cls=MyModelClient) + +user_proxy = UserProxyAgent(name="user", llm_config=llm_config, code_execution_config={"use_docker": False}) +user_proxy.register_model_client(MyModelClient) +``` + +Note that the agents definition illustrated here is minimal and might not suit your needs. The only aim is to show a +basic setup for a group chat scenario. + +We then create a `GroupChat` and, if we want the underlying agents used by GroupChat to use our +custom client, we will pass it in the `model_client_cls` attribute. + +Finally we create an instance of `GroupChatManager` and pass the config to it. This same config will be forwarded to +the GroupChat, that (if needed) will automatically handle registration of custom models only. + +```python +# Create a GroupChat instance and add the agents +group_chat = GroupChat(agents=[agent1, agent2, agent3], messages=[], model_client_cls=MyModelClient) + +# Create the GroupChatManager with the GroupChat, UserProxy, and model configuration +chat_manager = GroupChatManager(groupchat=group_chat, llm_config=llm_config) +chat_manager.register_model_client(model_client_cls=MyModelClient) + +# Initiate the chat using the UserProxy +user_proxy.initiate_chat(chat_manager, initial_message="Suggest me the most trending papers in microbiology that you think might interest me") + +``` + +This attribute can either be a class or a list of classes which adheres to the `ModelClient` protocol (see +[this link](../non-openai-models/about-using-nonopenai-models) for more info about defining a custom model client +class). + +Note that it is not necessary to define a `model_client_cls` when working with Azure OpenAI, OpenAI or other non-custom +models natively supported by the library. diff --git a/website/docs/topics/non-openai-models/cloud-gemini.ipynb b/website/docs/topics/non-openai-models/cloud-gemini.ipynb index 0a36dd62cf9..673d8a80f31 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini.ipynb @@ -495,7 +495,7 @@ "user_proxy.initiate_chat(\n", " image_agent,\n", " message=\"\"\"Describe what is in this image?\n", - ".\"\"\",\n", + ".\"\"\",\n", ")" ] }, diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index 3456a803f48..2389092e8dc 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -392,7 +392,7 @@ "user_proxy.initiate_chat(\n", " image_agent,\n", " message=\"\"\"Describe what is in this image?\n", - ".\"\"\",\n", + ".\"\"\",\n", ")" ] }, diff --git a/website/docs/topics/retrieval_augmentation.md b/website/docs/topics/retrieval_augmentation.md index 7fdd39f31f9..1ac32e85fef 100644 --- a/website/docs/topics/retrieval_augmentation.md +++ b/website/docs/topics/retrieval_augmentation.md @@ -124,10 +124,10 @@ ragproxyagent.initiate_chat( ## More Examples and Notebooks For more detailed examples and notebooks showcasing the usage of retrieval augmented agents in AutoGen, refer to the following: - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](/docs/notebooks/agentchat_RetrieveChat) -- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) -- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) -- Automated Code Generation and Question Answering with [MongoDB Atlas](https://www.mongodb.com/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_mongodb.ipynb) -- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_retrieval.ipynb) +- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_qdrant.ipynb) +- Automated Code Generation and Question Answering with [MongoDB Atlas](https://www.mongodb.com/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_mongodb.ipynb) +- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_retrieval.ipynb) - **RAG**: Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_RAG) ## Roadmap diff --git a/website/docs/tutorial/conversation-patterns.ipynb b/website/docs/tutorial/conversation-patterns.ipynb index 56004e3b3b8..a0c8ddc217c 100644 --- a/website/docs/tutorial/conversation-patterns.ipynb +++ b/website/docs/tutorial/conversation-patterns.ipynb @@ -728,7 +728,7 @@ "If we were to use the `round_robin` strategy, this list would specify the order\n", "of the agents to be selected.\n", "We also initialize the group chat with an empty message list and a maximum\n", - "round of 6, which means there will be at most 6 iteratiosn of selecting speaker,\n", + "round of 6, which means there will be at most 6 iterations of selecting a speaker,\n", "agent speaks and broadcasting message." ] }, diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 1ea8e50aafa..1a2de465f1f 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -16,7 +16,7 @@ module.exports = { title: "AutoGen", tagline: "An Open-Source Programming Framework for Agentic AI", url: "https://microsoft.github.io", - baseUrl: "/autogen/", + baseUrl: "/autogen/0.2/", onBrokenLinks: "throw", onBrokenMarkdownLinks: "warn", favicon: "img/ag.ico", @@ -184,14 +184,14 @@ module.exports = { ], copyright: `Copyright © ${new Date().getFullYear()} AutoGen Authors | Privacy and Cookies | Consumer Health Privacy`, }, - // announcementBar: { - // id: "whats_new", - // content: - // 'What\'s new in AutoGen? Read this blog for an overview of updates', - // backgroundColor: "#fafbfc", - // textColor: "#091E42", - // isCloseable: true, - // }, + announcementBar: { + id: "newdocs", + content: + 'Go here to view the documentation for the work in progress version of AutoGen 0.4.', + backgroundColor: "#fafbfc", + textColor: "#091E42", + isCloseable: true, + }, /* Clarity Config */ clarity: { ID: "lnxpe6skj1", // The Tracking ID provided by Clarity diff --git a/website/process_notebooks.py b/website/process_notebooks.py index 5e25f5e6951..a3157363ab8 100755 --- a/website/process_notebooks.py +++ b/website/process_notebooks.py @@ -348,7 +348,7 @@ def post_process_mdx(rendered_mdx: Path, source_notebooks: Path, front_matter: D if "{" in title: title = title[: title.find("{")].strip() - github_link = f"https://github.com/microsoft/autogen/blob/main/{repo_relative_notebook}" + github_link = f"https://github.com/microsoft/autogen/blob/0.2/{repo_relative_notebook}" content = ( content[:title_end] + "\n[![Open on GitHub](https://img.shields.io/badge/Open%20on%20GitHub-grey?logo=github)](" diff --git a/website/src/components/GalleryPage.js b/website/src/components/GalleryPage.js index 11f4abe2e7f..aec7907b41e 100644 --- a/website/src/components/GalleryPage.js +++ b/website/src/components/GalleryPage.js @@ -91,7 +91,7 @@ const GalleryPage = (props) => { return null; } const colab_href = `https://colab.research.google.com/github/microsoft/autogen/blob/main/${item.source}`; - const github_href = `https://github.com/microsoft/autogen/blob/main/${item.source}`; + const github_href = `https://github.com/microsoft/autogen/blob/0.2/${item.source}`; return ( Open In Colab Static Badge