Disable strong cache in integration tests

.github/workflows/integration_test.yml

@@ -8,56 +8,26 @@ on:
     branches:
       - main
 
-# If another push to the same PR or branch happens while this workflow is still running,
-# cancel the earlier run in favor of the next run.
-#
-# There's no point in testing an outdated version of the code. GitHub only allows
-# a limited number of job runners to be active at the same time, so it's better to cancel
-# pointless jobs early so that more useful jobs can run sooner.
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  test:
+  pre-test:
+    # default smoke test against python-3.10 on ubuntu-latest
+    uses: ./.github/workflows/integration_test_workflow.yml
+    secrets:
+      openai-api-key: ${{ secrets.OPENAI_API_KEY }}
+
+  integration-test:
+    needs: pre-test
     strategy:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python ${{ matrix.python-version }}
-      id: setup-python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install poetry
-      run: pip install -U poetry
-
-    - name: Configure poetry
-      run: |
-        poetry config virtualenvs.create true
-        poetry config virtualenvs.in-project true
-        poetry config installer.parallel true
-
-    - name: Load cached venv
-      id: cached-poetry-dependencies
-      uses: actions/cache@v4
-      with:
-        path: .venv
-        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-    - name: Install extra dependencies
-      run: poetry run pip install -r requirements-extra.txt
-
-    - name: Install dependencies
-      run: poetry install --no-interaction --all-extras
-
-    - name: Run integration tests
-      env:
-        OPENAI_API_KEY: fake_key_1337
-        TIKTOKEN_CACHE_DIR: tests/itest_cache/tiktoken_cache
-      run: poetry run python tests/run_integration_tests.py
+    uses: ./.github/workflows/integration_test_workflow.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+    secrets:
+      openai-api-key: ${{ secrets.OPENAI_API_KEY }}

.github/workflows/integration_test_minimal.yml

@@ -8,56 +8,20 @@ on:
     branches:
       - main
 
-# If another push to the same PR or branch happens while this workflow is still running,
-# cancel the earlier run in favor of the next run.
-#
-# There's no point in testing an outdated version of the code. GitHub only allows
-# a limited number of job runners to be active at the same time, so it's better to cancel
-# pointless jobs early so that more useful jobs can run sooner.
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  test:
+  minimal-test:
     strategy:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python ${{ matrix.python-version }}
-      id: setup-python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install poetry
-      run: pip install -U poetry
-
-    - name: Configure poetry
-      run: |
-        poetry config virtualenvs.create true
-        poetry config virtualenvs.in-project true
-        poetry config installer.parallel true
-
-    - name: Load cached venv
-      id: cached-poetry-dependencies
-      uses: actions/cache@v4
-      with:
-        path: .venv
-        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-    - name: Install extra dependencies
-      run: poetry run pip install -r requirements-extra.txt
-
-    - name: Install dependencies
-      run: poetry install --no-interaction
-
-    - name: Run integration tests
-      env:
-        OPENAI_API_KEY: fake_key_1337
-        TIKTOKEN_CACHE_DIR: tests/itest_cache/tiktoken_cache
-      run: poetry run python tests/run_integration_tests.py --minimal-only
+    uses: ./.github/workflows/integration_test_workflow.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+      minimal-only: true
+    secrets:
+      openai-api-key: ${{ secrets.OPENAI_API_KEY }}

.github/workflows/integration_test_workflow.yml

@@ -0,0 +1,81 @@
+name: Integration test workflow
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        default: ubuntu-latest
+        type: string
+      python-version:
+        default: "3.12"
+        type: string
+      minimal-only:
+        default: false
+        type: boolean
+    secrets:
+      openai-api-key:
+        required: true
+
+
+jobs:
+  test:
+    environment: integration_test
+    runs-on: ${{ inputs.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ inputs.python-version }}
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Install poetry
+        run: pip install -U poetry
+
+      - name: Configure poetry
+        run: |
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+          poetry config installer.parallel true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Load test cache
+        uses: actions/cache@v4
+        with:
+          path: tests/itest_cache
+          enableCrossOsArchive: true
+          key: integration-tests-cache
+          restore-keys: |
+            integration-tests-cache
+
+      - name: Install extra dependencies
+        run: poetry run pip install -r requirements-extra.txt
+
+      - name: Install minimal dependencies
+        if: ${{ inputs.minimal-only }}
+        run: poetry install --no-interaction
+
+      - name: Install dependencies
+        if: ${{ !inputs.minimal-only }}
+        run: poetry install --no-interaction --all-extras
+
+      - name: Run minimal integration tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.openai-api-key }}
+          TIKTOKEN_CACHE_DIR: tests/itest_cache/tiktoken_cache
+        if: ${{ inputs.minimal-only }}
+        run: poetry run python tests/run_integration_tests.py --minimal-only --os ${{ runner.os }}
+
+      - name: Run integration tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.openai-api-key }}
+          TIKTOKEN_CACHE_DIR: tests/itest_cache/tiktoken_cache
+        if: ${{ !inputs.minimal-only }}
+        run: poetry run python tests/run_integration_tests.py --os ${{ runner.os }}

examples/Advanced output handling.ipynb

@@ -17,9 +17,25 @@
     "This should give a good understanding of output handler semantics and prepare you to build your own ones."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Install lunary package if you need logging."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install lunary==1.1.5"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -860,8 +876,6 @@
       "\n",
       "Let's proceed to return the final result.\n",
       "\n",
-      "\n",
-      "\n",
       "\u001b[0mCritic response:  OK\n",
       "\u001b[32;1m\u001b[1;3m('def bubble_sort(array):\\n    \"\"\"\\n    Sorts an array using the bubble sort algorithm.\\n\\n    The bubble sort algorithm works by repeatedly stepping through the list to be sorted,\\n    comparing each pair of adjacent items and swapping them if they are in the wrong order.\\n    This process is repeated until the list is sorted. The algorithm gets its name because\\n    smaller elements \\'bubble\\' to the top of the list.\\n\\n    Parameters:\\n    array (list): The list of elements to be sorted.\\n\\n    Returns:\\n    list: The sorted list.\\n\\n    Time Complexity:\\n    - Worst and Average Case: O(n^2)\\n    - Best Case: O(n) when the list is already sorted\\n\\n    Space Complexity:\\n    - O(1) as it is an in-place sorting algorithm\\n    \"\"\"\\n    length = len(array)\\n    for i in range(length):\\n        swapped = False\\n        for j in range(0, length - i - 1):\\n            if array[j] > array[j + 1]:\\n                # Swap if the element found is greater than the next element\\n                array[j], array[j + 1] = array[j + 1], array[j]\\n                swapped = True\\n        # If no elements were swapped, the array is already sorted\\n        if not swapped:\\n            break\\n    return array\\n\\n# Test the bubble sort function with various cases\\ntest_cases = [\\n    [64, 34, 25, 12, 22, 11, 90],\\n    [],\\n    [1],\\n    [2, 1],\\n    [3, 3, 3],\\n    [5, 1, 4, 2, 8],\\n    [-1, -3, -2, -5, -4],\\n    [10, 9, 8, 7, 6, 5, 4, 3, 2, 1],\\n    [7, 7, 7, 7, 7]\\n]\\n\\nfor i, sample_list in enumerate(test_cases):\\n    sorted_list = bubble_sort(sample_list.copy())\\n    print(f\"Test case {i + 1}: Original: {sample_list}, Sorted: {sorted_list}\")', \"The bubble sort algorithm works by repeatedly stepping through the list to be sorted, comparing each pair of adjacent items and swapping them if they are in the wrong order. This process is repeated until the list is sorted. The algorithm gets its name because smaller elements 'bubble' to the top of the list. The outer loop runs n times, and the inner loop runs n-i-1 times, where n is the length of the list and i is the current iteration of the outer loop. An optimization is added with a flag to detect if any swapping happened in the inner loop. If no swapping happened, the list is already sorted, and the loop can be terminated early. The time complexity of bubble sort is O(n^2) in the worst and average cases, and O(n) in the best case when the list is already sorted. The space complexity is O(1) as it is an in-place sorting algorithm.\")\u001b[0m\n",
       "\n",
@@ -1183,9 +1197,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.2"
+   "version": "3.12.1"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

examples/Blog with Images.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "16b8ccc4-b34b-47c3-94ee-cbd82ef4c8f2",
+   "id": "bf3534fb-ed51-4a82-ad66-71259b2e282f",
    "metadata": {},
    "source": [
     "We use this example of creating a web search-based blog with illustrations to show how to \n",
@@ -20,9 +20,32 @@
     "* create tasks and link them using the >> operator"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "6ab41b51-2064-4623-866b-f4310d062425",
+   "metadata": {},
+   "source": [
+    "Let's install external dependencies for the example."
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
+   "id": "e8d44edb-4841-4f3c-822c-136276606e09",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install duckduckgo-search\n",
+    "%pip install llama-index\n",
+    "%pip install crewai==0.51.1 --no-deps\n",
+    "\n",
+    "# install lunary if you need logging\n",
+    "%pip install lunary==1.1.5"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
    "id": "2596164c",
    "metadata": {},
    "outputs": [
@@ -84,7 +107,7 @@
    "source": [
     "search_tool = MotleyTool.from_supported_tool(\n",
     "    DuckDuckGoSearchRun(),\n",
-    "    retry_config=RetryConfig(max_retries=5)  # for retrying rate limit errors\n",
+    "    retry_config=RetryConfig(max_retries=7)  # for retrying rate limit errors\n",
     ")\n",
     "\n",
     "researcher = CrewAIMotleyAgent(\n",
@@ -518,7 +541,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.2"
+   "version": "3.12.6"
   }
  },
  "nbformat": 4,

examples/Math via python code with a single agent.ipynb

@@ -16,9 +16,30 @@
     "The point of this example is to illustrate that for agents that have an AgentExecutor (which most frameworks have apart from Autogen), the approach from [MathChat](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat/) can be done with a single agent, as the second conversation partner there just plays the role of an AgentExecutor."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "92b1d63b-b288-46f5-8eb7-19ee56853e94",
+   "metadata": {},
+   "source": [
+    "Let's install external dependencies for the example."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e5fc0be9-1d35-4902-9a1e-bd19bb119f4a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install crewai==0.51.1 --no-deps\n",
+    "\n",
+    "# install lunary if you need logging\n",
+    "%pip install lunary==1.1.5"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "id": "51527a55-d470-4509-9930-01466e3f78f3",
    "metadata": {},
    "outputs": [
@@ -772,8 +793,7 @@
    ],
    "source": [
     "from IPython.display import display, Math, Latex\n",
-    "for line in task.output.split(\"\\n\"):\n",
-    "    display(Latex(line))"
+    "display(Latex(task.output))"
    ]
   },
   {
@@ -803,7 +823,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.10"
+   "version": "3.12.6"
   }
  },
  "nbformat": 4,

examples/Multi-step research agent.ipynb

@@ -254,7 +254,7 @@
       "Question:  Why did Arjuna kill Karna, his half-brother?\n",
       "Answer:  Arjuna killed Karna, his half-brother, during their duel on the battlefield of Kurukshetra. The pivotal moment came when Karna's chariot got stuck in the mud, and he requested Arjuna and Krishna to honorably allow him to free his chariot. Krishna responded by reminding Karna of his past dishonorable actions, such as humiliating Draupadi and killing Abhimanyu, thereby justifying Arjuna's actions against Karna. Additionally, when Karna attempted to use the Brahmastra, he suddenly forgot the mantra, which can be seen as divine intervention. Krishna's words and the subsequent events reinforced Arjuna's resolve to kill Karna, aligning with his duty and dharma as a warrior.\n",
       "To explore the graph:\n",
-      "docker run -p 8000:8000  -v /var/folders/fv/tyhll76x0fn6l7j_q2nhvyg00000gn/T/tmp4obi9n3p/kuzu_db:/database --rm kuzudb/explorer:latest\n",
+      "docker run -p 8000:8000  -v /tmp/tmp5gpyhq4x/kuzu_db:/database --rm kuzudb/explorer:0.4.2\n",
       "And in the kuzu explorer at http://localhost:8000 enter\n",
       "MATCH (A)-[r]->(B) RETURN *;\n"
      ]
@@ -266,7 +266,7 @@
     "print(\"Question: \", final_answer.question)\n",
     "print(\"Answer: \", final_answer.answer)\n",
     "print(\"To explore the graph:\")\n",
-    "print(f\"docker run -p 8000:8000  -v {crew.graph_store.database_path}:/database --rm kuzudb/explorer:latest\")\n",
+    "print(f\"docker run -p 8000:8000  -v {crew.graph_store.database_path}:/database --rm kuzudb/explorer:0.4.2\")\n",
     "print(\"And in the kuzu explorer at http://localhost:8000 enter\")\n",
     "print(\"MATCH (A)-[r]->(B) RETURN *;\")\n",
     "final_result = \"{}\\n\\n{}\".format(final_answer.question, final_answer.answer)"