Linter tools (#50)

* add PgSqlLinterTool

* add pglast in poetry

* add PgSqlLinterTool in motleycrew.tools

* add PythonLinterTool

* Minor refactoring of linter tools

* fixes

* cancel older concurrent workflows

---------

Co-authored-by: User <[email protected]>
Co-authored-by: whimo <[email protected]>

.github/workflows/build.yml

@@ -8,6 +8,16 @@ on:
     branches:
       - main
 
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     strategy:

.github/workflows/integration_test.yml

@@ -8,6 +8,16 @@ on:
     branches:
       - main
 
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     strategy:
@@ -41,10 +51,6 @@ jobs:
         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
-      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      run: poetry install --no-interaction --no-root
-
-    - name: Install project
       run: poetry install --no-interaction --all-extras
 
     - name: Run integration tests

.github/workflows/integration_test_minimal.yml

@@ -8,6 +8,16 @@ on:
     branches:
       - main
 
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     strategy:
@@ -41,10 +51,6 @@ jobs:
         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
-      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      run: poetry install --no-interaction --no-root
-
-    - name: Install project
       run: poetry install --no-interaction
 
     - name: Run integration tests

.github/workflows/publish.yml

@@ -38,10 +38,6 @@ jobs:
         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
-      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      run: poetry install --no-interaction --no-root
-
-    - name: Install project
       run: poetry install --no-interaction
 
     - name: Run tests

.github/workflows/test.yml

@@ -8,6 +8,16 @@ on:
     branches:
       - main
 
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     strategy:
@@ -41,11 +51,7 @@ jobs:
         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
-      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      run: poetry install --no-interaction --no-root
-
-    - name: Install project
-      run: poetry install --no-interaction --extras "crewai llama-index"
+      run: poetry install --no-interaction --all-extras --with dev
 
     - name: Run tests
       run: poetry run pytest

examples/aider_code_generation.py

@@ -8,10 +8,10 @@
 
 from langchain_community.tools import ShellTool
 from motleycrew.agents.crewai import CrewAIMotleyAgent
-from motleycrew.common import configure_logging, AsyncBackend
+from motleycrew.common import configure_logging
 from motleycrew.tasks import SimpleTask
 from motleycache import logger
-from motleycrew.tools.aider_tool import AiderTool
+from motleycrew.tools.code.aider_tool import AiderTool
 
 
 logger.setLevel(logging.INFO)

motleycrew/common/defaults.py

@@ -28,7 +28,10 @@ class Defaults:
         "crewai": "pip install crewai",
         "llama_index": "pip install llama-index",
         "autogen": "pip install pyautogen",
-        "lunary": "pip install lunary"
+        "lunary": "pip install lunary",
+        "aider": "pip install aider-chat",
+        "pglast": "pip install pglast",
     }
+
     DEFAULT_NUM_THREADS = 4
     DEFAULT_EVENT_LOOP_SLEEP = 1

motleycrew/tools/__init__.py

@@ -1,7 +1,9 @@
 from .tool import MotleyTool
 
 from .autogen_chat_tool import AutoGenChatTool
-from motleycrew.tools.image.dall_e import DallEImageGeneratorTool
+from .image.dall_e import DallEImageGeneratorTool
 from .llm_tool import LLMTool
 from .mermaid_evaluator_tool import MermaidEvaluatorTool
 from .python_repl import PythonREPLTool
+from .code.postgresql_linter import PostgreSQLLinterTool
+from .code.python_linter import PythonLinterTool

motleycrew/tools/aider_tool.py → motleycrew/tools/code/aider_tool.py

@@ -23,7 +23,7 @@ def __init__(self, model: str = None, **kwargs):
             model (str): model name
             **kwargs:
         """
-        ensure_module_is_installed("aider", "pip install aider-chat")
+        ensure_module_is_installed("aider")
 
         model = model or Defaults.DEFAULT_LLM_NAME
         llm_model = Model(model=model)
@@ -44,7 +44,7 @@ class AiderToolInput(BaseModel):
 
 
 def create_aider_tool(coder: Coder):
-    """Create langchain tool from Aider coder.run() method
+    """Create langchain tool from Aider Coder.run() method
 
     Returns:
         Tool:

motleycrew/tools/code/postgresql_linter.py

@@ -0,0 +1,55 @@
+from langchain_core.tools import Tool
+from langchain_core.pydantic_v1 import BaseModel, Field
+
+try:
+    from pglast import parse_sql, prettify
+    from pglast.parser import ParseError
+except ImportError:
+    parse_sql = None
+    prettify = None
+    ParseError = None
+
+from motleycrew.tools import MotleyTool
+from motleycrew.common.utils import ensure_module_is_installed
+
+
+class PostgreSQLLinterTool(MotleyTool):
+
+    def __init__(self):
+        """PostgreSQL code verification tool
+        """
+        ensure_module_is_installed("pglast")
+
+        langchain_tool = create_pgsql_linter_tool()
+        super().__init__(langchain_tool)
+
+
+class PostgreSQLLinterInput(BaseModel):
+    """Input for the PostgreSQLLinterTool.
+
+    Attributes:
+        query (str):
+    """
+
+    query: str = Field(description="SQL code for validation")
+
+
+def create_pgsql_linter_tool() -> Tool:
+    """Create the underlying langchain tool for PostgreSQLLinterTool
+
+    Returns:
+        Tool:
+    """
+    def parse_func(query: str) -> str:
+        try:
+            parse_sql(query)
+            return prettify(query)
+        except ParseError as e:
+            return str(e)
+
+    return Tool.from_function(
+        func=parse_func,
+        name="PostgreSQL linter tool",
+        description="Tool for validating PostgreSQL code",
+        args_schema=PostgreSQLLinterInput,
+    )

motleycrew/tools/code/python_linter.py

@@ -0,0 +1,67 @@
+import os
+from typing import Union
+
+from langchain_core.tools import StructuredTool
+from langchain_core.pydantic_v1 import BaseModel, Field
+
+from motleycrew.common.utils import ensure_module_is_installed
+from motleycrew.tools import MotleyTool
+
+Linter = None
+
+
+class PythonLinterTool(MotleyTool):
+
+    def __init__(self):
+        """Python code verification tool
+        """
+        ensure_module_is_installed("aider")
+
+        langchain_tool = create_python_linter_tool()
+        super().__init__(langchain_tool)
+
+
+class PythonLinterInput(BaseModel):
+    """Input for the PythonLinterTool.
+
+    Attributes:
+        code (str):
+        file_name (str):
+    """
+
+    code: str = Field(description="Python code for linting")
+    file_name: str = Field(description="file name for the code", default="code.py")
+
+
+def create_python_linter_tool() -> StructuredTool:
+    """Create the underlying langchain tool for PythonLinterTool
+
+    Returns:
+        Tool:
+    """
+
+    def lint(code: str, file_name: str = None) -> Union[str, None]:
+        # create temp python file
+        temp_file_name = file_name or "code.py"
+        _, file_ext = os.path.splitext(temp_file_name)
+        if file_ext != ".py":
+            raise ValueError("The file extension must be .py")
+
+        with open(temp_file_name, 'w') as f:
+            f.write(code)
+
+        # lint code
+        try:
+            linter = Linter()
+            return linter.lint(temp_file_name)
+        except Exception as e:
+            return str(e)
+        finally:
+            os.remove(temp_file_name)
+
+    return StructuredTool.from_function(
+        func=lint,
+        name="python linter tool",
+        description="Tool for validating Python code",
+        args_schema=PythonLinterInput,
+    )

motleycrew/tools/tool.py

@@ -1,6 +1,6 @@
 """ Module description """
 
-from typing import Union, Annotated, Optional, Dict, Any
+from typing import Union, Optional, Dict, Any
 from typing import Callable
 
 from langchain.tools import BaseTool