Workflow cleanup (#85)

* fix typo thanks @giddyphysicist

* fix bad logic in is_git

* remove cwd_info from context

* workflow improvements

* always stream the script output
* make terminal output (breaks, spaces, etc) more legible
* remove duplicate prints
* always say "Retrying..." on error

* let it put on / take off its leash

* update version number

examples/simple/cat.py

@@ -9,7 +9,7 @@
     },
     {
         "role": "system",
-        "content": "Today's date is 2024-02-14 13:32:19.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe contents of the current working directory are:\n2024-02-05 07:56:58      11339 bytes LICENSE\n2024-02-14 13:22:51         32 bytes requirements.txt\n2024-02-07 09:47:43          4 items /dist\n2024-02-14 13:22:51        630 bytes pyproject.toml\n2024-02-13 09:38:11      60746 bytes training_data.jsonl\n2024-02-05 07:56:58          1 items /rawdog\n2024-02-13 08:58:21       5247 bytes script_2024-02-13_08-58-21.py\n2024-02-14 08:47:33        308 bytes todo\n2024-02-13 01:40:23       3081 bytes README.md\n2024-02-05 07:57:14         45 bytes .gitignore\n2024-02-07 09:47:28          5 items /.env\n2024-02-14 13:26:01         13 items /examples\n2024-02-13 13:19:10          3 items /scripts\n2024-02-14 09:25:18       3380 bytes octagon.png\n2024-02-14 13:22:51         13 items /.git\n2024-02-09 02:29:04      58388 bytes old_training_data.jsonl\n2024-02-14 08:46:44          4 items /my_env\n2024-02-05 07:56:58          1 items /src\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
+        "content": "Today's date is 2024-02-14 13:32:19.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
     },
     {
         "role": "user",

examples/simple/cwd.py

@@ -9,7 +9,7 @@
     },
     {
         "role": "system",
-        "content": "Today's date is 2024-02-14 13:27:51.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe contents of the current working directory are:\n2024-02-05 07:56:58      11339 bytes LICENSE\n2024-02-14 13:22:51         32 bytes requirements.txt\n2024-02-07 09:47:43          4 items /dist\n2024-02-14 13:22:51        630 bytes pyproject.toml\n2024-02-13 09:38:11      60746 bytes training_data.jsonl\n2024-02-05 07:56:58          1 items /rawdog\n2024-02-13 08:58:21       5247 bytes script_2024-02-13_08-58-21.py\n2024-02-14 08:47:33        308 bytes todo\n2024-02-13 01:40:23       3081 bytes README.md\n2024-02-05 07:57:14         45 bytes .gitignore\n2024-02-07 09:47:28          5 items /.env\n2024-02-14 13:26:01         13 items /examples\n2024-02-13 13:19:10          3 items /scripts\n2024-02-14 09:25:18       3380 bytes octagon.png\n2024-02-14 13:22:51         13 items /.git\n2024-02-09 02:29:04      58388 bytes old_training_data.jsonl\n2024-02-14 08:46:44          4 items /my_env\n2024-02-05 07:56:58          1 items /src\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
+        "content": "Today's date is 2024-02-14 13:27:51.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
     },
     {
         "role": "user",

examples/simple/grep.py

@@ -9,7 +9,7 @@
     },
     {
         "role": "system",
-        "content": "Today's date is 2024-02-14 13:28:17.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe contents of the current working directory are:\n2024-02-05 07:56:58      11339 bytes LICENSE\n2024-02-14 13:22:51         32 bytes requirements.txt\n2024-02-07 09:47:43          4 items /dist\n2024-02-14 13:22:51        630 bytes pyproject.toml\n2024-02-13 09:38:11      60746 bytes training_data.jsonl\n2024-02-05 07:56:58          1 items /rawdog\n2024-02-13 08:58:21       5247 bytes script_2024-02-13_08-58-21.py\n2024-02-14 08:47:33        308 bytes todo\n2024-02-13 01:40:23       3081 bytes README.md\n2024-02-05 07:57:14         45 bytes .gitignore\n2024-02-07 09:47:28          5 items /.env\n2024-02-14 13:26:01         13 items /examples\n2024-02-13 13:19:10          3 items /scripts\n2024-02-14 09:25:18       3380 bytes octagon.png\n2024-02-14 13:22:51         13 items /.git\n2024-02-09 02:29:04      58388 bytes old_training_data.jsonl\n2024-02-14 08:46:44          4 items /my_env\n2024-02-05 07:56:58          1 items /src\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
+        "content": "Today's date is 2024-02-14 13:28:17.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
     },
     {
         "role": "user",

examples/simple/ls.py

@@ -9,7 +9,7 @@
     },
     {
         "role": "system",
-        "content": "Today's date is 2024-02-14 13:26:11.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe contents of the current working directory are:\n2024-02-05 07:56:58      11339 bytes LICENSE\n2024-02-14 13:22:51         32 bytes requirements.txt\n2024-02-07 09:47:43          4 items /dist\n2024-02-14 13:22:51        630 bytes pyproject.toml\n2024-02-13 09:38:11      60746 bytes training_data.jsonl\n2024-02-05 07:56:58          1 items /rawdog\n2024-02-13 08:58:21       5247 bytes script_2024-02-13_08-58-21.py\n2024-02-14 08:47:33        308 bytes todo\n2024-02-13 01:40:23       3081 bytes README.md\n2024-02-05 07:57:14         45 bytes .gitignore\n2024-02-07 09:47:28          5 items /.env\n2024-02-14 13:26:01         13 items /examples\n2024-02-13 13:19:10          3 items /scripts\n2024-02-14 09:25:18       3380 bytes octagon.png\n2024-02-14 13:22:51         13 items /.git\n2024-02-09 02:29:04      58388 bytes old_training_data.jsonl\n2024-02-14 08:46:44          4 items /my_env\n2024-02-05 07:56:58          1 items /src\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
+        "content": "Today's date is 2024-02-14 13:26:11.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
     },
     {
         "role": "user",

examples/simple/math.py

@@ -9,7 +9,7 @@
     },
     {
         "role": "system",
-        "content": "Today's date is 2024-02-14 13:33:57.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe contents of the current working directory are:\n2024-02-05 07:56:58      11339 bytes LICENSE\n2024-02-14 13:22:51         32 bytes requirements.txt\n2024-02-07 09:47:43          4 items /dist\n2024-02-14 13:22:51        630 bytes pyproject.toml\n2024-02-13 09:38:11      60746 bytes training_data.jsonl\n2024-02-05 07:56:58          1 items /rawdog\n2024-02-13 08:58:21       5247 bytes script_2024-02-13_08-58-21.py\n2024-02-14 08:47:33        308 bytes todo\n2024-02-13 01:40:23       3081 bytes README.md\n2024-02-05 07:57:14         45 bytes .gitignore\n2024-02-07 09:47:28          5 items /.env\n2024-02-14 13:26:01         13 items /examples\n2024-02-13 13:19:10          3 items /scripts\n2024-02-14 09:25:18       3380 bytes octagon.png\n2024-02-14 13:22:51         13 items /.git\n2024-02-09 02:29:04      58388 bytes old_training_data.jsonl\n2024-02-14 08:46:44          4 items /my_env\n2024-02-05 07:56:58          1 items /src\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
+        "content": "Today's date is 2024-02-14 13:33:57.\nThe current working directory is /Users/jakekoenig/rawdog, which IS a git repository.\nThe user's operating system is Darwin.\nThe last commit message is: Pip install not found modules (#61)\n\nIf the llm tries to use a module that cannot be found then prompt the\r\nuser to install it with pip.\r\nMisc:\r\n* Moved execution specific utils from utils to execute_script\r\n* Removed almost all the requirements\r\n* Give errors from the script to the llm (was broken by move to subprocess)\r\n\r\nCo-authored-by: biobootloader <[email protected]>\r\n\r\n---------\r\n\r\nCo-authored-by: biobootloader <[email protected]>"
     },
     {
         "role": "user",

pyproject.toml

@@ -7,7 +7,7 @@ packages=["src/rawdog"]
 
 [project]
 name = "rawdog-ai"
-version = "0.1.5"
+version = "0.1.6"
 description = "An AI command-line assistant"
 readme = "README.md"
 dependencies = [

src/rawdog/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.5"
+__version__ = "0.1.6"

src/rawdog/__main__.py

@@ -9,48 +9,42 @@
 
 
 def rawdog(prompt: str, config, llm_client):
+    llm_client.add_message("user", prompt)
     leash = config.get("leash")
     retries = int(config.get("retries"))
     _continue = True
-    _first = True
     while _continue is True:
+        _continue = False
         error, script, output, return_code = "", "", "", 0
         try:
-            if _first:
-                message, script = llm_client.get_script(prompt, stream=leash)
-                _first = False
-            else:
-                message, script = llm_client.get_script(stream=leash)
+            if leash:
+                print(80 * "-")
+            message, script = llm_client.get_script()
             if script:
                 if leash:
-                    print(f"\n{80 * '-'}")
-                    if (
-                        input("Execute script in markdown block? (Y/n): ")
-                        .strip()
-                        .lower()
-                        == "n"
-                    ):
+                    _ok = input(
+                        f"\n{38 * '-'} Execute script in markdown block? (Y/n):"
+                    )
+                    if _ok.strip().lower() == "n":
                         llm_client.add_message("user", "User chose not to run script")
                         break
                 output, error, return_code = execute_script(script, llm_client)
-            elif message:
+            elif not leash and message:
                 print(message)
         except KeyboardInterrupt:
             break
 
-        _continue = (output and output.strip().endswith("CONTINUE")) or (
-            return_code != 0 and error and retries > 0
-        )
-        if error:
-            retries -= 1
-            llm_client.add_message("user", f"Error: {error}")
-            print(f"Error: {error}")
-            if script and not leash:
-                print(f"{80 * '-'}\n{script}\n{80 * '-'}")
         if output:
             llm_client.add_message("user", f"LAST SCRIPT OUTPUT:\n{output}")
-            if leash or not _continue:
-                print(output)
+            if output.endswith("CONTINUE"):
+                _continue = True
+        if error:
+            llm_client.add_message("user", f"Error: {error}")
+        if return_code != 0:
+            retries -= 1
+            if retries > 0:
+                print("Retrying...\n")
+                _continue = True
 
 
 def banner(config):

src/rawdog/execute_script.py

@@ -36,39 +36,71 @@ def install_pip_packages(*packages: str):
     )
 
 
-def execute_script(script: str, llm_client) -> tuple[str, str, int]:
-    python_executable = get_rawdog_python_executable()
-    with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp_script:
-        tmp_script_name = tmp_script.name
-        tmp_script.write(script)
-        tmp_script.flush()
+def _execute_script_in_subprocess(script) -> tuple[str, str, int]:
+    """Write script to tempfile, execute from .rawdog/venv, stream and return output"""
+    output, error, return_code = "", "", 0
+    try:
+        python_executable = get_rawdog_python_executable()
+        with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp_script:
+            tmp_script_name = tmp_script.name
+            tmp_script.write(script)
+            tmp_script.flush()
 
-        retry = True
-        while retry:
-            retry = False
-            result = subprocess.run(
-                [python_executable, tmp_script_name], capture_output=True, text=True
+            process = subprocess.Popen(
+                [python_executable, tmp_script_name],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                stdin=subprocess.DEVNULL,  # Raises EOF error if subprocess asks for input
+                text=True,
             )
-            output = result.stdout
-            error = result.stderr
-            return_code = result.returncode
-            if error and "ModuleNotFoundError: No module named" in error:
-                match = re.search(r"No module named '(\w+)'", error)
-                if match:
-                    module = match.group(1)
-                    module_name = llm_client.get_python_package(module)
-                    if (
-                        input(
-                            f"Rawdog wants to use {module_name}. Install to rawdog's"
-                            " venv with pip? (Y/n): "
-                        )
-                        .strip()
-                        .lower()
-                        != "n"
-                    ):
-                        install_result = install_pip_packages(module_name)
-                        if install_result.returncode == 0:
-                            retry = True
-                        else:
-                            print("Failed to install package")
+            while True:
+                _stdout = process.stdout.readline()
+                _stderr = process.stderr.readline()
+                if _stdout:
+                    output += _stdout
+                    print(_stdout, end="")
+                if _stderr:
+                    error += _stderr
+                    print(_stderr, end="", file=sys.stderr)
+                if _stdout == "" and _stderr == "" and process.poll() is not None:
+                    break
+            return_code = process.returncode
+    except Exception as e:
+        error += str(e)
+        print(e)
+        return_code = 1
+    return output, error, return_code
+
+
+def _execute_script_with_dependency_resolution(
+    script, llm_client
+) -> tuple[str, str, int]:
+    retry = True
+    output, error, return_code = "", "", 0
+    while retry:
+        retry = False
+        output, error, return_code = _execute_script_in_subprocess(script)
+        if error and "ModuleNotFoundError: No module named" in error:
+            match = re.search(r"No module named '(\w+)'", error)
+            if match:
+                module = match.group(1)
+                module_name = llm_client.get_python_package(module)
+                if (
+                    input(
+                        f"Rawdog wants to use {module_name}. Install to rawdog's"
+                        " venv with pip? (Y/n): "
+                    )
+                    .strip()
+                    .lower()
+                    != "n"
+                ):
+                    install_result = install_pip_packages(module_name)
+                    if install_result.returncode == 0:
+                        retry = True
+                    else:
+                        print("Failed to install package")
     return output, error, return_code
+
+
+def execute_script(script: str, llm_client) -> tuple[str, str, int]:
+    return _execute_script_with_dependency_resolution(script, llm_client)