Ansible Scrolling Output (#38)

* feat: wip - starting on trying to stream output to labyrinth

* chore(debug): added additional debug info

* feat: working scrolling ansible output

* fix(ui): small ui fixes

* chore(cicd): corrected to new docker compose format

* chore: adding break system packages for pip

* chore: adding break system packages for pip

* chore(cicd): corrected the docker names

* chore(cicd): updated docker name

* test: testing for ansible async

* test: updated for changed function

* chore: updated alertmanager to api v2

* Automatic linting fix

.github/workflows/push.yml

@@ -18,11 +18,11 @@ jobs:
         run: docker ps
 
       - name: List Docker logs (backend)
-        run: docker logs labyrinth_backend_1
+        run: docker logs labyrinth-backend-1
 
 
       - name: List Docker logs
-        run: docker logs labyrinth_alertmanager_1
+        run: docker logs labyrinth-alertmanager-1
 
       - name: Copy .env.sample to backend
         run: cp backend/.env.sample backend/.env

alertmanager/Dockerfile

@@ -15,7 +15,7 @@ RUN apt update && apt -y install \
     curl 
 
 ADD requirements.txt /
-RUN pip3 install -r /requirements.txt
+RUN pip3 install -r /requirements.txt --break-system-packages
 
 ENV REPO="prometheus/alertmanager"
 WORKDIR /tmp

alertmanager/test_alert.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-url='http://localhost:9093/api/v1/alerts'
+url='http://localhost:9093/api/v2/alerts'
 name="TEST ALERT"
 echo "firing up alert $name" 
 

alertmanager/test_resolve.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-url='http://localhost:9093/api/v1/alerts'
+url='http://localhost:9093/api/v2/alerts'
 name="TEST ALERT"
 echo "resolving alert $name" 
 

backend/Dockerfile

@@ -27,7 +27,7 @@ RUN echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] h
 RUN apt update && apt -y install telegraf
 
 COPY requirements.txt /requirements.txt
-RUN pip3 install -r /requirements.txt
+RUN pip3 install -r /requirements.txt --break-system-packages
 
 ADD entrypoint.sh /
 RUN dos2unix -n /entrypoint.sh /entrypoint-fixed.sh

backend/alive.py

@@ -47,7 +47,7 @@ def check_port(host, port):
         return False
 
 
-def check_all_hosts():
+def check_all_hosts():  # pragma: no cover
     """
     Pulls in all hosts and checks them all
     """

backend/ansible_helper.py

@@ -185,24 +185,5 @@ def run_ansible(
         f.write(vault_password)
 
     # Run ansible and return HTML
-    try:
-        a = ansible_runner.run(
-            private_data_dir=RUN_DIR,
-            playbook="{}.yml".format(playbook),
-            cmdline="-vvvvv --vault-password-file ../vault.pass",
-        )
-        raise Exception("Done.")
-    except Exception:
-        # Delete Vault Password
-        if "vault.pass" in os.listdir(RUN_DIR):
-            os.remove("{}/vault.pass".format(RUN_DIR))
-
-    if os.path.exists("/vault.pass"):
-        os.remove("/vault.pass")
-
-    z = ansi2html.Ansi2HTMLConverter()
-    output = z.convert("".join(a.stdout))
-    # Delete all files
-    shutil.rmtree(RUN_DIR)
 
-    return output
+    return RUN_DIR, playbook

backend/run_tests.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 CODECOV="95"
-DOCKER_PROD_NAME="labyrinth_backend_1"
+DOCKER_PROD_NAME="labyrinth-backend-1"
 ARGS="$@"
 # Running Python unit tests and coverage
 echo "Running Pytest..."

backend/serve.py

@@ -34,6 +34,8 @@
 from PIL import Image
 from pid import PidFile
 
+
+import ansible_runner
 import ansible_helper
 
 from concurrent.futures import ThreadPoolExecutor
@@ -823,8 +825,12 @@ def list_alerts():
     """
     url = "http://alertmanager:9093/api/v2/alerts"
     password = open("/alertmanager/pass").read()
+    headers = {"Content-Type": "application/json"}
 
-    return json.dumps(requests.get(url, auth=("admin", password)).json()), 200
+    return (
+        json.dumps(requests.get(url, auth=("admin", password), headers=headers).json()),
+        200,
+    )
 
 
 @app.route("/alertmanager/alert", methods=["POST"])
@@ -840,15 +846,17 @@ def resolve_alert(data=""):
     else:  # pragma: no cover
         return "Invalid data", 419
 
-    url = "http://alertmanager:9093/api/v1/alerts"
+    url = "http://alertmanager:9093/api/v2/alerts"
     password = open("/alertmanager/pass").read()
 
     del parsed_data["startsAt"]
     parsed_data["status"] = "resolved"
     parsed_data["endsAt"] = "2021-08-03T14:34:41-05:00"
 
+    headers = {"Content-Type": "application/json"}
+
     retval = requests.post(
-        url, data=json.dumps([parsed_data]), auth=("admin", password)
+        url, data=json.dumps([parsed_data]), auth=("admin", password), headers=headers
     )
 
     return retval.text, retval.status_code
@@ -863,7 +871,9 @@ def restart_alertmanager():
     url = "http://alertmanager:9093/-/reload"
     password = open("/alertmanager/pass").read()
 
-    retval = requests.post(url, auth=("admin", password))
+    headers = {"Content-Type": "application/json"}
+
+    retval = requests.post(url, auth=("admin", password), headers=headers)
     return retval.text, retval.status_code
 
 
@@ -1190,7 +1200,7 @@ def save_ansible_file(fname, inp_data="", vars_file=""):
 # Ansible runner
 @app.route("/ansible_runner/", methods=["POST"])
 @requires_auth_admin
-def run_ansible(inp_data=""):  # pragma: no cover
+def run_ansible(inp_data=""):
     if inp_data != "":
         data = inp_data
     elif request.method == "POST":  # pragma: no cover
@@ -1210,17 +1220,51 @@ def run_ansible(inp_data=""):  # pragma: no cover
     if "ssh_key" not in data:
         data["ssh_key"] = ""
 
-    return (
-        ansible_helper.run_ansible(
-            data["hosts"],
-            data["playbook"],
-            data["vault_password"],
-            data["become_file"],
-            ssh_key_file=data["ssh_key"],
-        ),
-        200,
+    RUN_DIR, playbook = ansible_helper.run_ansible(
+        data["hosts"],
+        data["playbook"],
+        data["vault_password"],
+        data["become_file"],
+        ssh_key_file=data["ssh_key"],
     )
 
+    try:
+        thread, runner = ansible_runner.run_async(
+            private_data_dir=RUN_DIR,
+            playbook="{}.yml".format(playbook),
+            cmdline="-vvvvv --vault-password-file ../vault.pass",
+            quiet=True,
+        )
+    except Exception as e:  # pragma: no cover
+        # Delete Vault Password
+        if "vault.pass" in os.listdir(RUN_DIR):
+            os.remove("{}/vault.pass".format(RUN_DIR))
+
+        if os.path.exists("/vault.pass"):
+            os.remove("/vault.pass")
+        shutil.rmtree(RUN_DIR)
+        return f"Error: {e}", 200
+
+    def ansible_stream():
+        try:
+            while thread.is_alive():
+                try:
+                    for event in runner.events:
+                        yield ("<div>" + str(event["stdout"]) + "</div>").encode(
+                            "utf-8"
+                        )
+                        time.sleep(0.1)
+                except Exception as e:
+                    yield f"Error: {e}".encode("utf-8")
+        finally:
+            if os.path.exists("/vault.pass"):
+                os.remove("/vault.pass")
+
+            # Delete all files
+            shutil.rmtree(RUN_DIR)
+
+    return ansible_stream(), {"Content-Type": "text/plain"}
+
 
 @app.route("/mac/<old_mac>/<new_mac>/")
 @requires_auth_write

backend/test/test_01_alertmanager.py

@@ -83,14 +83,17 @@ def test_send_alert():
         - Will just print out URL and payload
     """
     a = unwrap(serve.list_alerts)()
+    print(a[0])
     assert a[1] == 200
     b = json.loads(a[0])
 
-    assert watcher.send_alert(
+    output = watcher.send_alert(
         "test-alert",
         "test-service",
         "test-host",
     )
+    print(output.text)
+    assert output
     a = unwrap(serve.list_alerts)()
     assert a[1] == 200
 

backend/test/test_07_ansible.py

@@ -213,33 +213,14 @@ def test_run_ansible():
             shutil.copy(src, dest)
     # Check a clean run
 
-    x = run_ansible(
+    x, y = run_ansible(
         hosts="sampleclient",
         playbook="install",
         become_file="vault",
         vault_password="test",
     )
-
-    assert "<style" in x
-    assert "ok:[localhost]" in x.replace(" ", "")
-    assert "body" in x
-
-    assert not os.path.exists("/vault.pass")
-
-    # Check a second run
-
-    x = run_ansible(
-        hosts="sampleclient",
-        playbook="install",
-        become_file="vault",
-        vault_password="test",
-    )
-
-    assert "<style" in x
-    assert "ok:[localhost]" in x.replace(" ", "")
-    assert "body" in x
-
-    assert not os.path.exists("/vault.pass")
+    assert "run" in x
+    assert y == "install"
 
     # Check failing SSH key
     try: