From 0b3fe3042f76cc68f038b32f0aa62d2452584c84 Mon Sep 17 00:00:00 2001 From: Chester Enright <30327507+amunchet@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:20:20 -0600 Subject: [PATCH] Ansible Scrolling Output (#38) * feat: wip - starting on trying to stream output to labyrinth * chore(debug): added additional debug info * feat: working scrolling ansible output * fix(ui): small ui fixes * chore(cicd): corrected to new docker compose format * chore: adding break system packages for pip * chore: adding break system packages for pip * chore(cicd): corrected the docker names * chore(cicd): updated docker name * test: testing for ansible async * test: updated for changed function * chore: updated alertmanager to api v2 * Automatic linting fix --- .github/workflows/push.yml | 4 +- alertmanager/Dockerfile | 2 +- alertmanager/test_alert.sh | 2 +- alertmanager/test_resolve.sh | 2 +- backend/Dockerfile | 2 +- backend/alive.py | 2 +- backend/ansible_helper.py | 21 +- backend/run_tests.sh | 2 +- backend/serve.py | 72 ++++- backend/test/test_01_alertmanager.py | 5 +- backend/test/test_07_ansible.py | 25 +- backend/test/test_07_ansible_async.py | 193 +++++++++++++ backend/watcher.py | 11 +- cron/Dockerfile | 2 +- .../src/components/CreateEditHost.vue | 6 +- frontend/labyrinth/src/helper.js | 49 +++- frontend/labyrinth/src/main.js | 4 +- frontend/labyrinth/src/views/Deploy.vue | 262 +++++++++--------- frontend/labyrinth/src/views/Services.vue | 31 ++- start_dev.sh | 4 +- 20 files changed, 470 insertions(+), 231 deletions(-) create mode 100644 backend/test/test_07_ansible_async.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 94082e15..f9410a1e 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -18,11 +18,11 @@ jobs: run: docker ps - name: List Docker logs (backend) - run: docker logs labyrinth_backend_1 + run: docker logs labyrinth-backend-1 - name: List Docker logs - run: docker logs labyrinth_alertmanager_1 + run: docker logs labyrinth-alertmanager-1 - name: Copy .env.sample to backend run: cp backend/.env.sample backend/.env diff --git a/alertmanager/Dockerfile b/alertmanager/Dockerfile index 4c7670a8..d541cce0 100644 --- a/alertmanager/Dockerfile +++ b/alertmanager/Dockerfile @@ -15,7 +15,7 @@ RUN apt update && apt -y install \ curl ADD requirements.txt / -RUN pip3 install -r /requirements.txt +RUN pip3 install -r /requirements.txt --break-system-packages ENV REPO="prometheus/alertmanager" WORKDIR /tmp diff --git a/alertmanager/test_alert.sh b/alertmanager/test_alert.sh index 9b29f668..cd4baf68 100644 --- a/alertmanager/test_alert.sh +++ b/alertmanager/test_alert.sh @@ -1,5 +1,5 @@ #!/bin/sh -url='http://localhost:9093/api/v1/alerts' +url='http://localhost:9093/api/v2/alerts' name="TEST ALERT" echo "firing up alert $name" diff --git a/alertmanager/test_resolve.sh b/alertmanager/test_resolve.sh index e16bc522..8544431e 100644 --- a/alertmanager/test_resolve.sh +++ b/alertmanager/test_resolve.sh @@ -1,5 +1,5 @@ #!/bin/sh -url='http://localhost:9093/api/v1/alerts' +url='http://localhost:9093/api/v2/alerts' name="TEST ALERT" echo "resolving alert $name" diff --git a/backend/Dockerfile b/backend/Dockerfile index 174f82b0..1b79ae71 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ RUN echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] h RUN apt update && apt -y install telegraf COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt +RUN pip3 install -r /requirements.txt --break-system-packages ADD entrypoint.sh / RUN dos2unix -n /entrypoint.sh /entrypoint-fixed.sh diff --git a/backend/alive.py b/backend/alive.py index 239b59db..a778f623 100644 --- a/backend/alive.py +++ b/backend/alive.py @@ -47,7 +47,7 @@ def check_port(host, port): return False -def check_all_hosts(): +def check_all_hosts(): # pragma: no cover """ Pulls in all hosts and checks them all """ diff --git a/backend/ansible_helper.py b/backend/ansible_helper.py index a8e32f67..114fdc2a 100644 --- a/backend/ansible_helper.py +++ b/backend/ansible_helper.py @@ -185,24 +185,5 @@ def run_ansible( f.write(vault_password) # Run ansible and return HTML - try: - a = ansible_runner.run( - private_data_dir=RUN_DIR, - playbook="{}.yml".format(playbook), - cmdline="-vvvvv --vault-password-file ../vault.pass", - ) - raise Exception("Done.") - except Exception: - # Delete Vault Password - if "vault.pass" in os.listdir(RUN_DIR): - os.remove("{}/vault.pass".format(RUN_DIR)) - - if os.path.exists("/vault.pass"): - os.remove("/vault.pass") - - z = ansi2html.Ansi2HTMLConverter() - output = z.convert("".join(a.stdout)) - # Delete all files - shutil.rmtree(RUN_DIR) - return output + return RUN_DIR, playbook diff --git a/backend/run_tests.sh b/backend/run_tests.sh index 61145063..96fb239f 100644 --- a/backend/run_tests.sh +++ b/backend/run_tests.sh @@ -1,6 +1,6 @@ #!/bin/sh CODECOV="95" -DOCKER_PROD_NAME="labyrinth_backend_1" +DOCKER_PROD_NAME="labyrinth-backend-1" ARGS="$@" # Running Python unit tests and coverage echo "Running Pytest..." diff --git a/backend/serve.py b/backend/serve.py index 2976deb3..60ada6af 100755 --- a/backend/serve.py +++ b/backend/serve.py @@ -34,6 +34,8 @@ from PIL import Image from pid import PidFile + +import ansible_runner import ansible_helper from concurrent.futures import ThreadPoolExecutor @@ -823,8 +825,12 @@ def list_alerts(): """ url = "http://alertmanager:9093/api/v2/alerts" password = open("/alertmanager/pass").read() + headers = {"Content-Type": "application/json"} - return json.dumps(requests.get(url, auth=("admin", password)).json()), 200 + return ( + json.dumps(requests.get(url, auth=("admin", password), headers=headers).json()), + 200, + ) @app.route("/alertmanager/alert", methods=["POST"]) @@ -840,15 +846,17 @@ def resolve_alert(data=""): else: # pragma: no cover return "Invalid data", 419 - url = "http://alertmanager:9093/api/v1/alerts" + url = "http://alertmanager:9093/api/v2/alerts" password = open("/alertmanager/pass").read() del parsed_data["startsAt"] parsed_data["status"] = "resolved" parsed_data["endsAt"] = "2021-08-03T14:34:41-05:00" + headers = {"Content-Type": "application/json"} + retval = requests.post( - url, data=json.dumps([parsed_data]), auth=("admin", password) + url, data=json.dumps([parsed_data]), auth=("admin", password), headers=headers ) return retval.text, retval.status_code @@ -863,7 +871,9 @@ def restart_alertmanager(): url = "http://alertmanager:9093/-/reload" password = open("/alertmanager/pass").read() - retval = requests.post(url, auth=("admin", password)) + headers = {"Content-Type": "application/json"} + + retval = requests.post(url, auth=("admin", password), headers=headers) return retval.text, retval.status_code @@ -1190,7 +1200,7 @@ def save_ansible_file(fname, inp_data="", vars_file=""): # Ansible runner @app.route("/ansible_runner/", methods=["POST"]) @requires_auth_admin -def run_ansible(inp_data=""): # pragma: no cover +def run_ansible(inp_data=""): if inp_data != "": data = inp_data elif request.method == "POST": # pragma: no cover @@ -1210,17 +1220,51 @@ def run_ansible(inp_data=""): # pragma: no cover if "ssh_key" not in data: data["ssh_key"] = "" - return ( - ansible_helper.run_ansible( - data["hosts"], - data["playbook"], - data["vault_password"], - data["become_file"], - ssh_key_file=data["ssh_key"], - ), - 200, + RUN_DIR, playbook = ansible_helper.run_ansible( + data["hosts"], + data["playbook"], + data["vault_password"], + data["become_file"], + ssh_key_file=data["ssh_key"], ) + try: + thread, runner = ansible_runner.run_async( + private_data_dir=RUN_DIR, + playbook="{}.yml".format(playbook), + cmdline="-vvvvv --vault-password-file ../vault.pass", + quiet=True, + ) + except Exception as e: # pragma: no cover + # Delete Vault Password + if "vault.pass" in os.listdir(RUN_DIR): + os.remove("{}/vault.pass".format(RUN_DIR)) + + if os.path.exists("/vault.pass"): + os.remove("/vault.pass") + shutil.rmtree(RUN_DIR) + return f"Error: {e}", 200 + + def ansible_stream(): + try: + while thread.is_alive(): + try: + for event in runner.events: + yield ("
" + str(event["stdout"]) + "
").encode( + "utf-8" + ) + time.sleep(0.1) + except Exception as e: + yield f"Error: {e}".encode("utf-8") + finally: + if os.path.exists("/vault.pass"): + os.remove("/vault.pass") + + # Delete all files + shutil.rmtree(RUN_DIR) + + return ansible_stream(), {"Content-Type": "text/plain"} + @app.route("/mac///") @requires_auth_write diff --git a/backend/test/test_01_alertmanager.py b/backend/test/test_01_alertmanager.py index e3c15b01..cc76c39f 100644 --- a/backend/test/test_01_alertmanager.py +++ b/backend/test/test_01_alertmanager.py @@ -83,14 +83,17 @@ def test_send_alert(): - Will just print out URL and payload """ a = unwrap(serve.list_alerts)() + print(a[0]) assert a[1] == 200 b = json.loads(a[0]) - assert watcher.send_alert( + output = watcher.send_alert( "test-alert", "test-service", "test-host", ) + print(output.text) + assert output a = unwrap(serve.list_alerts)() assert a[1] == 200 diff --git a/backend/test/test_07_ansible.py b/backend/test/test_07_ansible.py index b1cab98f..1b72162b 100644 --- a/backend/test/test_07_ansible.py +++ b/backend/test/test_07_ansible.py @@ -213,33 +213,14 @@ def test_run_ansible(): shutil.copy(src, dest) # Check a clean run - x = run_ansible( + x, y = run_ansible( hosts="sampleclient", playbook="install", become_file="vault", vault_password="test", ) - - assert "First Output" in output + assert "
Second Output
" in output + assert "
Third Output
" in output + + # Ensure mocks were called + mock_run_ansible.assert_called_once() + mock_run_async.assert_called_once() + mock_rmtree.assert_called_once() + + +# Helper function to simulate events with an exception +def event_generator_with_exception(): + yield {"stdout": "Sample Output"} + raise Exception("Test Exception") + + +@patch("serve.ansible_runner.run_async") +@patch("serve.ansible_helper.run_ansible") +@patch("serve.os.path.exists") +@patch("serve.shutil.rmtree") +@patch("serve.os.remove") +def test_ansible_stream_error_handling( + mock_os_remove, mock_rmtree, mock_exists, mock_run_ansible, mock_run_async +): + # Mock the ansible_helper.run_ansible return value + mock_run_ansible.return_value = ("RUN_DIR_PATH", "playbook_name") + + # Mock the ansible_runner.run_async return values + mock_thread = MagicMock() + # Simulate the thread being alive a couple of times + mock_thread.is_alive.side_effect = [True, True, False] + mock_runner = MagicMock() + + # Simulate an exception occurring during event iteration + mock_runner.events = event_generator_with_exception() + + mock_run_async.return_value = (mock_thread, mock_runner) + + # Mock os.path.exists to return True, simulating the presence of a vault file + mock_exists.return_value = True + + # Sample input data + sample_data = json.dumps( + { + "hosts": "localhost", + "playbook": "sample_playbook", + "vault_password": "password", + "become_file": "become_file_path", + } + ) + + # Call the function + response, headers = unwrap(run_ansible)(inp_data=sample_data) + + # Consume the generator to test its output + output = list(response) + + # Check that the output contains the error message + assert b"Error: Test Exception" in output + + # Ensure mocks were called + mock_run_ansible.assert_called_once() + mock_run_async.assert_called_once() + mock_rmtree.assert_called_once() + mock_os_remove.assert_called_once() + + +@patch("serve.ansible_helper.run_ansible") +def test_run_ansible_missing_data(mock_run_ansible): + # Mock the ansible_helper.run_ansible to prevent actual calls + mock_run_ansible.return_value = ("RUN_DIR_PATH", "playbook_name") + + # Missing "vault_password" field + sample_data = json.dumps( + { + "hosts": "localhost", + "playbook": "sample_playbook", + "become_file": "become_file_path", + } + ) + + # Call the function + response, headers = unwrap(run_ansible)(inp_data=sample_data) + + # Check that the response indicates invalid data + assert response == "Invalid data" + assert headers == 482 + + +@patch("serve.ansible_runner.run_async") +@patch("serve.ansible_helper.run_ansible") +@patch("serve.os.path.exists") +@patch("serve.shutil.rmtree") +def test_run_ansible_no_ssh_key( + mock_rmtree, mock_exists, mock_run_ansible, mock_run_async +): + # Mock the ansible_helper.run_ansible return value + mock_run_ansible.return_value = ("RUN_DIR_PATH", "playbook_name") + + # Mock the ansible_runner.run_async return values + mock_thread = MagicMock() + mock_thread.is_alive.side_effect = [True, False] + mock_runner = MagicMock() + + # Simulate one event in runner.events + mock_runner.events = [{"stdout": "Sample Output"}] + + mock_run_async.return_value = (mock_thread, mock_runner) + + # Mock os.path.exists to return False + mock_exists.return_value = False + + # Sample input data without "ssh_key" + sample_data = json.dumps( + { + "hosts": "localhost", + "playbook": "sample_playbook", + "vault_password": "password", + "become_file": "become_file_path", + } + ) + + # Call the function + response, headers = unwrap(run_ansible)(inp_data=sample_data) + + # Check headers + assert headers["Content-Type"] == "text/plain" + + # Consume the generator to test its output + output = b"".join(list(response)).decode("utf-8") + + # Check that the output contains the expected event output + assert "
Sample Output
" in output + + # Ensure mocks were called + mock_run_ansible.assert_called_once() + mock_run_async.assert_called_once() + mock_rmtree.assert_called_once() diff --git a/backend/watcher.py b/backend/watcher.py index c2e8fda7..77e24c3c 100644 --- a/backend/watcher.py +++ b/backend/watcher.py @@ -28,7 +28,8 @@ def send_alert( :param serverity - defaults to error """ - url = "http://alertmanager:9093/api/v1/alerts" + url = "http://alertmanager:9093/api/v2/alerts" + headers = {"Content-Type": "application/json"} data = { "status": "firing", "labels": { @@ -37,13 +38,13 @@ def send_alert( "severity": severity, "instance": instance, }, - "annotations": { - "summary": summary, - }, + "annotations": {"summary": summary}, "generatorURL": url, } password = open("/alertmanager/pass").read() - retval = requests.post(url, data=json.dumps([data]), auth=("admin", password)) + retval = requests.post( + url, headers=headers, data=json.dumps([data]), auth=("admin", password) + ) return retval diff --git a/cron/Dockerfile b/cron/Dockerfile index 040fc9ca..1b4cacc2 100644 --- a/cron/Dockerfile +++ b/cron/Dockerfile @@ -22,7 +22,7 @@ RUN apt-get update && apt-get -y install python3 \ COPY backend/requirements.txt / -RUN pip3 install -r /requirements.txt +RUN pip3 install -r /requirements.txt --break-system-packages # Cron installation diff --git a/frontend/labyrinth/src/components/CreateEditHost.vue b/frontend/labyrinth/src/components/CreateEditHost.vue index f4b4ee79..d1d2f048 100644 --- a/frontend/labyrinth/src/components/CreateEditHost.vue +++ b/frontend/labyrinth/src/components/CreateEditHost.vue @@ -245,13 +245,11 @@