From 34381ecf3b3196196da8fd5a3a54cad2a1137f4b Mon Sep 17 00:00:00 2001 From: Steve Astels Date: Tue, 3 Oct 2023 12:24:58 -0400 Subject: [PATCH] Soak test everything (#1961) * 2 soak tests now * set host to base url * tweak readme --- scripts/soak_test/README.md | 33 +++++++--- scripts/soak_test/locust.conf | 2 +- scripts/soak_test/soak_test_all_servers.py | 60 +++++++++++++++++++ .../{soak-test.py => soak_test_send_email.py} | 3 + scripts/soak_test/soak_utils.py | 6 ++ 5 files changed, 96 insertions(+), 8 deletions(-) create mode 100644 scripts/soak_test/soak_test_all_servers.py rename scripts/soak_test/{soak-test.py => soak_test_send_email.py} (91%) create mode 100644 scripts/soak_test/soak_utils.py diff --git a/scripts/soak_test/README.md b/scripts/soak_test/README.md index 5d24c31baf..a9e2ecb50e 100644 --- a/scripts/soak_test/README.md +++ b/scripts/soak_test/README.md @@ -4,13 +4,17 @@ The goal of this code is to do a soak test of api while we make significant application or infrastructure changes. +There are two soak tests here: +- `soak_test_send_email.py` will POST an email to api every second. +- `soak_test_all_servers.py` will do a GET to all our servers (admin, api, dd-api, api-k8s, documentation), on average hitting each server once a second + ## How to configure Run the setup.sh to install the python pre-requisites or run in the repo devcontainer. -Default configuration is in the `locust.conf` file. +Default configuration is in the `locust.conf` file. Note that the `host` is the base address of the system you are testing, for example `https://staging.notification.cdssandbox.xyz` **not** `https://api.staging.notification.cdssandbox.xyz`. The "api" prefix will be added in the code. -The python file `soak_test.py` requires environment variables `API_KEY` and `EMAIL_TEMPLATE_ID`. The template should have no variables. +The python file `soak_test_send_email.py` requires environment variables `API_KEY` and `EMAIL_TEMPLATE_ID`. The template should have no variables. ``` API_KEY=gcntfy-notAKey-f6c7cc49-b5b7-4e67-a8ff-24f34be34523-f6c7cc49-b5b7-4e67-a8ff-24f34be34523 @@ -22,7 +26,7 @@ __See Last Pass note "Soak Test Staging API Key and Template" in Shared-New-Noti Note that the default configuration in `locust.conf` is to send one email per second. -You should supply a `--ref` option that will set the notification's `client_reference`. This is useful in testing that all POSTs were processed successfully. +You can supply a `--ref` option to `soak_test_send_email.py` that will set the notification's `client_reference`. This is useful in testing that all POSTs were processed successfully. ## How to run @@ -30,25 +34,39 @@ There are two ways to run Locust, with the UI or headless. ### With the UI -Locally, simply run: +Locally you can run the email soak test with: ```shell -locust -f ./soak-test.py --ref=soak-2023-05-30-A +locust -f ./soak_test_send_email.py --ref=soak-2023-05-30-A ``` Follow the localhost address that the console will display to get to the UI. It will ask you how many total users and spawned users you want configured. Once setup, you can manually start the tests via the UI and follow the summary data and charts visually. +The server soak test can be run with + +```shell +locust -f ./soak_test_all_servers.py +``` + ### Headless, via the command line You can pass the necessary parameters to the command line to run in the headless mode. For example: ```shell -locust -f ./soak-test.py --headless --ref=soak-2023-05-30-A +locust -f ./soak_test_send_email.py --headless --ref=soak-2023-05-30-A ``` The defaults in `locust.conf` may be overridden by command line options -To check whether all the POSTs from locust made it into the database, run the "Soak test" query on blazer. The query is already in staging, or you can run: +The server soak test can be run with + +```shell +locust -f ./soak_test_all_servers.py --headless +``` + +## Checking if all emails were sent + +To check whether all the POSTs from `soak_test_send_email.py` made it into the database, run the "Soak test" query on blazer. The query is already in staging, or you can run: ```sql WITH @@ -73,3 +91,4 @@ stats as ( ) select * from stats ``` + diff --git a/scripts/soak_test/locust.conf b/scripts/soak_test/locust.conf index d88ebcd001..44c0e7a6ad 100644 --- a/scripts/soak_test/locust.conf +++ b/scripts/soak_test/locust.conf @@ -1,3 +1,3 @@ users=1 stop-timeout=10 -host=https://api.staging.notification.cdssandbox.xyz +host=https://staging.notification.cdssandbox.xyz diff --git a/scripts/soak_test/soak_test_all_servers.py b/scripts/soak_test/soak_test_all_servers.py new file mode 100644 index 0000000000..089ef07b77 --- /dev/null +++ b/scripts/soak_test/soak_test_all_servers.py @@ -0,0 +1,60 @@ +from dotenv import load_dotenv +from locust import HttpUser, TaskSet, constant_pacing, task +from locust.clients import HttpSession +from soak_utils import url_with_prefix + +load_dotenv() + + +class MultipleHostsUser(HttpUser): + abstract = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.admin_client = HttpSession( + base_url=self.host, request_event=self.client.request_event, user=self + ) + + self.api_client = HttpSession( + base_url=url_with_prefix(self.host, "api"), request_event=self.client.request_event, user=self + ) + + self.api_k8s_client = HttpSession( + base_url=url_with_prefix(self.host, "api-k8s"), request_event=self.client.request_event, user=self + ) + + self.dd_api_client = HttpSession( + base_url=url_with_prefix(self.host, "api.document"), request_event=self.client.request_event, user=self + ) + + self.documentation_client = HttpSession( + base_url=url_with_prefix(self.host, "documentation"), request_event=self.client.request_event, user=self + ) + + +class UserTasks(TaskSet): + @task + def test_admin(self): + self.user.admin_client.get("/_status?simple=true", name=f"{self.user.admin_client.base_url}/_status?simple=true") + + @task + def test_api(self): + self.user.api_client.get("/_status?status=true", name=f"{self.user.api_client.base_url}/_status?simple=true") + + @task + def test_api_k8s(self): + self.user.api_k8s_client.get("/_status?status=true", name=f"{self.user.api_k8s_client.base_url}/_status?simple=true") + + @task + def test_dd_api(self): + self.user.dd_api_client.get("/_status?simple=true", name=f"{self.user.dd_api_client.base_url}/_status?simple=true") + + @task + def test_documentation(self): + self.user.documentation_client.get("/", name=f"{self.user.documentation_client.base_url}/") + + +class WebsiteUser(MultipleHostsUser): + wait_time = constant_pacing(0.2) # 5 GETs a second, so each server every second on average + tasks = [UserTasks] diff --git a/scripts/soak_test/soak-test.py b/scripts/soak_test/soak_test_send_email.py similarity index 91% rename from scripts/soak_test/soak-test.py rename to scripts/soak_test/soak_test_send_email.py index d761d08f85..f9cefd88d1 100644 --- a/scripts/soak_test/soak-test.py +++ b/scripts/soak_test/soak_test_send_email.py @@ -2,6 +2,7 @@ from dotenv import load_dotenv from locust import HttpUser, constant_pacing, events, task +from soak_utils import url_with_prefix load_dotenv() @@ -15,6 +16,8 @@ class NotifyApiUser(HttpUser): wait_time = constant_pacing(1) # each user makes one post per second def __init__(self, *args, **kwargs): + self.host = url_with_prefix(self.host, "api") + super(NotifyApiUser, self).__init__(*args, **kwargs) self.headers = {"Authorization": f"apikey-v1 {os.getenv('API_KEY')}"} self.email_address = "success@simulator.amazonses.com" diff --git a/scripts/soak_test/soak_utils.py b/scripts/soak_test/soak_utils.py new file mode 100644 index 0000000000..3a62cf115d --- /dev/null +++ b/scripts/soak_test/soak_utils.py @@ -0,0 +1,6 @@ +from urllib.parse import urlparse + + +def url_with_prefix(url: str, prefix: str) -> str: + parsed_url = urlparse(url) + return parsed_url._replace(netloc=f"{prefix}.{parsed_url.netloc}").geturl()