chore: add e2e playwright tests for grafana (#844)

## Description

Adds e2e testing structure and specific e2e test for Grafana using
playwright. This test:
- Validates existence and successful connection to datasources (Loki and
Prometheus)
- Validates two custom dashboards exist and dropdowns populate for ns
selection (resources and loki quicksearch)
- Validates SSO login success

## Related Issue

Fixes #764

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [x] Other (security config, docs update, etc)

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [x] [Contributor
Guide](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)
followed

.codespellrc

@@ -1,5 +1,5 @@
 # Lint Codespell configurations
 [codespell]
 skip = .codespellrc,.git,node_modules,build,dist,*.zst,CHANGELOG.md
-ignore-words-list = NotIn,AKS,LICENS
-enable-colors =
+ignore-words-list = NotIn,AKS,LICENS,aks
+enable-colors =

.eslintrc.json

@@ -3,21 +3,38 @@
     "browser": false,
     "es2021": true
   },
-  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
   "parser": "@typescript-eslint/parser",
   "parserOptions": {
-    "project": ["./tsconfig.json"],
+    "project": [
+      "./tsconfig.json"
+    ],
     "ecmaVersion": 2022
   },
-  "plugins": ["@typescript-eslint"],
-  "ignorePatterns": ["node_modules", "dist", "jest.*.js"],
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "ignorePatterns": [
+    "node_modules",
+    "dist",
+    "jest.*.js",
+    "e2e/"
+  ],
   "root": true,
   "rules": {
-    "@typescript-eslint/no-floating-promises": ["error"]
+    "@typescript-eslint/no-floating-promises": [
+      "error"
+    ]
   },
   "overrides": [
     {
-      "files": [ "src/pepr/operator/crd/generated/**/*.ts", "src/pepr/operator/crd/generated/*.ts" ],
+      "files": [
+        "src/pepr/operator/crd/generated/**/*.ts",
+        "src/pepr/operator/crd/generated/*.ts"
+      ],
       "rules": {
         "@typescript-eslint/no-explicit-any": "off"
       }

.github/actions/save-logs/action.yaml

@@ -37,6 +37,12 @@ runs:
         sudo chown $USER /tmp/uds-*.log || echo ""
       shell: bash
 
+    - name: Move Playwright Artifacts
+      run: |
+        sudo mkdir -p /tmp/playwright
+        sudo mv e2e/playwright/.playwright/* /tmp/playwright || true
+      shell: bash
+
     - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       with:
         name: debug-log${{ inputs.suffix }}
@@ -47,3 +53,5 @@ runs:
           /tmp/debug-*.log
           /tmp/uds-containerd-logs
           /tmp/k3d-uds-*.log
+          /tmp/playwright/output
+          /tmp/playwright/reports

.github/test-infra/azure/aks/variables.tf

@@ -191,4 +191,4 @@ variable "db_name" {
   description = "The name to give the database"
   type        = string
   default     = "grafana"
-}
+}

.github/workflows/test.yaml

@@ -67,7 +67,7 @@ jobs:
 
       - name: Test UDS Core Install
         if: ${{ inputs.package == 'all' && inputs.test_type == 'install' }}
-        run: uds run test-uds-core --set FLAVOR=${{ inputs.flavor }} --no-progress
+        run: uds run test:uds-core-e2e --set FLAVOR=${{ inputs.flavor }} --no-progress
 
       - name: Compose UDS Core Component Definitions
         if: ${{ inputs.package == 'all' && inputs.test_type == 'install' }}

.gitignore

@@ -4,9 +4,12 @@ build/
 .DS_Store
 *.tar.zst
 zarf-sbom
+zarf-sbom/**
 tmp/
 env.ts
-node_modules/**
+**/node_modules
+**/node_modules/**
+dist
 dist/**
 insecure*
 .env
@@ -19,3 +22,5 @@ extract-terraform.sh
 cluster-config.yaml
 **.tfstate
 **.backup
+**/.playwright/**
+**/.playwright

CONTRIBUTING.md

@@ -1,8 +1,6 @@
 # Welcome to UDS Core
 
-Thank you for your interest in Defense Unicorns UDS Core!
-
-This document describes the process and requirements for contributing to this UDS Core repo.
+Thank you for your interest in contributing to Defense Unicorns UDS Core! This document will guide you through the contribution process.
 
 ## Table of Contents
 
@@ -20,35 +18,97 @@ Continuous Delivery is core to our development philosophy. Check out [https://mi
 
 Specifically:
 
-* We do trunk-based development (main) with short-lived feature branches that originate from the trunk, get merged into the trunk, and are deleted after the merge
-* We don't merge code into main that isn't releasable
-* We perform automated testing on all changes before they get merged to main
-* Continuous integration (CI) pipeline tests are definitive
-* We create immutable release artifacts
+- We practice trunk-based development (main) with short-lived feature branches that are merged and deleted after the merge.
+- We don't merge code into main that isn't releasable.
+- All changes are tested automatically before being merged into main.
+- Continuous integration (CI) pipeline tests are the source of truth.
+- We produce immutable release artifacts.
+
+### Pre-Commit Checks
 
-Commits:
+We use [codespell](https://github.com/codespell-project/codespell) and [yamllint](https://yamllint.readthedocs.io/en/stable/) for pre-commit checks. Please install these before committing, or your commit may fail.
 
-We use [codespell](https://github.com/codespell-project/codespell) and [yamllint](https://yamllint.readthedocs.io/en/stable/) for our precommit checking. It is recommended to have these installed before attempting to commit to the a branch otherwise your commit will not finalize and you will be shown an error.
+To install these tools, run:
+
+```console
+uds run lint-check
+```
 
-To install both of these tools you can run `uds run lint-check` to install them or utilize `pip` to install them manually.
+Alternatively, you can install them with `pip`:
 
-```bash
+```console
 pip install yamllint codespell
 ```
 
 ## Definition of Done
 
-We apply these general principles to all User Stories and activities contributing to the UDS.
+We apply these principles to all User Stories and contributions:
+
+- Automated continuous integration (CI) pipeline tests pass
+- CI tests are updated to cover new system changes
+- Changes are peer-reviewed
+- Acceptance criteria is met
+- Documentation is updated to reflect changes
+
+### Testing
+
+Each individual component of UDS Core contains lightweight validations in its own `src/<component>/tasks.yaml` file. These validations focus on the bare minimum functionality, typically covering pod health and endpoint accessibility.
+
+We also place end-to-end tests under the `e2e` folder. In particular we use [Playwright](https://playwright.dev/) for browser based testing and have authentication setup to login to applications with a shared SSO session. Playwright provides a [test recorder](https://playwright.dev/docs/codegen#generate-tests-with-the-playwright-inspector) which can be beneficial to get a quickstart on new tests.
 
-* Automated continuous integration (CI) pipeline tests pass
-* CI pipeline tests have been updated to meet system changes
-* Changes are peer reviewed
-* Acceptance criteria is met
-* Documentation is updated to reflect what changed
+In general our testing focuses on the unique configuration and setup provided by UDS Core, rather than exhaustive functionality testing. We take this approach since each of the opensource applications we package and configure also have extensive end-to-end testing in their upstream repositories.
 
 ## Getting Started
 
-TBD
+This section will help you get set up and ready to contribute to UDS Core.
+
+### 1. Prerequisites
+
+Before starting, ensure that you have the following installed:
+
+- **Git**: [Install Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+- **K3d**: [Install K3d](https://k3d.io/#installation)
+- **Node.js** (for building and running Pepr): [Install Node.js](https://nodejs.org/en/download/)
+- **UDS CLI** (for running tasks and deploying): [Install UDS](https://uds.defenseunicorns.com/cli/quickstart-and-usage/)
+
+### 2. Clone the Repository and Make a Branch
+
+Clone the UDS Core repository to your local machine using Git (note that you may want to [fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo) this repository):
+
+```console
+git clone https://github.com/DefenseUnicorns/uds-core.git
+cd uds-core
+```
+
+Then make a branch for your changes:
+
+```console
+git checkout -b my-feature-branch
+```
+
+### 3. Make Changes and Test Locally
+
+Make the changes to add the new feature, bug fix, or other change necessary. Keep in mind any documentation or testing changes that are relevant while making code changes.
+
+When you are ready to test locally you can run the same tests as CI using the below UDS commands:
+
+```console
+# Lightweight validations
+uds run test-uds-core
+
+# Full e2e tests (run in CI)
+uds run test:uds-core-e2e
+```
+
+Each of these tasks will create a local k3d cluster, install UDS Core, and run a series of tests against it, the same tests that are run in CI.
+
+If you want to run a subset of core for faster iterations against a specific package, you can use the `LAYER` variable with the below task (example for metrics-server):
+
+```console
+uds run test-single-layer --set LAYER=metrics-server
+```
+
+Note you can also specify the `--set FLAVOR=<registry1/unicorn>` flag to test using with either the Iron Bank or Unicorn sourced images instead of the upstream ones.
 
 ## Submitting a Pull Request
 

e2e/playwright/auth.setup.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2024 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { expect, test as setup } from "@playwright/test";
+import { authFile } from "./playwright.config";
+import { domain } from "./uds.config";
+
+const baseURL = `https://sso.${domain}`;
+
+setup("authenticate", async ({ page, context }) => {
+  await page.goto(baseURL);
+
+  await page.getByLabel("Username or email").fill("doug");
+  await page.getByLabel("Password").fill("unicorn123!@#UN");
+  await page.getByRole("button", { name: "Log In" }).click();
+
+  await page.waitForURL(`${baseURL}/realms/uds/account`); // successful redirect
+
+  // ensure auth cookies were set
+  const cookies = await context.cookies();
+  const keycloakCookie = cookies.find(
+    (cookie) => cookie.name === "KEYCLOAK_SESSION",
+  );
+
+  expect(keycloakCookie).toBeDefined();
+  expect(keycloakCookie?.value).not.toBe("");
+  expect(keycloakCookie?.domain).toContain("sso.");
+
+  await page.context().storageState({ path: authFile });
+});

e2e/playwright/grafana.test.ts

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2024 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { expect, test } from "@playwright/test";
+import { domain, fullCore } from "./uds.config";
+
+test.use({ baseURL: `https://grafana.admin.${domain}` });
+test.describe.configure({ mode: 'serial' });
+
+test("validate loki datasource", async ({ page }) => {
+  test.skip(!fullCore, "Loki is only present on full core deploys");
+  await test.step("check loki", async () => {
+    await page.goto(`/connections/datasources`);
+    await page.getByRole('link', { name: 'Loki' }).click();
+    await page.click('text=Save & test');
+    // Allow 40 second timeout for datasource validation
+    await expect(page.locator('[data-testid="data-testid Alert success"]')).toBeVisible({ timeout: 40000 });
+  });
+});
+
+test("validate prometheus datasource", async ({ page }) => {
+  await test.step("check prometheus", async () => {
+    await page.goto(`/connections/datasources`);
+    await page.getByRole('link', { name: 'Prometheus' }).click();
+    await page.click('text=Save & test');
+    // Allow 20 second timeout for datasource validation
+    await expect(page.locator('[data-testid="data-testid Alert success"]')).toBeVisible({ timeout: 20000 });
+  });
+});
+
+// This dashboard is added by the upstream kube-prometheus-stack
+test("validate namespace dashboard", async ({ page }) => {
+  await test.step("check dashboard", async () => {
+    await page.goto(`/dashboards`);
+    await page.click('text="Kubernetes / Compute Resources / Namespace (Pods)"');
+    await page.getByTestId('data-testid Dashboard template variables Variable Value DropDown value link text authservice').click();
+    await page.getByRole('checkbox', { name: 'grafana' }).click();
+  });
+});
+
+// This dashboard is deployed "custom" by our uds config chart
+test("validate loki dashboard", async ({ page }) => {
+  test.skip(!fullCore, "Loki is only present on full core deploys");
+  await test.step("check dashboard", async () => {
+    await page.goto(`/dashboards`);
+    await page.getByPlaceholder('Search for dashboards and folders').fill('Loki');
+    await page.click('text="Loki Dashboard quick search"');
+    await page.getByTestId('data-testid Dashboard template variables Variable Value DropDown value link text authservice').click();
+    await page.getByRole('checkbox', { name: 'grafana' }).click();
+    await expect(page.getByTestId('data-testid Panel header Logs Panel').getByTestId('data-testid panel content')).toBeVisible();
+  });
+});