diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 38200ad7..ad27beba 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -1,5 +1,5 @@ --- -name: Playwright Tests +name: UI Testing on: deployment_status: push: @@ -13,10 +13,6 @@ jobs: runs-on: ubuntu-latest container: image: mcr.microsoft.com/playwright:v1.41.2-jammy - # TODO: Refactor action to use updated logic for Login.gov - # when a service account to run tests is available for use. - # Issue: CRASH-337 - if: true == false steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 diff --git a/playwright/axe-test.ts b/playwright/axe-test.ts new file mode 100644 index 00000000..5f97df04 --- /dev/null +++ b/playwright/axe-test.ts @@ -0,0 +1,27 @@ +import { test as base, defineConfig } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; +import { describe } from 'node:test'; +import { configure } from 'axe-core'; + +type AxeFixture = { + makeAxeBuilder: () => AxeBuilder; +}; + +// Extend base test by providing "makeAxeBuilder" +// +// This new "test" can be used in multiple test files, and each of them will get +// a consistently configured AxeBuilder instance. +export const test = base.extend({ + makeAxeBuilder: async ({ page }, use, testInfo) => { + const makeAxeBuilder = () => + new AxeBuilder({ page }).withTags([ + 'wcag2a', + 'wcag2aa', + 'wcag21a', + 'wcag21aa' + ]); + await use(makeAxeBuilder); + } +}); +export { expect } from '@playwright/test'; +export { Page } from '@playwright/test'; diff --git a/playwright/e2e/global-admin/home.spec.ts b/playwright/e2e/global-admin/home.spec.ts index 1e1795e2..1ab3a6ea 100644 --- a/playwright/e2e/global-admin/home.spec.ts +++ b/playwright/e2e/global-admin/home.spec.ts @@ -1,66 +1,32 @@ -import { test, expect, Page } from '@playwright/test'; -import exp from 'constants'; +import { describe } from 'node:test'; -test.describe.configure({ mode: 'serial' }); -let page: Page; +const { test, expect, Page } = require('../../axe-test'); -test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); - await page.goto('/'); -}); +test.describe.configure({ mode: 'parallel' }); +let page: InstanceType; -test.afterAll(async () => { - await page.close(); -}); -test('home', async () => { - // Expect home page to show Latest Vulnerabilities. - await expect( - page.getByRole('heading', { name: 'Latest Vulnerabilities' }) - ).toBeVisible(); - await expect( - page.getByRole('heading', { name: 'Open Vulnerabilities by Domain' }) - ).toBeVisible(); - await expect( - page.getByRole('heading', { name: 'Most Common Ports' }) - ).toBeVisible(); - await expect( - page.getByRole('heading', { name: 'Most Common Vulnerabilities' }) - ).toBeVisible(); - await expect( - page.getByRole('heading', { name: 'Severity Levels' }) - ).toBeVisible(); - await expect( - page.getByPlaceholder('Search a domain, vuln, port, service, IP') - ).toBeVisible(); - await expect(page.getByRole('link', { name: 'Inventory' })).toBeVisible(); - await page.screenshot({ path: 'test-results/img/global-admin/home.png' }); -}); +test.describe('home', () => { + test.beforeEach(async ({ browser }) => { + const context = await browser.newContext(); + page = await context.newPage(); + await page.goto('/'); + }); -test('Open Vulnerabilities by Domain', async () => { - await page.getByRole('button', { name: 'All' }).click(); - await page.screenshot({ - path: 'test-results/img/global-admin/open_vuln_all.png' + test.afterEach(async () => { + await page.close(); }); - if ( - (await page.getByRole('button', { name: 'Medium' }).isDisabled()) == false - ) { - await page.getByRole('button', { name: 'Medium' }).click(); - await page.screenshot({ - path: 'test-results/img/global-admin/open_vuln_medium.png' - }); - } - if ( - (await page.getByRole('button', { name: 'High' }).isDisabled()) == false - ) { - await page.getByRole('button', { name: 'High' }).click(); - await page.screenshot({ - path: 'test-results/img/global-admin/open_vuln_high.png' + + test('Test homepage accessibility', async ({ + page, + makeAxeBuilder + }, testInfo) => { + const accessibilityScanResults = await makeAxeBuilder().analyze(); + + await testInfo.attach('accessibility-scan-results', { + body: JSON.stringify(accessibilityScanResults, null, 2), + contentType: 'application/json' }); - } - if ((await page.getByLabel('Go to next page').isDisabled()) == false) { - await page.getByLabel('Go to next page').click(); - } - if ((await page.getByLabel('Go to previous page').isDisabled()) == false) { - await page.getByLabel('Go to previous page').click(); - } + + expect(accessibilityScanResults.violations).toHaveLength(0); + }); }); diff --git a/playwright/e2e/global-admin/inventory.spec.ts b/playwright/e2e/global-admin/inventory.spec.ts index 559b7679..d253f637 100644 --- a/playwright/e2e/global-admin/inventory.spec.ts +++ b/playwright/e2e/global-admin/inventory.spec.ts @@ -1,72 +1,84 @@ -import { test, expect, chromium, Page } from '@playwright/test'; +import { describe } from 'node:test'; -test.describe.configure({ mode: 'serial' }); -let page: Page; +const { test, expect, Page } = require('../../axe-test'); -test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); - await page.goto('/'); -}); +test.describe.configure({ mode: 'parallel' }); +let page: InstanceType; -test.afterAll(async () => { - await page.close(); -}); -test('Inventory', async () => { - await page.getByRole('link', { name: 'Inventory' }).click(); - await expect(page).toHaveURL('/inventory'); - await page.getByRole('button', { name: 'IP(s)' }).click(); - await page.getByRole('button', { name: 'Severity' }).click(); - await page.getByLabel('Sort by:').first().click(); - await page.getByRole('option', { name: 'Domain Name' }).click(); - await page.getByLabel('Sort by:').first().click(); - await page.getByRole('option', { name: 'IP' }).click(); - await page.getByLabel('Sort by:').first().click(); - await page.getByRole('option', { name: 'Last Seen' }).click(); - await page.getByLabel('Sort by:').first().click(); - await page.getByRole('option', { name: 'First Seen' }).click(); - await page.screenshot({ - path: 'test-results/img/global-admin/inventory.png' +test.describe('Inventory', () => { + test.beforeEach(async ({ browser }) => { + const context = await browser.newContext(); + page = await context.newPage(); + await page.goto('/'); }); -}); -test('Domains', async () => { - await page.goto('/inventory'); - await page.getByRole('link', { name: 'All Domains' }).click(); - await expect(page).toHaveURL('/inventory/domains'); - if ((await page.getByLabel('Go to next page').isDisabled()) == false) { - await page.getByLabel('Go to next page').click(); - } - if ((await page.getByLabel('Go to previous page').isDisabled()) == false) { - await page.getByLabel('Go to previous page').click(); - } - await page.screenshot({ path: 'test-results/img/global-admin/domains.png' }); -}); + test.afterEach(async () => { + await page.close(); + }); + test('Test inventory accessibility', async ({ makeAxeBuilder }, testInfo) => { + await page.getByRole('link', { name: 'Inventory' }).click(); + await expect(page).toHaveURL('/inventory'); -test('Domain details', async () => { - await page.goto('/inventory/domains'); - await page.getByRole('row').nth(2).getByRole('link').click(); - await expect(page).toHaveURL(new RegExp('/inventory/domain/')); - await expect(page.getByText('IP:')).toBeVisible(); - await expect(page.getByText('First Seen:')).toBeVisible(); - await expect(page.getByText('Last Seen:')).toBeVisible(); - await expect(page.getByText('Organization:')).toBeVisible(); - await page.screenshot({ - path: 'test-results/img/global-admin/domain_details.png' + const accessibilityScanResults = await makeAxeBuilder().analyze(); + + await testInfo.attach('accessibility-scan-results', { + body: JSON.stringify(accessibilityScanResults, null, 2), + contentType: 'application/json' + }); + + expect(accessibilityScanResults.violations).toHaveLength(0); }); -}); -test('Domains filter', async () => { - await page.goto('/inventory/domains'); - await page.locator('#organizationName').click(); - await page.locator('#organizationName').fill('Homeland'); - await page.locator('#organizationName').press('Enter'); - let rowCount = await page.getByRole('row').count(); - for (let it = 2; it < rowCount; it++) { - await expect( - page.getByRole('row').nth(it).getByRole('cell').nth(1) - ).toContainText('Homeland'); - } - await page.screenshot({ - path: 'test-results/img/global-admin/domain_filter.png' + test('Test domain accessibility', async ({ makeAxeBuilder }, testInfo) => { + await page.goto('/inventory'); + await page.getByRole('link', { name: 'All Domains' }).click(); + await expect(page).toHaveURL('/inventory/domains'); + + const accessibilityScanResults = await makeAxeBuilder().analyze(); + + await testInfo.attach('accessibility-scan-results', { + body: JSON.stringify(accessibilityScanResults, null, 2), + contentType: 'application/json' + }); + + expect(accessibilityScanResults.violations).toHaveLength(0); + }); + + test('Test domain details accessibility', async ({ + makeAxeBuilder + }, testInfo) => { + await page.goto('/inventory/domains'); + await page + .getByRole('row') + .nth(1) + .getByRole('cell') + .nth(8) + .getByRole('button') + .click(); + await expect(page).toHaveURL(new RegExp('/inventory/domain/')); + + const accessibilityScanResults = await makeAxeBuilder().analyze(); + + await testInfo.attach('accessibility-scan-results', { + body: JSON.stringify(accessibilityScanResults, null, 2), + contentType: 'application/json' + }); + + expect(accessibilityScanResults.violations).toHaveLength(0); + }); + + test('Test domain table filter', async () => { + await page.goto('/inventory/domains'); + await page.getByLabel('Show filters').click(); + await page.getByPlaceholder('Filter value').click(); + await page.getByPlaceholder('Filter value').fill('Homeland'); + await page.getByPlaceholder('Filter value').press('Enter'); + + let rowCount = await page.getByRole('row').count(); + for (let it = 2; it < rowCount; it++) { + await expect( + page.getByRole('row').nth(it).getByRole('cell').nth(0) + ).toContainText('Homeland'); + } }); }); diff --git a/playwright/e2e/global-admin/vulnerabilities.spec.ts b/playwright/e2e/global-admin/vulnerabilities.spec.ts index ca92064f..7256f3f8 100644 --- a/playwright/e2e/global-admin/vulnerabilities.spec.ts +++ b/playwright/e2e/global-admin/vulnerabilities.spec.ts @@ -1,59 +1,68 @@ -import { test, expect, chromium, Page } from '@playwright/test'; +const { test, expect, Page } = require('../../axe-test'); -test.describe.configure({ mode: 'serial' }); -let page: Page; +test.describe.configure({ mode: 'parallel' }); +let page: InstanceType; -test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); - await page.goto('/'); -}); +test.describe('Vulnerabilities', () => { + test.beforeEach(async ({ browser }) => { + const context = await browser.newContext(); + page = await context.newPage(); + await page.goto('/'); + }); -test.afterAll(async () => { - await page.close(); -}); + test.afterEach(async () => { + await page.close(); + }); + + test('Test vulnerabilities accessibility', async ({ + makeAxeBuilder + }, testInfo) => { + await page.getByRole('link', { name: 'Inventory' }).click(); + await page.getByRole('link', { name: 'All Vulnerabilities' }).click(); + await expect(page).toHaveURL('/inventory/vulnerabilities'); -test('Vulnerabilities', async () => { - await page.getByRole('link', { name: 'Inventory' }).click(); - await page.getByRole('link', { name: 'All Vulnerabilities' }).click(); - await expect(page).toHaveURL('/inventory/vulnerabilities'); - if ((await page.getByLabel('Go to next page').isDisabled()) == false) { - await page.getByLabel('Go to next page').click(); - } - if ((await page.getByLabel('Go to previous page').isDisabled()) == false) { - await page.getByLabel('Go to previous page').click(); - } - await page.screenshot({ - path: 'test-results/img/global-admin/vulnerabilities.png' + const accessibilityScanResults = await makeAxeBuilder().analyze(); + + await testInfo.attach('accessibility-scan-results', { + body: JSON.stringify(accessibilityScanResults, null, 2), + contentType: 'application/json' + }); + + expect(accessibilityScanResults.violations).toHaveLength(0); }); -}); -test('Vulnerability details NIST', async () => { - await page.goto('/inventory/vulnerabilities'); - const newTabPromise = page.waitForEvent('popup'); - await page.getByRole('row').nth(2).getByRole('link').nth(0).click(); - const newTab = await newTabPromise; - await newTab.waitForLoadState(); - await expect(newTab).toHaveURL( - new RegExp('^https://nvd\\.nist\\.gov/vuln/detail/') - ); -}); + test('Test vulnerability details NIST link', async () => { + await page.goto('/inventory/vulnerabilities'); + const newTabPromise = page.waitForEvent('popup'); -test('Domain details link', async () => { - await page.goto('/inventory/vulnerabilities'); - await page.getByRole('row').nth(2).getByRole('link').nth(1).click(); - await expect(page).toHaveURL(new RegExp('/inventory/domain/')); -}); + await page.getByRole('row').nth(1).getByRole('cell').nth(0).click(); + const newTab = await newTabPromise; + await newTab.waitForLoadState(); + await expect(newTab).toHaveURL( + new RegExp('^https://nvd\\.nist\\.gov/vuln/detail/') + ); + }); + + test('Test domain details link', async () => { + await page.goto('/inventory/vulnerabilities'); + await page.getByRole('row').nth(1).getByRole('cell').nth(3).click(); + await expect(page).toHaveURL(new RegExp('/inventory/domain/')); + }); -test('Vulnerability details', async () => { - await page.goto('/inventory/vulnerabilities'); - await page.getByRole('row').nth(2).getByRole('link').nth(2).click(); - await expect(page).toHaveURL(new RegExp('/inventory/vulnerability/')); - await expect(page.getByRole('heading', { name: 'Overview' })).toBeVisible(); - await expect( - page.getByRole('heading', { name: 'Installed (Known) Products' }) - ).toBeVisible(); - await expect(page.getByRole('heading', { name: 'Provenance' })).toBeVisible(); - await expect( - page.getByRole('heading', { name: 'Vulnerability Detection History' }) - ).toBeVisible(); + test('Test vulnerability details accessibility', async ({ + makeAxeBuilder + }, testInfo) => { + await page.goto('/inventory/vulnerabilities'); + await page.getByRole('row').nth(1).getByRole('cell').nth(7).click(); + await expect(page).toHaveURL(new RegExp('/inventory/vulnerability/')); + + const accessibilityScanResults = await makeAxeBuilder().analyze(); + + await testInfo.attach('accessibility-scan-results', { + body: JSON.stringify(accessibilityScanResults, null, 2), + contentType: 'application/json' + }); + + expect(accessibilityScanResults.violations).toHaveLength(0); + }); }); diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index 410d250a..6d045fd4 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -22,19 +22,22 @@ async function globalSetup(config: FullConfig) { //Log in with credentials. await page.goto(String(process.env.PW_XFD_URL)); + await page.getByTestId('button').click(); await page - .getByPlaceholder('Enter your email address') + .getByLabel('Username (Email)') .fill(String(process.env.PW_XFD_USERNAME)); + await page.getByRole('button', { name: 'Next' }).click(); await page - .getByPlaceholder('Enter your password') + .getByLabel('Email address') + .fill(String(process.env.PW_XFD_USERNAME)); + await page + .getByLabel('Password', { exact: true }) .fill(String(process.env.PW_XFD_PASSWORD)); await page.getByRole('button', { name: 'Sign in' }).click(); - await page - .getByPlaceholder('Enter code from your authenticator app') - .fill(totp.generate()); - await page.getByRole('button', { name: 'Confirm' }).click(); + await page.getByLabel('One-time code').fill(totp.generate()); + await page.getByRole('button', { name: 'Submit' }).click(); //Wait for storageState to write to json file for other tests to use. - await page.waitForTimeout(1000); + await page.waitForTimeout(7000); await page.context().storageState({ path: authFile }); await page.close(); } diff --git a/playwright/package-lock.json b/playwright/package-lock.json index 39b36fb4..96455de7 100644 --- a/playwright/package-lock.json +++ b/playwright/package-lock.json @@ -4,6 +4,8 @@ "packages": { "": { "dependencies": { + "@axe-core/playwright": "^4.9.1", + "axe-core": "^4.9.1", "dotenv": "^16.4.5", "otpauth": "^9.2.2" }, @@ -15,6 +17,17 @@ "name": "xfd_playwright", "version": "1.0.0" }, + "node_modules/@axe-core/playwright": { + "dependencies": { + "axe-core": "~4.9.1" + }, + "integrity": "sha512-8m4WZbZq7/aq7ZY5IG8GqV+ZdvtGn/iJdom+wBg+iv/3BAOBIfNQtIu697a41438DzEEyptXWmC3Xl5Kx/o9/g==", + "peerDependencies": { + "playwright-core": ">= 1.0.0" + }, + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.9.1.tgz", + "version": "4.9.1" + }, "node_modules/@playwright/test": { "bin": { "playwright": "cli.js" @@ -39,6 +52,14 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.20.tgz", "version": "20.11.20" }, + "node_modules/axe-core": { + "engines": { + "node": ">=4" + }, + "integrity": "sha512-QbUdXJVTpvUTHU7871ppZkdOLBeGUKBQWHkHrvN2V9IQWGMt61zf3B45BtzjxEJzYuj0JBjBZP/hmYS/R9pmAw==", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.9.1.tgz", + "version": "4.9.1" + }, "node_modules/dotenv": { "engines": { "node": ">=12" @@ -105,7 +126,6 @@ "bin": { "playwright-core": "cli.js" }, - "dev": true, "engines": { "node": ">=16" }, diff --git a/playwright/package.json b/playwright/package.json index 38e48e97..20ef5dc0 100644 --- a/playwright/package.json +++ b/playwright/package.json @@ -1,6 +1,8 @@ { "author": "", "dependencies": { + "@axe-core/playwright": "^4.9.1", + "axe-core": "^4.9.1", "dotenv": "^16.4.5", "otpauth": "^9.2.2" }, diff --git a/playwright/playwright.config.ts b/playwright/playwright.config.ts index 2781f82f..e2e7c41d 100644 --- a/playwright/playwright.config.ts +++ b/playwright/playwright.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ /* Retry on CI only */ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, + workers: process.env.CI ? 2 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [ ['list', { printSteps: true }],