From 18beef555ddafc31cdc4189cb82fe3a3d3e3258d Mon Sep 17 00:00:00 2001 From: Murad Talibov Date: Mon, 18 Nov 2024 16:11:33 +0100 Subject: [PATCH] Run slow parallel tests as a separate project to assign higher timeout for all. Rename @slow tests to @sequential to make it clearer. --- .../e2e/exam/ExamAssessment.spec.ts | 12 +++++------ .../e2e/exam/ExamParticipation.spec.ts | 9 ++++----- .../playwright/e2e/exam/ExamResults.spec.ts | 3 +-- .../test-exam/TestExamParticipation.spec.ts | 3 +-- .../test-exam/TestExamStudentExams.spec.ts | 3 +-- .../exam/test-exam/TestExamTestRun.spec.ts | 3 +-- .../e2e/exercise/ExerciseImport.spec.ts | 2 +- .../ProgrammingExerciseAssessment.spec.ts | 2 +- .../ProgrammingExerciseParticipation.spec.ts | 2 +- ...grammingExerciseStaticCodeAnalysis.spec.ts | 2 +- src/test/playwright/package.json | 8 ++++---- src/test/playwright/playwright.config.ts | 20 ++++++++++++++----- 12 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/test/playwright/e2e/exam/ExamAssessment.spec.ts b/src/test/playwright/e2e/exam/ExamAssessment.spec.ts index 35754d3f897b..ea6e00a07aea 100644 --- a/src/test/playwright/e2e/exam/ExamAssessment.spec.ts +++ b/src/test/playwright/e2e/exam/ExamAssessment.spec.ts @@ -54,12 +54,12 @@ test.beforeAll('Create course', async ({ browser }) => { }); test.describe('Exam assessment', () => { - test.setTimeout(150000); + test.describe.configure({ mode: 'serial' }); let programmingAssessmentSuccessful = false; let modelingAssessmentSuccessful = false; let textAssessmentSuccessful = false; - test.describe.serial('Programming exercise assessment', { tag: '@slow' }, () => { + test.describe.serial('Programming exercise assessment', { tag: '@sequential' }, () => { test.beforeAll('Prepare exam', async ({ browser }) => { examEnd = dayjs().add(2, 'minutes'); const page = await newBrowserPage(browser); @@ -86,7 +86,7 @@ test.describe('Exam assessment', () => { }); }); - test.describe.serial('Modeling exercise assessment', () => { + test.describe.serial('Modeling exercise assessment', { tag: '@slow' }, () => { test.beforeAll('Prepare exam', async ({ browser }) => { examEnd = dayjs().add(45, 'seconds'); const page = await newBrowserPage(browser); @@ -127,7 +127,7 @@ test.describe('Exam assessment', () => { }); }); - test.describe.serial('Text exercise assessment', () => { + test.describe.serial('Text exercise assessment', { tag: '@slow' }, () => { test.beforeAll('Prepare exam', async ({ browser }) => { examEnd = dayjs().add(40, 'seconds'); const page = await newBrowserPage(browser); @@ -154,7 +154,7 @@ test.describe('Exam assessment', () => { }); }); - test.describe('Quiz exercise assessment', () => { + test.describe('Quiz exercise assessment', { tag: '@slow' }, () => { let resultDate: Dayjs; test.beforeAll('Prepare exam', async ({ browser }) => { @@ -223,7 +223,7 @@ test.describe('Exam grading', { tag: '@fast' }, () => { }); }); -test.describe('Exam statistics', { tag: '@slow' }, () => { +test.describe('Exam statistics', { tag: '@sequential' }, () => { let exercise: Exercise; const students = [studentOne, studentTwo, studentThree, studentFour]; diff --git a/src/test/playwright/e2e/exam/ExamParticipation.spec.ts b/src/test/playwright/e2e/exam/ExamParticipation.spec.ts index 13da48cbb7df..0e137ed96528 100644 --- a/src/test/playwright/e2e/exam/ExamParticipation.spec.ts +++ b/src/test/playwright/e2e/exam/ExamParticipation.spec.ts @@ -18,7 +18,6 @@ const textFixture = 'loremIpsum.txt'; const textFixtureShort = 'loremIpsum-short.txt'; test.describe('Exam participation', () => { - test.setTimeout(150000); let course: Course; let exerciseArray: Array = []; let studentTwoName: string; @@ -44,7 +43,7 @@ test.describe('Exam participation', () => { studentFourName = studentFourInfo.name!; }); - test.describe('Early Hand-in', () => { + test.describe('Early Hand-in', { tag: '@slow' }, () => { let exam: Exam; const examTitle = 'exam' + generateUUID(); @@ -118,7 +117,7 @@ test.describe('Exam participation', () => { }); }); - test.describe('Early hand-in with continue and reload page', () => { + test.describe('Early hand-in with continue and reload page', { tag: '@slow' }, () => { let exam: Exam; const examTitle = 'exam' + generateUUID(); @@ -222,7 +221,7 @@ test.describe('Exam participation', () => { }); }); - test.describe('Normal Hand-in', { tag: '@slow' }, () => { + test.describe('Normal Hand-in', { tag: '@sequential' }, () => { let exam: Exam; const examTitle = 'exam' + generateUUID(); @@ -259,7 +258,7 @@ test.describe('Exam participation', () => { }); }); - test.describe('Exam announcements', () => { + test.describe('Exam announcements', { tag: '@slow' }, () => { let exam: Exam; const students = [studentOne, studentTwo]; diff --git a/src/test/playwright/e2e/exam/ExamResults.spec.ts b/src/test/playwright/e2e/exam/ExamResults.spec.ts index c4ab8aa152cf..4b57dfc2e196 100644 --- a/src/test/playwright/e2e/exam/ExamResults.spec.ts +++ b/src/test/playwright/e2e/exam/ExamResults.spec.ts @@ -16,7 +16,6 @@ import { Page } from '@playwright/test'; import { StudentExam } from 'app/entities/student-exam.model'; test.describe('Exam Results', () => { - test.setTimeout(150000); let course: Course; test.beforeEach('Create course', async ({ browser }) => { @@ -129,7 +128,7 @@ test.describe('Exam Results', () => { test( `Check exam ${exerciseTypeString} exercise results`, - { tag: testCase.exerciseType === ExerciseType.PROGRAMMING ? '@slow' : '@fast' }, + { tag: testCase.exerciseType === ExerciseType.PROGRAMMING ? '@sequential' : '@slow' }, async ({ page, login, examParticipation, examResultsPage }) => { await login(studentOne); await waitForExamEnd(examEndDate, page); diff --git a/src/test/playwright/e2e/exam/test-exam/TestExamParticipation.spec.ts b/src/test/playwright/e2e/exam/test-exam/TestExamParticipation.spec.ts index c3f1f4633c37..5b2824dbf8f6 100644 --- a/src/test/playwright/e2e/exam/test-exam/TestExamParticipation.spec.ts +++ b/src/test/playwright/e2e/exam/test-exam/TestExamParticipation.spec.ts @@ -14,8 +14,7 @@ import { expect } from '@playwright/test'; // Common primitives const textFixture = 'loremIpsum-short.txt'; -test.describe('Test exam participation', { tag: '@fast' }, () => { - test.setTimeout(150000); +test.describe('Test exam participation', { tag: '@slow' }, () => { let course: Course; let exerciseArray: Array = []; diff --git a/src/test/playwright/e2e/exam/test-exam/TestExamStudentExams.spec.ts b/src/test/playwright/e2e/exam/test-exam/TestExamStudentExams.spec.ts index 044e971b5733..3ce0a4ef2365 100644 --- a/src/test/playwright/e2e/exam/test-exam/TestExamStudentExams.spec.ts +++ b/src/test/playwright/e2e/exam/test-exam/TestExamStudentExams.spec.ts @@ -17,8 +17,7 @@ const studentNames = new Map(); let examExercise: Exercise; -test.describe('Test Exam - student exams', { tag: '@fast' }, () => { - test.setTimeout(150000); +test.describe('Test Exam - student exams', { tag: '@slow' }, () => { let course: Course; let exam: Exam; diff --git a/src/test/playwright/e2e/exam/test-exam/TestExamTestRun.spec.ts b/src/test/playwright/e2e/exam/test-exam/TestExamTestRun.spec.ts index ea5f119225fd..35e37b163b9d 100644 --- a/src/test/playwright/e2e/exam/test-exam/TestExamTestRun.spec.ts +++ b/src/test/playwright/e2e/exam/test-exam/TestExamTestRun.spec.ts @@ -13,8 +13,7 @@ import { expect } from '@playwright/test'; const textFixture = 'loremIpsum.txt'; const examTitle = 'exam' + generateUUID(); -test.describe('Test exam test run', { tag: '@fast' }, () => { - test.setTimeout(150000); +test.describe('Test exam test run', { tag: '@slow' }, () => { let course: Course; let exam: Exam; let exerciseArray: Array = []; diff --git a/src/test/playwright/e2e/exercise/ExerciseImport.spec.ts b/src/test/playwright/e2e/exercise/ExerciseImport.spec.ts index edc6e35e9f80..d54c4f4268db 100644 --- a/src/test/playwright/e2e/exercise/ExerciseImport.spec.ts +++ b/src/test/playwright/e2e/exercise/ExerciseImport.spec.ts @@ -125,7 +125,7 @@ test.describe('Import exercises', () => { test( 'Imports programming exercise', - { tag: '@slow' }, + { tag: '@sequential' }, async ({ login, page, courseManagementExercises, programmingExerciseCreation, courseOverview, programmingExerciseEditor }) => { await login(instructor, `/course-management/${secondCourse.id}/exercises`); await courseManagementExercises.importProgrammingExercise(); diff --git a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseAssessment.spec.ts b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseAssessment.spec.ts index d4575589cf42..0299bd423774 100644 --- a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseAssessment.spec.ts +++ b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseAssessment.spec.ts @@ -17,7 +17,7 @@ const tutorCodeFeedback = 'The input parameter should be mentioned in javadoc!'; const tutorCodeFeedbackPoints = -2; const complaint = "That feedback wasn't very useful!"; -test.describe('Programming exercise assessment', { tag: '@slow' }, () => { +test.describe('Programming exercise assessment', { tag: '@sequential' }, () => { let course: Course; let exercise: ProgrammingExercise; let dueDate: dayjs.Dayjs; diff --git a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseParticipation.spec.ts b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseParticipation.spec.ts index b6a38b85fbf7..cd28fe48b5b0 100644 --- a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseParticipation.spec.ts +++ b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseParticipation.spec.ts @@ -20,7 +20,7 @@ import { Team } from 'app/entities/team.model'; import { ProgrammingExerciseOverviewPage } from '../../../support/pageobjects/exercises/programming/ProgrammingExerciseOverviewPage'; import { Participation } from 'app/entities/participation/participation.model'; -test.describe('Programming exercise participation', { tag: '@slow' }, () => { +test.describe('Programming exercise participation', { tag: '@sequential' }, () => { let course: Course; test.beforeEach('Create course', async ({ login, courseManagementAPIRequests }) => { diff --git a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseStaticCodeAnalysis.spec.ts b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseStaticCodeAnalysis.spec.ts index e5cc38f72fbf..a5da3a899584 100644 --- a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseStaticCodeAnalysis.spec.ts +++ b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseStaticCodeAnalysis.spec.ts @@ -6,7 +6,7 @@ import { admin, studentOne } from '../../../support/users'; import { test } from '../../../support/fixtures'; import { expect } from '@playwright/test'; -test.describe('Static code analysis tests', { tag: '@slow' }, () => { +test.describe('Static code analysis tests', { tag: '@sequential' }, () => { let course: Course; let exercise: ProgrammingExercise; diff --git a/src/test/playwright/package.json b/src/test/playwright/package.json index 6721f5a919a0..f3161e557b77 100644 --- a/src/test/playwright/package.json +++ b/src/test/playwright/package.json @@ -15,12 +15,12 @@ "uuid": "11.0.3" }, "scripts": { - "playwright:test": "npm-run-all --serial --continue-on-error playwright:test:fast playwright:test:slow merge-reports", - "playwright:test:fast": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-fast.xml playwright test e2e --project=fast-tests", - "playwright:test:slow": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-slow.xml playwright test e2e --project=slow-tests --workers 1", + "playwright:test": "npm-run-all --serial --continue-on-error playwright:test:parallel playwright:test:sequential merge-reports", + "playwright:test:parallel": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-parallel.xml playwright test e2e --project=fast-tests --project=slow-tests", + "playwright:test:sequential": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-sequential.xml playwright test e2e --project=sequential-tests --workers 1", "playwright:open": "playwright test e2e --ui", "playwright:setup": "npx playwright install --with-deps chromium && playwright test init", - "merge-reports": "junit-merge ./test-reports/results-fast.xml ./test-reports/results-slow.xml -o ./test-reports/results.xml", + "merge-reports": "junit-merge ./test-reports/results-parallel.xml ./test-reports/results-sequential.xml -o ./test-reports/results.xml", "update": "ncu -i --format group" } } diff --git a/src/test/playwright/playwright.config.ts b/src/test/playwright/playwright.config.ts index 2e64acd023b8..a963e42fadcb 100644 --- a/src/test/playwright/playwright.config.ts +++ b/src/test/playwright/playwright.config.ts @@ -32,18 +32,28 @@ export default defineConfig({ /* Configure projects for fast and slow tests */ projects: [ - // Tests with @slow tag + // Tests with @fast tag or without any tags. These are the lightweight tests with lower timeout. + { + name: 'fast-tests', + grep: /@fast|^[^@]*$/, + timeout: (parseNumber(process.env.FAST_TEST_TIMEOUT_SECONDS) ?? 45) * 1000, + use: { ...devices['Desktop Chrome'] }, + }, + // Tests with @slow tag. These tests are expected to run longer + // than faster tests and have higher timeout. { name: 'slow-tests', grep: /@slow/, timeout: (parseNumber(process.env.SLOW_TEST_TIMEOUT_SECONDS) ?? 180) * 1000, use: { ...devices['Desktop Chrome'] }, }, - // Tests with @fast tag or without any tags + // Tests with @sequential tag. These tests are triggering programming exercise submissions. + // Running only one programming exercise evaluation at a time could make the tests more stable. + // Thus, it is recommended to run this project with a single worker. { - name: 'fast-tests', - grep: /@fast|^[^@]*$/, - timeout: (parseNumber(process.env.FAST_TEST_TIMEOUT_SECONDS) ?? 45) * 1000, + name: 'sequential-tests', + grep: /@sequential/, + timeout: (parseNumber(process.env.SLOW_TEST_TIMEOUT_SECONDS) ?? 180) * 1000, use: { ...devices['Desktop Chrome'] }, }, ],