Run slow parallel tests as a separate project to assign higher timeou…

…t for all. Rename @slow tests to @Sequential to make it clearer.
ls1intum · Nov 18, 2024 · 18beef5 · 18beef5
1 parent 460b074
commit 18beef5
Show file tree

Hide file tree

Showing 12 changed files with 37 additions and 32 deletions.
diff --git a/src/test/playwright/e2e/exam/ExamAssessment.spec.ts b/src/test/playwright/e2e/exam/ExamAssessment.spec.ts
@@ -54,12 +54,12 @@ test.beforeAll('Create course', async ({ browser }) => {
 });
 
 test.describe('Exam assessment', () => {
-    test.setTimeout(150000);
+    test.describe.configure({ mode: 'serial' });
     let programmingAssessmentSuccessful = false;
     let modelingAssessmentSuccessful = false;
     let textAssessmentSuccessful = false;
 
-    test.describe.serial('Programming exercise assessment', { tag: '@slow' }, () => {
+    test.describe.serial('Programming exercise assessment', { tag: '@sequential' }, () => {
         test.beforeAll('Prepare exam', async ({ browser }) => {
             examEnd = dayjs().add(2, 'minutes');
             const page = await newBrowserPage(browser);
@@ -86,7 +86,7 @@ test.describe('Exam assessment', () => {
         });
     });
 
-    test.describe.serial('Modeling exercise assessment', () => {
+    test.describe.serial('Modeling exercise assessment', { tag: '@slow' }, () => {
         test.beforeAll('Prepare exam', async ({ browser }) => {
             examEnd = dayjs().add(45, 'seconds');
             const page = await newBrowserPage(browser);
@@ -127,7 +127,7 @@ test.describe('Exam assessment', () => {
         });
     });
 
-    test.describe.serial('Text exercise assessment', () => {
+    test.describe.serial('Text exercise assessment', { tag: '@slow' }, () => {
         test.beforeAll('Prepare exam', async ({ browser }) => {
             examEnd = dayjs().add(40, 'seconds');
             const page = await newBrowserPage(browser);
@@ -154,7 +154,7 @@ test.describe('Exam assessment', () => {
         });
     });
 
-    test.describe('Quiz exercise assessment', () => {
+    test.describe('Quiz exercise assessment', { tag: '@slow' }, () => {
         let resultDate: Dayjs;
 
         test.beforeAll('Prepare exam', async ({ browser }) => {
@@ -223,7 +223,7 @@ test.describe('Exam grading', { tag: '@fast' }, () => {
     });
 });
 
-test.describe('Exam statistics', { tag: '@slow' }, () => {
+test.describe('Exam statistics', { tag: '@sequential' }, () => {
     let exercise: Exercise;
     const students = [studentOne, studentTwo, studentThree, studentFour];
 

diff --git a/src/test/playwright/e2e/exam/ExamParticipation.spec.ts b/src/test/playwright/e2e/exam/ExamParticipation.spec.ts
@@ -18,7 +18,6 @@ const textFixture = 'loremIpsum.txt';
 const textFixtureShort = 'loremIpsum-short.txt';
 
 test.describe('Exam participation', () => {
-    test.setTimeout(150000);
     let course: Course;
     let exerciseArray: Array<Exercise> = [];
     let studentTwoName: string;
@@ -44,7 +43,7 @@ test.describe('Exam participation', () => {
         studentFourName = studentFourInfo.name!;
     });
 
-    test.describe('Early Hand-in', () => {
+    test.describe('Early Hand-in', { tag: '@slow' }, () => {
         let exam: Exam;
         const examTitle = 'exam' + generateUUID();
 
@@ -118,7 +117,7 @@ test.describe('Exam participation', () => {
         });
     });
 
-    test.describe('Early hand-in with continue and reload page', () => {
+    test.describe('Early hand-in with continue and reload page', { tag: '@slow' }, () => {
         let exam: Exam;
         const examTitle = 'exam' + generateUUID();
 
@@ -222,7 +221,7 @@ test.describe('Exam participation', () => {
         });
     });
 
-    test.describe('Normal Hand-in', { tag: '@slow' }, () => {
+    test.describe('Normal Hand-in', { tag: '@sequential' }, () => {
         let exam: Exam;
         const examTitle = 'exam' + generateUUID();
 
@@ -259,7 +258,7 @@ test.describe('Exam participation', () => {
         });
     });
 
-    test.describe('Exam announcements', () => {
+    test.describe('Exam announcements', { tag: '@slow' }, () => {
         let exam: Exam;
         const students = [studentOne, studentTwo];
 

diff --git a/src/test/playwright/e2e/exam/ExamResults.spec.ts b/src/test/playwright/e2e/exam/ExamResults.spec.ts
@@ -16,7 +16,6 @@ import { Page } from '@playwright/test';
 import { StudentExam } from 'app/entities/student-exam.model';
 
 test.describe('Exam Results', () => {
-    test.setTimeout(150000);
     let course: Course;
 
     test.beforeEach('Create course', async ({ browser }) => {
@@ -129,7 +128,7 @@ test.describe('Exam Results', () => {
 
                 test(
                     `Check exam ${exerciseTypeString} exercise results`,
-                    { tag: testCase.exerciseType === ExerciseType.PROGRAMMING ? '@slow' : '@fast' },
+                    { tag: testCase.exerciseType === ExerciseType.PROGRAMMING ? '@sequential' : '@slow' },
                     async ({ page, login, examParticipation, examResultsPage }) => {
                         await login(studentOne);
                         await waitForExamEnd(examEndDate, page);

diff --git a/src/test/playwright/e2e/exam/test-exam/TestExamParticipation.spec.ts b/src/test/playwright/e2e/exam/test-exam/TestExamParticipation.spec.ts
@@ -14,8 +14,7 @@ import { expect } from '@playwright/test';
 // Common primitives
 const textFixture = 'loremIpsum-short.txt';
 
-test.describe('Test exam participation', { tag: '@fast' }, () => {
-    test.setTimeout(150000);
+test.describe('Test exam participation', { tag: '@slow' }, () => {
     let course: Course;
     let exerciseArray: Array<Exercise> = [];
 

diff --git a/src/test/playwright/e2e/exam/test-exam/TestExamStudentExams.spec.ts b/src/test/playwright/e2e/exam/test-exam/TestExamStudentExams.spec.ts
@@ -17,8 +17,7 @@ const studentNames = new Map<UserCredentials, string>();
 
 let examExercise: Exercise;
 
-test.describe('Test Exam - student exams', { tag: '@fast' }, () => {
-    test.setTimeout(150000);
+test.describe('Test Exam - student exams', { tag: '@slow' }, () => {
     let course: Course;
     let exam: Exam;
 

diff --git a/src/test/playwright/e2e/exam/test-exam/TestExamTestRun.spec.ts b/src/test/playwright/e2e/exam/test-exam/TestExamTestRun.spec.ts
@@ -13,8 +13,7 @@ import { expect } from '@playwright/test';
 const textFixture = 'loremIpsum.txt';
 const examTitle = 'exam' + generateUUID();
 
-test.describe('Test exam test run', { tag: '@fast' }, () => {
-    test.setTimeout(150000);
+test.describe('Test exam test run', { tag: '@slow' }, () => {
     let course: Course;
     let exam: Exam;
     let exerciseArray: Array<Exercise> = [];

diff --git a/src/test/playwright/e2e/exercise/ExerciseImport.spec.ts b/src/test/playwright/e2e/exercise/ExerciseImport.spec.ts
@@ -125,7 +125,7 @@ test.describe('Import exercises', () => {
 
         test(
             'Imports programming exercise',
-            { tag: '@slow' },
+            { tag: '@sequential' },
             async ({ login, page, courseManagementExercises, programmingExerciseCreation, courseOverview, programmingExerciseEditor }) => {
                 await login(instructor, `/course-management/${secondCourse.id}/exercises`);
                 await courseManagementExercises.importProgrammingExercise();

diff --git a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseAssessment.spec.ts b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseAssessment.spec.ts
@@ -17,7 +17,7 @@ const tutorCodeFeedback = 'The input parameter should be mentioned in javadoc!';
 const tutorCodeFeedbackPoints = -2;
 const complaint = "That feedback wasn't very useful!";
 
-test.describe('Programming exercise assessment', { tag: '@slow' }, () => {
+test.describe('Programming exercise assessment', { tag: '@sequential' }, () => {
     let course: Course;
     let exercise: ProgrammingExercise;
     let dueDate: dayjs.Dayjs;

diff --git a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseParticipation.spec.ts b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseParticipation.spec.ts
@@ -20,7 +20,7 @@ import { Team } from 'app/entities/team.model';
 import { ProgrammingExerciseOverviewPage } from '../../../support/pageobjects/exercises/programming/ProgrammingExerciseOverviewPage';
 import { Participation } from 'app/entities/participation/participation.model';
 
-test.describe('Programming exercise participation', { tag: '@slow' }, () => {
+test.describe('Programming exercise participation', { tag: '@sequential' }, () => {
     let course: Course;
 
     test.beforeEach('Create course', async ({ login, courseManagementAPIRequests }) => {

diff --git a/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseStaticCodeAnalysis.spec.ts b/src/test/playwright/e2e/exercise/programming/ProgrammingExerciseStaticCodeAnalysis.spec.ts
@@ -6,7 +6,7 @@ import { admin, studentOne } from '../../../support/users';
 import { test } from '../../../support/fixtures';
 import { expect } from '@playwright/test';
 
-test.describe('Static code analysis tests', { tag: '@slow' }, () => {
+test.describe('Static code analysis tests', { tag: '@sequential' }, () => {
     let course: Course;
     let exercise: ProgrammingExercise;
 

diff --git a/src/test/playwright/package.json b/src/test/playwright/package.json
@@ -15,12 +15,12 @@
         "uuid": "11.0.3"
     },
     "scripts": {
-        "playwright:test": "npm-run-all --serial --continue-on-error playwright:test:fast playwright:test:slow merge-reports",
-        "playwright:test:fast": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-fast.xml playwright test e2e --project=fast-tests",
-        "playwright:test:slow": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-slow.xml playwright test e2e --project=slow-tests --workers 1",
+        "playwright:test": "npm-run-all --serial --continue-on-error playwright:test:parallel playwright:test:sequential merge-reports",
+        "playwright:test:parallel": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-parallel.xml playwright test e2e --project=fast-tests --project=slow-tests",
+        "playwright:test:sequential": "cross-env PLAYWRIGHT_JUNIT_OUTPUT_NAME=./test-reports/results-sequential.xml playwright test e2e --project=sequential-tests --workers 1",
         "playwright:open": "playwright test e2e --ui",
         "playwright:setup": "npx playwright install --with-deps chromium && playwright test init",
-        "merge-reports": "junit-merge ./test-reports/results-fast.xml ./test-reports/results-slow.xml -o ./test-reports/results.xml",
+        "merge-reports": "junit-merge ./test-reports/results-parallel.xml ./test-reports/results-sequential.xml -o ./test-reports/results.xml",
         "update": "ncu -i --format group"
     }
 }
diff --git a/src/test/playwright/playwright.config.ts b/src/test/playwright/playwright.config.ts
@@ -32,18 +32,28 @@ export default defineConfig({
 
     /* Configure projects for fast and slow tests */
     projects: [
-        // Tests with @slow tag
+        // Tests with @fast tag or without any tags. These are the lightweight tests with lower timeout.
+        {
+            name: 'fast-tests',
+            grep: /@fast|^[^@]*$/,
+            timeout: (parseNumber(process.env.FAST_TEST_TIMEOUT_SECONDS) ?? 45) * 1000,
+            use: { ...devices['Desktop Chrome'] },
+        },
+        // Tests with @slow tag. These tests are expected to run longer
+        // than faster tests and have higher timeout.
         {
             name: 'slow-tests',
             grep: /@slow/,
             timeout: (parseNumber(process.env.SLOW_TEST_TIMEOUT_SECONDS) ?? 180) * 1000,
             use: { ...devices['Desktop Chrome'] },
         },
-        // Tests with @fast tag or without any tags
+        // Tests with @sequential tag. These tests are triggering programming exercise submissions.
+        // Running only one programming exercise evaluation at a time could make the tests more stable.
+        // Thus, it is recommended to run this project with a single worker.
         {
-            name: 'fast-tests',
-            grep: /@fast|^[^@]*$/,
-            timeout: (parseNumber(process.env.FAST_TEST_TIMEOUT_SECONDS) ?? 45) * 1000,
+            name: 'sequential-tests',
+            grep: /@sequential/,
+            timeout: (parseNumber(process.env.SLOW_TEST_TIMEOUT_SECONDS) ?? 180) * 1000,
             use: { ...devices['Desktop Chrome'] },
         },
     ],