fix: fix unit and e2e tests, further refactoring

gemini-testing · Dec 11, 2023 · 1820225 · 1820225
1 parent 9ec97dd
commit 1820225
Show file tree

Hide file tree

Showing 31 changed files with 701 additions and 550 deletions.
diff --git a/hermione.ts b/hermione.ts
@@ -6,37 +6,42 @@ import PQueue from 'p-queue';
 import {CommanderStatic} from '@gemini-testing/commander';
 
 import {cliCommands} from './lib/cli-commands';
-import {hasDiff} from './lib/common-utils';
+import {hasFailedImages} from './lib/common-utils';
 import {parseConfig} from './lib/config';
-import {ERROR, FAIL, SUCCESS, ToolName} from './lib/constants';
+import {SKIPPED, SUCCESS, TestStatus, ToolName, UNKNOWN_ATTEMPT} from './lib/constants';
 import {HtmlReporter} from './lib/plugin-api';
 import {StaticReportBuilder} from './lib/report-builder/static';
 import {formatTestResult, logPathToHtmlReport, logError} from './lib/server-utils';
 import {SqliteClient} from './lib/sqlite-client';
-import {HermioneTestAdapter, ReporterTestResult} from './lib/test-adapter';
-import {TestAttemptManager} from './lib/test-attempt-manager';
-import {HtmlReporterApi, ReporterConfig, ReporterOptions} from './lib/types';
+import {HtmlReporterApi, ImageInfoFull, ReporterOptions} from './lib/types';
 import {createWorkers, CreateWorkersRunner} from './lib/workers/create-workers';
 
-let workers: ReturnType<typeof createWorkers>;
-
 export = (hermione: Hermione, opts: Partial<ReporterOptions>): void => {
-    if (hermione.isWorker()) {
+    if (hermione.isWorker() || !opts.enabled) {
         return;
     }
 
     const config = parseConfig(opts);
 
-    if (!config.enabled) {
-        return;
-    }
-
     const htmlReporter = HtmlReporter.create(config, {toolName: ToolName.Hermione});
 
     (hermione as Hermione & HtmlReporterApi).htmlReporter = htmlReporter;
 
     let isCliCommandLaunched = false;
     let handlingTestResults: Promise<void>;
+    let staticReportBuilder: StaticReportBuilder;
+
+    const withMiddleware = <T extends (...args: unknown[]) => unknown>(fn: T):
+        (...args: Parameters<T>) => ReturnType<T> | undefined => {
+        return (...args: unknown[]) => {
+            // If any CLI command was launched, e.g. merge-reports, we need to interrupt regular flow
+            if (isCliCommandLaunched) {
+                return;
+            }
+
+            return fn.call(undefined, ...args) as ReturnType<T>;
+        };
+    };
 
     hermione.on(hermione.events.CLI, (commander: CommanderStatic) => {
         _.values(cliCommands).forEach((command: string) => {
@@ -49,88 +54,75 @@ export = (hermione: Hermione, opts: Partial<ReporterOptions>): void => {
         });
     });
 
-    hermione.on(hermione.events.INIT, async () => {
-        if (isCliCommandLaunched) {
-            return;
-        }
-
+    hermione.on(hermione.events.INIT, withMiddleware(async () => {
         const dbClient = await SqliteClient.create({htmlReporter, reportPath: config.path});
-        const testAttemptManager = new TestAttemptManager();
-        const staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient, testAttemptManager});
+        staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient});
 
         handlingTestResults = Promise.all([
             staticReportBuilder.saveStaticFiles(),
-            handleTestResults(hermione, staticReportBuilder, config)
+            handleTestResults(hermione, staticReportBuilder)
         ]).then(async () => {
             await staticReportBuilder.finalize();
         }).then(async () => {
             await htmlReporter.emitAsync(htmlReporter.events.REPORT_SAVED, {reportPath: config.path});
         });
-    });
 
-    hermione.on(hermione.events.RUNNER_START, (runner) => {
-        workers = createWorkers(runner as unknown as CreateWorkersRunner);
-    });
+        htmlReporter.emit(htmlReporter.events.DATABASE_CREATED, dbClient.getRawConnection());
+    }));
 
-    hermione.on(hermione.events.RUNNER_END, async () => {
+    hermione.on(hermione.events.RUNNER_START, withMiddleware((runner) => {
+        staticReportBuilder.registerWorkers(createWorkers(runner as unknown as CreateWorkersRunner));
+    }));
+
+    hermione.on(hermione.events.RUNNER_END, withMiddleware(async () => {
         try {
             await handlingTestResults;
 
             logPathToHtmlReport(config);
         } catch (e: unknown) {
             logError(e as Error);
         }
-    });
+    }));
 };
 
-async function handleTestResults(hermione: Hermione, reportBuilder: StaticReportBuilder, pluginConfig: ReporterConfig): Promise<void> {
-    const {path: reportPath} = pluginConfig;
-    const {imageHandler} = reportBuilder;
-
-    const failHandler = async (testResult: HermioneTestResult): Promise<ReporterTestResult> => {
-        const status = hasDiff(testResult.assertViewResults as {name?: string}[]) ? FAIL : ERROR;
-        const attempt = reportBuilder.testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, status);
-        const formattedResult = formatTestResult(testResult, status, attempt, reportBuilder);
-
-        const actions: Promise<unknown>[] = [imageHandler.saveTestImages(formattedResult, workers)];
-
-        if (pluginConfig.saveErrorDetails && formattedResult.errorDetails) {
-            actions.push((formattedResult as HermioneTestAdapter).saveErrorDetails(reportPath));
-        }
-
-        await Promise.all(actions);
-
-        return formattedResult;
-    };
-
-    const addFail = (formattedResult: ReporterTestResult): ReporterTestResult => {
-        return reportBuilder.addFail(formattedResult);
-    };
-
+async function handleTestResults(hermione: Hermione, reportBuilder: StaticReportBuilder): Promise<void> {
     return new Promise((resolve, reject) => {
         const queue = new PQueue({concurrency: os.cpus().length});
         const promises: Promise<unknown>[] = [];
 
         hermione.on(hermione.events.TEST_PASS, testResult => {
             promises.push(queue.add(async () => {
-                const attempt = reportBuilder.testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, FAIL);
-                const formattedResult = formatTestResult(testResult, SUCCESS, attempt, reportBuilder);
-                await imageHandler.saveTestImages(formattedResult, workers);
-
-                return reportBuilder.addSuccess(formattedResult);
+                const formattedResult = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder);
+                await reportBuilder.addSuccess(formattedResult);
             }).catch(reject));
         });
 
         hermione.on(hermione.events.RETRY, testResult => {
-            promises.push(queue.add(() => failHandler(testResult).then(addFail)).catch(reject));
+            promises.push(queue.add(async () => {
+                const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR;
+
+                const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder);
+
+                await reportBuilder.addFail(formattedResult);
+            }).catch(reject));
         });
 
         hermione.on(hermione.events.TEST_FAIL, testResult => {
-            promises.push(queue.add(() => failHandler(testResult).then(addFail)).catch(reject));
+            promises.push(queue.add(async () => {
+                const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR;
+
+                const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder);
+
+                await reportBuilder.addFail(formattedResult);
+            }).catch(reject));
         });
 
         hermione.on(hermione.events.TEST_PENDING, testResult => {
-            promises.push(queue.add(() => failHandler(testResult as HermioneTestResult).then((testResult) => reportBuilder.addSkipped(testResult)).catch(reject)));
+            promises.push(queue.add(async () => {
+                const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder);
+
+                await reportBuilder.addSkipped(formattedResult);
+            }).catch(reject));
         });
 
         hermione.on(hermione.events.RUNNER_END, () => {

diff --git a/lib/common-utils.ts b/lib/common-utils.ts
@@ -5,7 +5,7 @@ import axios, {AxiosRequestConfig} from 'axios';
 import {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED, TestStatus} from './constants';
 
 import {UNCHECKED, INDETERMINATE, CHECKED} from './constants/checked-statuses';
-import {ImageData, ImageBase64, ImageInfoFull, TestError, ImageInfoError} from './types';
+import {ImageData, ImageBase64, ImageInfoFull, TestError, ImageInfoFail} from './types';
 import {ErrorName, ImageDiffError, NoRefImageError} from './errors';
 import {ReporterTestResult} from './test-adapter';
 export const getShortMD5 = (str: string): string => {
@@ -104,17 +104,17 @@ export const hasNoRefImageErrors = ({assertViewResults = []}: {assertViewResults
     return assertViewResults.some((assertViewResult) => isNoRefImageError(assertViewResult));
 };
 
-export const hasFailedImages = (result: {imagesInfo?: ImageInfoFull[]}): boolean => {
-    const {imagesInfo = []} = result;
-
+export const hasFailedImages = (imagesInfo: ImageInfoFull[] = []): boolean => {
     return imagesInfo.some((imageInfo: ImageInfoFull) => {
-        return !isAssertViewError((imageInfo as ImageInfoError).error) &&
-            (isErrorStatus(imageInfo.status) || isFailStatus(imageInfo.status));
+        return (imageInfo as ImageInfoFail).stateName &&
+            (isErrorStatus(imageInfo.status) || isFailStatus(imageInfo.status) || isNoRefImageError(imageInfo) || isImageDiffError(imageInfo));
     });
 };
 
-export const hasResultFails = (testResult: {status: TestStatus, imagesInfo?: ImageInfoFull[]}): boolean => {
-    return hasFailedImages(testResult) || isErrorStatus(testResult.status) || isFailStatus(testResult.status);
+export const hasUnrelatedToScreenshotsErrors = (error: TestError): boolean => {
+    return !isNoRefImageError(error) &&
+        !isImageDiffError(error) &&
+        !isAssertViewError(error);
 };
 
 export const getError = (error?: TestError): undefined | Pick<TestError, 'name' | 'message' | 'stack' | 'stateName'> => {
@@ -131,7 +131,11 @@ export const hasDiff = (assertViewResults: {name?: string}[]): boolean => {
 
 /* This method tries to determine true status of testResult by using fields like error, imagesInfo */
 export const determineStatus = (testResult: Pick<ReporterTestResult, 'status' | 'error' | 'imagesInfo'>): TestStatus => {
-    if (!hasFailedImages(testResult) && !isSkippedStatus(testResult.status) && isEmpty(testResult.error)) {
+    if (
+        !hasFailedImages(testResult.imagesInfo) &&
+        !isSkippedStatus(testResult.status) &&
+        (!testResult.error || !hasUnrelatedToScreenshotsErrors(testResult.error))
+    ) {
         return SUCCESS;
     }
 

diff --git a/lib/constants/tests.ts b/lib/constants/tests.ts
@@ -1 +1,5 @@
+export const HERMIONE_TITLE_DELIMITER = ' ';
+
 export const PWT_TITLE_DELIMITER = ' › ';
+
+export const UNKNOWN_ATTEMPT = -1;