refactor: prepare html-reporter for pwt GUI integration (#522)

* refactor: rename sqlite-adapter to sqlite-client * chore: re-write hermione.js to typescript * refactor: get rid of init method in sqlite-client * refactor: streamline test result types in sqlite-client * refactor: handle attempt number explicitly * fix: unit test and minor bug fixes * refactor: get rid of plugin-adapter * fix: fix review issues * test: fix tests building * fix: fix test attempt manager, review issues, status computing * fix: fix unit and e2e tests, further refactoring * test: update chrome installation flow * test: update browser-utils package-lock * test: grant chrome for testing exec permissions
gemini-testing · Dec 12, 2023 · 57589a5 · 57589a5
1 parent ba202d7
commit 57589a5
Show file tree

Hide file tree

Showing 52 changed files with 1,672 additions and 3,544 deletions.
diff --git a/hermione.js b/hermione.js
diff --git a/hermione.ts b/hermione.ts
@@ -0,0 +1,132 @@
+import os from 'os';
+import path from 'path';
+import Hermione, {TestResult as HermioneTestResult} from 'hermione';
+import _ from 'lodash';
+import PQueue from 'p-queue';
+import {CommanderStatic} from '@gemini-testing/commander';
+
+import {cliCommands} from './lib/cli-commands';
+import {hasFailedImages} from './lib/common-utils';
+import {parseConfig} from './lib/config';
+import {SKIPPED, SUCCESS, TestStatus, ToolName, UNKNOWN_ATTEMPT} from './lib/constants';
+import {HtmlReporter} from './lib/plugin-api';
+import {StaticReportBuilder} from './lib/report-builder/static';
+import {formatTestResult, logPathToHtmlReport, logError} from './lib/server-utils';
+import {SqliteClient} from './lib/sqlite-client';
+import {HtmlReporterApi, ImageInfoFull, ReporterOptions} from './lib/types';
+import {createWorkers, CreateWorkersRunner} from './lib/workers/create-workers';
+
+export = (hermione: Hermione, opts: Partial<ReporterOptions>): void => {
+    if (hermione.isWorker() || !opts.enabled) {
+        return;
+    }
+
+    const config = parseConfig(opts);
+
+    const htmlReporter = HtmlReporter.create(config, {toolName: ToolName.Hermione});
+
+    (hermione as Hermione & HtmlReporterApi).htmlReporter = htmlReporter;
+
+    let isCliCommandLaunched = false;
+    let handlingTestResults: Promise<void>;
+    let staticReportBuilder: StaticReportBuilder;
+
+    const withMiddleware = <T extends (...args: unknown[]) => unknown>(fn: T):
+        (...args: Parameters<T>) => ReturnType<T> | undefined => {
+        return (...args: unknown[]) => {
+            // If any CLI command was launched, e.g. merge-reports, we need to interrupt regular flow
+            if (isCliCommandLaunched) {
+                return;
+            }
+
+            return fn.call(undefined, ...args) as ReturnType<T>;
+        };
+    };
+
+    hermione.on(hermione.events.CLI, (commander: CommanderStatic) => {
+        _.values(cliCommands).forEach((command: string) => {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            require(path.resolve(__dirname, 'lib/cli-commands', command))(commander, config, hermione);
+
+            commander.prependListener(`command:${command}`, () => {
+                isCliCommandLaunched = true;
+            });
+        });
+    });
+
+    hermione.on(hermione.events.INIT, withMiddleware(async () => {
+        const dbClient = await SqliteClient.create({htmlReporter, reportPath: config.path});
+        staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient});
+
+        handlingTestResults = Promise.all([
+            staticReportBuilder.saveStaticFiles(),
+            handleTestResults(hermione, staticReportBuilder)
+        ]).then(async () => {
+            await staticReportBuilder.finalize();
+        }).then(async () => {
+            await htmlReporter.emitAsync(htmlReporter.events.REPORT_SAVED, {reportPath: config.path});
+        });
+
+        htmlReporter.emit(htmlReporter.events.DATABASE_CREATED, dbClient.getRawConnection());
+    }));
+
+    hermione.on(hermione.events.RUNNER_START, withMiddleware((runner) => {
+        staticReportBuilder.registerWorkers(createWorkers(runner as unknown as CreateWorkersRunner));
+    }));
+
+    hermione.on(hermione.events.RUNNER_END, withMiddleware(async () => {
+        try {
+            await handlingTestResults;
+
+            logPathToHtmlReport(config);
+        } catch (e: unknown) {
+            logError(e as Error);
+        }
+    }));
+};
+
+async function handleTestResults(hermione: Hermione, reportBuilder: StaticReportBuilder): Promise<void> {
+    return new Promise((resolve, reject) => {
+        const queue = new PQueue({concurrency: os.cpus().length});
+        const promises: Promise<unknown>[] = [];
+
+        hermione.on(hermione.events.TEST_PASS, testResult => {
+            promises.push(queue.add(async () => {
+                const formattedResult = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder);
+                await reportBuilder.addSuccess(formattedResult);
+            }).catch(reject));
+        });
+
+        hermione.on(hermione.events.RETRY, testResult => {
+            promises.push(queue.add(async () => {
+                const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR;
+
+                const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder);
+
+                await reportBuilder.addFail(formattedResult);
+            }).catch(reject));
+        });
+
+        hermione.on(hermione.events.TEST_FAIL, testResult => {
+            promises.push(queue.add(async () => {
+                const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR;
+
+                const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder);
+
+                await reportBuilder.addFail(formattedResult);
+            }).catch(reject));
+        });
+
+        hermione.on(hermione.events.TEST_PENDING, testResult => {
+            promises.push(queue.add(async () => {
+                const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder);
+
+                await reportBuilder.addSkipped(formattedResult);
+            }).catch(reject));
+        });
+
+        hermione.on(hermione.events.RUNNER_END, () => {
+            return Promise.all(promises).then(() => resolve(), reject);
+        });
+    });
+}
diff --git a/lib/common-utils.ts b/lib/common-utils.ts
@@ -5,8 +5,9 @@ import axios, {AxiosRequestConfig} from 'axios';
 import {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED, TestStatus} from './constants';
 
 import {UNCHECKED, INDETERMINATE, CHECKED} from './constants/checked-statuses';
-import {ImageData, ImageBase64, ImageInfoFull, TestError, ImageInfoError} from './types';
+import {ImageData, ImageBase64, ImageInfoFull, TestError, ImageInfoFail} from './types';
 import {ErrorName, ImageDiffError, NoRefImageError} from './errors';
+import {ReporterTestResult} from './test-adapter';
 export const getShortMD5 = (str: string): string => {
     return crypto.createHash('md5').update(str, 'ascii').digest('hex').substr(0, 7);
 };
@@ -29,7 +30,7 @@ export const isErrorStatus = (status: TestStatus): boolean => status === ERROR;
 export const isSkippedStatus = (status: TestStatus): boolean => status === SKIPPED;
 export const isUpdatedStatus = (status: TestStatus): boolean => status === UPDATED;
 
-export const determineStatus = (statuses: TestStatus[]): TestStatus | null => {
+export const determineFinalStatus = (statuses: TestStatus[]): TestStatus | null => {
     if (!statuses.length) {
         return SUCCESS;
     }
@@ -103,17 +104,17 @@ export const hasNoRefImageErrors = ({assertViewResults = []}: {assertViewResults
     return assertViewResults.some((assertViewResult) => isNoRefImageError(assertViewResult));
 };
 
-const hasFailedImages = (result: {imagesInfo?: ImageInfoFull[]}): boolean => {
-    const {imagesInfo = []} = result;
-
+export const hasFailedImages = (imagesInfo: ImageInfoFull[] = []): boolean => {
     return imagesInfo.some((imageInfo: ImageInfoFull) => {
-        return !isAssertViewError((imageInfo as ImageInfoError).error) &&
-            (isErrorStatus(imageInfo.status) || isFailStatus(imageInfo.status));
+        return (imageInfo as ImageInfoFail).stateName &&
+            (isErrorStatus(imageInfo.status) || isFailStatus(imageInfo.status) || isNoRefImageError(imageInfo) || isImageDiffError(imageInfo));
     });
 };
 
-export const hasResultFails = (testResult: {status: TestStatus, imagesInfo?: ImageInfoFull[]}): boolean => {
-    return hasFailedImages(testResult) || isErrorStatus(testResult.status) || isFailStatus(testResult.status);
+export const hasUnrelatedToScreenshotsErrors = (error: TestError): boolean => {
+    return !isNoRefImageError(error) &&
+        !isImageDiffError(error) &&
+        !isAssertViewError(error);
 };
 
 export const getError = (error?: TestError): undefined | Pick<TestError, 'name' | 'message' | 'stack' | 'stateName'> => {
@@ -128,6 +129,28 @@ export const hasDiff = (assertViewResults: {name?: string}[]): boolean => {
     return assertViewResults.some((result) => isImageDiffError(result as {name?: string}));
 };
 
+/* This method tries to determine true status of testResult by using fields like error, imagesInfo */
+export const determineStatus = (testResult: Pick<ReporterTestResult, 'status' | 'error' | 'imagesInfo'>): TestStatus => {
+    if (
+        !hasFailedImages(testResult.imagesInfo) &&
+        !isSkippedStatus(testResult.status) &&
+        (!testResult.error || !hasUnrelatedToScreenshotsErrors(testResult.error))
+    ) {
+        return SUCCESS;
+    }
+
+    const imageErrors = (testResult.imagesInfo ?? []).map(imagesInfo => (imagesInfo as {error: {name?: string}}).error ?? {});
+    if (hasDiff(imageErrors) || hasNoRefImageErrors({assertViewResults: imageErrors})) {
+        return FAIL;
+    }
+
+    if (!isEmpty(testResult.error)) {
+        return ERROR;
+    }
+
+    return testResult.status;
+};
+
 export const isBase64Image = (image: ImageData | ImageBase64 | null | undefined): image is ImageBase64 => {
     return Boolean((image as ImageBase64 | undefined)?.base64);
 };

diff --git a/lib/constants/tests.ts b/lib/constants/tests.ts
@@ -1 +1,5 @@
+export const HERMIONE_TITLE_DELIMITER = ' ';
+
 export const PWT_TITLE_DELIMITER = ' › ';
+
+export const UNKNOWN_ATTEMPT = -1;
diff --git a/lib/db-utils/server.ts b/lib/db-utils/server.ts
@@ -14,7 +14,7 @@ import {DbLoadResult, HandleDatabasesOptions} from './common';
 import {DbUrlsJsonData, RawSuitesRow, ReporterConfig} from '../types';
 import {Tree} from '../tests-tree-builder/base';
 import {ReporterTestResult} from '../test-adapter';
-import {SqliteAdapter} from '../sqlite-adapter';
+import {SqliteClient} from '../sqlite-client';
 
 export * from './common';
 
@@ -121,8 +121,8 @@ async function rewriteDatabaseUrls(dbPaths: string[], mainDatabaseUrls: string,
     });
 }
 
-export const getTestFromDb = <T = unknown>(sqliteAdapter: SqliteAdapter, testResult: ReporterTestResult): T | undefined => {
-    return sqliteAdapter.query<T>({
+export const getTestFromDb = <T = unknown>(dbClient: SqliteClient, testResult: ReporterTestResult): T | undefined => {
+    return dbClient.query<T>({
         select: '*',
         where: `${DB_COLUMNS.SUITE_PATH} = ? AND ${DB_COLUMNS.NAME} = ? AND ${DB_COLUMNS.STATUS} = ?`,
         orderBy: DB_COLUMNS.TIMESTAMP,

diff --git a/lib/errors/db-not-initialized-error.ts b/lib/errors/db-not-initialized-error.ts