chore: manage packages installation by registry

gemini-testing · Dec 9, 2024 · 8c6d6c6 · 8c6d6c6
1 parent c7fa5aa
commit 8c6d6c6
Show file tree

Hide file tree

Showing 12 changed files with 392 additions and 235 deletions.
diff --git a/src/browser-installer/constants.ts b/src/browser-installer/constants.ts
@@ -10,7 +10,6 @@ export const MIN_CHROMIUM_VERSION = 73;
 export const MIN_FIREFOX_VERSION = 60;
 export const MIN_EDGEDRIVER_VERSION = 94;
 export const DRIVER_WAIT_TIMEOUT = 10 * 1000; // 10s
-export const BYTES_PER_KILOBYTE = 1 << 10; // eslint-disable-line no-bitwise
-export const BYTES_PER_MEGABYTE = BYTES_PER_KILOBYTE << 10; // eslint-disable-line no-bitwise
+export const LINUX_UBUNTU_RELEASE_ID = "ubuntu";
 export const LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME = "LD_LIBRARY_PATH";
 export const MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED = ["fontconfig"];
diff --git a/src/browser-installer/install.ts b/src/browser-installer/install.ts
@@ -1,5 +1,5 @@
 import _ from "lodash";
-import { Browser, getNormalizedBrowserName, type SupportedBrowser } from "./utils";
+import { Browser, browserInstallerDebug, getNormalizedBrowserName, type SupportedBrowser } from "./utils";
 
 /**
  * @returns path to installed browser binary
@@ -19,6 +19,15 @@ export const installBrowser = async (
 
     const needToInstallUbuntuPackages = shouldInstallUbuntuPackages && (await isUbuntu());
 
+    browserInstallerDebug(
+        [
+            `install ${browserName}@${browserVersion}`,
+            `shouldInstallWebDriver:${shouldInstallWebDriver}`,
+            `shouldInstallUbuntuPackages:${shouldInstallUbuntuPackages}`,
+            `needToInstallUbuntuPackages:${needToInstallUbuntuPackages}`,
+        ].join(", "),
+    );
+
     switch (browserName) {
         case Browser.CHROME:
         case Browser.CHROMIUM: {
@@ -83,6 +92,7 @@ const forceInstallBinaries = async (
     browserVersion?: string,
 ): ForceInstallBinaryResult => {
     const normalizedBrowserName = getNormalizedBrowserName(browserName);
+    const installOpts = { force: true, shouldInstallWebDriver: true, shouldInstallUbuntuPackages: true };
 
     if (!normalizedBrowserName) {
         return {
@@ -91,7 +101,7 @@ const forceInstallBinaries = async (
         };
     }
 
-    return installFn(normalizedBrowserName, browserVersion, { force: true, shouldInstallWebDriver: true })
+    return installFn(normalizedBrowserName, browserVersion, installOpts)
         .then(successResult => {
             return successResult
                 ? { status: BrowserInstallStatus.Ok }

diff --git a/src/browser-installer/registry/cli-progress-bar.ts b/src/browser-installer/registry/cli-progress-bar.ts
@@ -1,35 +1,39 @@
 import { MultiBar, type SingleBar } from "cli-progress";
 import type { DownloadProgressCallback } from "../utils";
-import { BYTES_PER_MEGABYTE } from "../constants";
 
 export type RegisterProgressBarFn = (browserName: string, browserVersion: string) => DownloadProgressCallback;
 
-export const createBrowserDownloadProgressBar = (): { register: RegisterProgressBarFn } => {
+export const createBrowserDownloadProgressBar = (): { register: RegisterProgressBarFn; stop: () => void } => {
     const progressBar = new MultiBar({
         stopOnComplete: true,
         forceRedraw: true,
         autopadding: true,
         hideCursor: true,
         fps: 5,
-        format: " [{bar}] | {filename} | {value}/{total} MB",
+        format: " [{bar}] | {filename} | {value}%",
     });
 
     const register: RegisterProgressBarFn = (browserName, browserVersion) => {
         let bar: SingleBar;
 
-        const downloadProgressCallback: DownloadProgressCallback = (downloadedBytes, totalBytes) => {
+        const downloadProgressCallback: DownloadProgressCallback = (done, total = 100) => {
             if (!bar) {
-                const totalMB = Math.round((totalBytes / BYTES_PER_MEGABYTE) * 100) / 100;
-                bar = progressBar.create(totalMB, 0, { filename: `${browserName}@${browserVersion}` });
+                bar = progressBar.create(100, 0, { filename: `${browserName}@${browserVersion}` });
             }
 
-            const downloadedMB = Math.round((downloadedBytes / BYTES_PER_MEGABYTE) * 100) / 100;
+            const downloadedPercents = Math.floor((done / total) * 100);
 
-            bar.update(downloadedMB);
+            bar.update(downloadedPercents);
         };
 
         return downloadProgressCallback;
     };
 
-    return { register };
+    const stop = (): void => {
+        progressBar.stop();
+    };
+
+    process.once("exit", stop);
+
+    return { register, stop };
 };
diff --git a/src/browser-installer/registry/index.ts b/src/browser-installer/registry/index.ts
@@ -1,8 +1,10 @@
 import type { BrowserPlatform } from "@puppeteer/browsers";
-import { readJSONSync, outputJSONSync, existsSync } from "fs-extra";
+import _ from "lodash";
+import { outputJSONSync } from "fs-extra";
 import path from "path";
 import {
     getRegistryPath,
+    readRegistry,
     browserInstallerDebug,
     Driver,
     Browser,
@@ -12,87 +14,123 @@ import {
     type SupportedBrowser,
     type SupportedDriver,
     type DownloadProgressCallback,
+    type BinaryKey,
+    type BinaryName,
+    type OsName,
+    type OsVersion,
+    type OsPackagesKey,
 } from "../utils";
 import { getFirefoxBuildId } from "../firefox/utils";
 import logger from "../../utils/logger";
-import type { createBrowserDownloadProgressBar } from "./cli-progress-bar";
-
-type VersionToPathMap = Record<string, string | Promise<string>>;
-type BinaryName = Exclude<SupportedBrowser | SupportedDriver, SupportedBrowser & SupportedDriver>;
-type RegistryKey = `${BinaryName}_${BrowserPlatform}`;
-type Registry = Record<RegistryKey, VersionToPathMap>;
 
 const registryPath = getRegistryPath();
-const registry: Registry = existsSync(registryPath) ? readJSONSync(registryPath) : {};
+const registry = readRegistry(registryPath);
+
+const getRegistryBinaryKey = (name: BinaryName, platform: BrowserPlatform): BinaryKey => `${name}_${platform}`;
+const getRegistryOsPackagesKey = (name: OsName, version: OsVersion): OsPackagesKey => `${name}_${version}`;
 
-let cliProgressBar: ReturnType<typeof createBrowserDownloadProgressBar> | null = null;
-let warnedFirstTimeInstall = false;
+const saveRegistry = (): void => {
+    const replacer = (_: string, value: unknown): unknown | undefined => {
+        if ((value as Promise<unknown>).then) {
+            return;
+        }
 
-const getRegistryKey = (name: BinaryName, platform: BrowserPlatform): RegistryKey => `${name}_${platform}`;
+        return value;
+    };
+
+    outputJSONSync(registryPath, registry, { replacer });
+};
+
+const getCliProgressBar = _.once(async () => {
+    const { createBrowserDownloadProgressBar } = await import("./cli-progress-bar");
+
+    return createBrowserDownloadProgressBar();
+});
 
 export const getBinaryPath = async (name: BinaryName, platform: BrowserPlatform, version: string): Promise<string> => {
-    const registryKey = getRegistryKey(name, platform);
+    const registryKey = getRegistryBinaryKey(name, platform);
 
-    if (!registry[registryKey]) {
+    if (!registry.binaries[registryKey]) {
         throw new Error(`Binary '${name}' on '${platform}' is not installed`);
     }
 
-    if (!registry[registryKey][version]) {
+    if (!registry.binaries[registryKey][version]) {
         throw new Error(`Version '${version}' of driver '${name}' on '${platform}' is not installed`);
     }
 
-    const binaryRelativePath = await registry[registryKey][version];
+    const binaryRelativePath = await registry.binaries[registryKey][version];
 
     browserInstallerDebug(`resolved '${name}@${version}' on ${platform} to ${binaryRelativePath}`);
 
     return path.resolve(registryPath, binaryRelativePath);
 };
 
+export const getOsPackagesPath = async (name: OsName, version: OsVersion): Promise<string> => {
+    const registryKey = getRegistryOsPackagesKey(name, version);
+
+    if (!registry.osPackages[registryKey]) {
+        throw new Error(`Packages for ${name}@${version} are not installed`);
+    }
+
+    const osPackagesRelativePath = await registry.osPackages[registryKey];
+
+    browserInstallerDebug(`resolved os packages for '${name}@${version}' to ${osPackagesRelativePath}`);
+
+    return path.resolve(registryPath, osPackagesRelativePath);
+};
+
 const addBinaryToRegistry = (
     name: BinaryName,
     platform: BrowserPlatform,
     version: string,
     absoluteBinaryPath: string,
 ): void => {
-    const registryKey = getRegistryKey(name, platform);
+    const registryKey = getRegistryBinaryKey(name, platform);
     const relativePath = path.relative(registryPath, absoluteBinaryPath);
 
-    registry[registryKey] ||= {};
-    registry[registryKey][version] = relativePath;
+    registry.binaries[registryKey] ||= {};
+    registry.binaries[registryKey][version] = relativePath;
 
-    const replacer = (_: string, value: unknown): unknown | undefined => {
-        if ((value as Promise<unknown>).then) {
-            return;
-        }
+    browserInstallerDebug(`adding '${name}@${version}' on '${platform}' to registry at ${relativePath}`);
 
-        return value;
-    };
+    saveRegistry();
+};
 
-    browserInstallerDebug(`adding '${name}@${version}' on '${platform}' to registry at ${relativePath}`);
-    outputJSONSync(registryPath, registry, { replacer });
+const addOsPackageToRegistry = (name: OsName, version: OsVersion, absolutePackagesDirPath: string): void => {
+    const registryKey = getRegistryOsPackagesKey(name, version);
+    const relativePath = path.relative(registryPath, absolutePackagesDirPath);
+
+    registry.osPackages[registryKey] = relativePath;
+
+    browserInstallerDebug(`adding os packages for '${name}@${version}' to registry at ${relativePath}`);
+
+    saveRegistry();
 };
 
 const getBinaryVersions = (name: BinaryName, platform: BrowserPlatform): string[] => {
-    const registryKey = getRegistryKey(name, platform);
+    const registryKey = getRegistryBinaryKey(name, platform);
 
-    if (!registry[registryKey]) {
+    if (!registry.binaries[registryKey]) {
         return [];
     }
 
-    return Object.keys(registry[registryKey]);
+    return Object.keys(registry.binaries[registryKey]);
 };
 
 const hasBinaryVersion = (name: BinaryName, platform: BrowserPlatform, version: string): boolean =>
     getBinaryVersions(name, platform).includes(version);
 
+export const hasOsPackages = (name: OsName, version: OsVersion): boolean =>
+    Boolean(registry.osPackages[getRegistryOsPackagesKey(name, version)]);
+
 export const getMatchedDriverVersion = (
     driverName: SupportedDriver,
     platform: BrowserPlatform,
     browserVersion: string,
 ): string | null => {
-    const registryKey = getRegistryKey(driverName, platform);
+    const registryKey = getRegistryBinaryKey(driverName, platform);
 
-    if (!registry[registryKey]) {
+    if (!registry.binaries[registryKey]) {
         return null;
     }
 
@@ -109,7 +147,7 @@ export const getMatchedDriverVersion = (
     }
 
     if (driverName === Driver.GECKODRIVER) {
-        const buildIds = Object.keys(registry[registryKey]);
+        const buildIds = Object.keys(registry.binaries[registryKey]);
         const buildIdsSorted = buildIds.sort(semverVersionsComparator);
 
         return buildIdsSorted.length ? buildIdsSorted[buildIdsSorted.length - 1] : null;
@@ -123,9 +161,9 @@ export const getMatchedBrowserVersion = (
     platform: BrowserPlatform,
     browserVersion: string,
 ): string | null => {
-    const registryKey = getRegistryKey(browserName, platform);
+    const registryKey = getRegistryBinaryKey(browserName, platform);
 
-    if (!registry[registryKey]) {
+    if (!registry.binaries[registryKey]) {
         return null;
     }
 
@@ -170,46 +208,88 @@ export const getMatchedBrowserVersion = (
     return suitableBuildIdsSorted[suitableBuildIdsSorted.length - 1];
 };
 
+const logDownloadingOsPackagesWarningOnce = _.once((osName: string) => {
+    logger.warn(`Downloading extra ${osName} packages`);
+});
+
+const logDownloadingBrowsersWarningOnce = _.once(() => {
+    logger.warn("Downloading Testplane browsers");
+    logger.warn("Note: this is one-time action. It may take a while...");
+});
+
 export const installBinary = async (
     name: BinaryName,
     platform: BrowserPlatform,
     version: string,
     installFn: (downloadProgressCallback: DownloadProgressCallback) => Promise<string>,
 ): Promise<string> => {
-    const registryKey = getRegistryKey(name, platform);
+    const registryKey = getRegistryBinaryKey(name, platform);
 
     if (hasBinaryVersion(name, platform, version)) {
         return getBinaryPath(name, platform, version);
     }
 
     browserInstallerDebug(`installing '${name}@${version}' on '${platform}'`);
 
-    if (!cliProgressBar) {
-        const { createBrowserDownloadProgressBar } = await import("./cli-progress-bar");
+    const progressBar = await getCliProgressBar();
 
-        cliProgressBar = createBrowserDownloadProgressBar();
-    }
-
-    const originalDownloadProgressCallback = cliProgressBar.register(name, version);
+    const originalDownloadProgressCallback = progressBar.register(name, version);
     const downloadProgressCallback: DownloadProgressCallback = (...args) => {
-        if (!warnedFirstTimeInstall) {
-            logger.warn("Downloading Testplane browsers");
-            logger.warn("Note: this is one-time action. It may take a while...");
-
-            warnedFirstTimeInstall = true;
-        }
+        logDownloadingBrowsersWarningOnce();
 
         return originalDownloadProgressCallback(...args);
     };
 
-    const installPromise = installFn(downloadProgressCallback).then(executablePath => {
-        addBinaryToRegistry(name, platform, version, executablePath);
+    const installPromise = installFn(downloadProgressCallback)
+        .then(executablePath => {
+            addBinaryToRegistry(name, platform, version, executablePath);
+
+            return executablePath;
+        })
+        .catch(err => {
+            progressBar?.stop();
+
+            throw err;
+        });
+
+    registry.binaries[registryKey] ||= {};
+    registry.binaries[registryKey][version] = installPromise;
+
+    return installPromise;
+};
+
+export const installOsPackages = async (
+    osName: OsName,
+    version: OsVersion,
+    installFn: (downloadProgressCallback: DownloadProgressCallback) => Promise<string>,
+): Promise<string> => {
+    const registryKey = getRegistryOsPackagesKey(osName, version);
+
+    if (hasOsPackages(osName, version)) {
+        return getOsPackagesPath(osName, version);
+    }
+
+    browserInstallerDebug(`installing os packages for '${osName}@${version}'`);
+
+    logDownloadingOsPackagesWarningOnce(osName);
+
+    const progressBar = await getCliProgressBar();
+
+    const downloadProgressCallback = progressBar.register(`extra packages for ${osName}`, version);
+
+    const installPromise = installFn(downloadProgressCallback)
+        .then(packagesPath => {
+            addOsPackageToRegistry(osName, version, packagesPath);
+
+            return packagesPath;
+        })
+        .catch(err => {
+            progressBar.stop();
 
-        return executablePath;
-    });
+            throw err;
+        });
 
-    registry[registryKey] ||= {};
-    registry[registryKey][version] = installPromise;
+    registry.osPackages[registryKey] = installPromise;
 
     return installPromise;
 };