From 8c6d6c620c0421665f9f252d5144d1af6a70de9a Mon Sep 17 00:00:00 2001 From: Roman Kuznetsov Date: Tue, 10 Dec 2024 02:36:00 +0300 Subject: [PATCH] chore: manage packages installation by registry --- src/browser-installer/constants.ts | 3 +- src/browser-installer/install.ts | 14 +- .../registry/cli-progress-bar.ts | 22 ++- src/browser-installer/registry/index.ts | 186 +++++++++++++----- src/browser-installer/ubuntu-packages/apt.ts | 43 ++-- .../browser-downloader.ts | 10 +- .../ubuntu-packages/index.ts | 77 +++----- .../ubuntu-packages/utils.ts | 16 +- src/browser-installer/utils.ts | 50 ++++- test/src/browser-installer/install.ts | 16 ++ test/src/browser-installer/registry.ts | 134 +++++++------ .../ubuntu-packages/index.ts | 56 +++--- 12 files changed, 392 insertions(+), 235 deletions(-) diff --git a/src/browser-installer/constants.ts b/src/browser-installer/constants.ts index 3fd81ffa5..dc3d4b86c 100644 --- a/src/browser-installer/constants.ts +++ b/src/browser-installer/constants.ts @@ -10,7 +10,6 @@ export const MIN_CHROMIUM_VERSION = 73; export const MIN_FIREFOX_VERSION = 60; export const MIN_EDGEDRIVER_VERSION = 94; export const DRIVER_WAIT_TIMEOUT = 10 * 1000; // 10s -export const BYTES_PER_KILOBYTE = 1 << 10; // eslint-disable-line no-bitwise -export const BYTES_PER_MEGABYTE = BYTES_PER_KILOBYTE << 10; // eslint-disable-line no-bitwise +export const LINUX_UBUNTU_RELEASE_ID = "ubuntu"; export const LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME = "LD_LIBRARY_PATH"; export const MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED = ["fontconfig"]; diff --git a/src/browser-installer/install.ts b/src/browser-installer/install.ts index 0e1a97261..3a7bb85bf 100644 --- a/src/browser-installer/install.ts +++ b/src/browser-installer/install.ts @@ -1,5 +1,5 @@ import _ from "lodash"; -import { Browser, getNormalizedBrowserName, type SupportedBrowser } from "./utils"; +import { Browser, browserInstallerDebug, getNormalizedBrowserName, type SupportedBrowser } from "./utils"; /** * @returns path to installed browser binary @@ -19,6 +19,15 @@ export const installBrowser = async ( const needToInstallUbuntuPackages = shouldInstallUbuntuPackages && (await isUbuntu()); + browserInstallerDebug( + [ + `install ${browserName}@${browserVersion}`, + `shouldInstallWebDriver:${shouldInstallWebDriver}`, + `shouldInstallUbuntuPackages:${shouldInstallUbuntuPackages}`, + `needToInstallUbuntuPackages:${needToInstallUbuntuPackages}`, + ].join(", "), + ); + switch (browserName) { case Browser.CHROME: case Browser.CHROMIUM: { @@ -83,6 +92,7 @@ const forceInstallBinaries = async ( browserVersion?: string, ): ForceInstallBinaryResult => { const normalizedBrowserName = getNormalizedBrowserName(browserName); + const installOpts = { force: true, shouldInstallWebDriver: true, shouldInstallUbuntuPackages: true }; if (!normalizedBrowserName) { return { @@ -91,7 +101,7 @@ const forceInstallBinaries = async ( }; } - return installFn(normalizedBrowserName, browserVersion, { force: true, shouldInstallWebDriver: true }) + return installFn(normalizedBrowserName, browserVersion, installOpts) .then(successResult => { return successResult ? { status: BrowserInstallStatus.Ok } diff --git a/src/browser-installer/registry/cli-progress-bar.ts b/src/browser-installer/registry/cli-progress-bar.ts index 61a278d49..6657c0b33 100644 --- a/src/browser-installer/registry/cli-progress-bar.ts +++ b/src/browser-installer/registry/cli-progress-bar.ts @@ -1,35 +1,39 @@ import { MultiBar, type SingleBar } from "cli-progress"; import type { DownloadProgressCallback } from "../utils"; -import { BYTES_PER_MEGABYTE } from "../constants"; export type RegisterProgressBarFn = (browserName: string, browserVersion: string) => DownloadProgressCallback; -export const createBrowserDownloadProgressBar = (): { register: RegisterProgressBarFn } => { +export const createBrowserDownloadProgressBar = (): { register: RegisterProgressBarFn; stop: () => void } => { const progressBar = new MultiBar({ stopOnComplete: true, forceRedraw: true, autopadding: true, hideCursor: true, fps: 5, - format: " [{bar}] | {filename} | {value}/{total} MB", + format: " [{bar}] | {filename} | {value}%", }); const register: RegisterProgressBarFn = (browserName, browserVersion) => { let bar: SingleBar; - const downloadProgressCallback: DownloadProgressCallback = (downloadedBytes, totalBytes) => { + const downloadProgressCallback: DownloadProgressCallback = (done, total = 100) => { if (!bar) { - const totalMB = Math.round((totalBytes / BYTES_PER_MEGABYTE) * 100) / 100; - bar = progressBar.create(totalMB, 0, { filename: `${browserName}@${browserVersion}` }); + bar = progressBar.create(100, 0, { filename: `${browserName}@${browserVersion}` }); } - const downloadedMB = Math.round((downloadedBytes / BYTES_PER_MEGABYTE) * 100) / 100; + const downloadedPercents = Math.floor((done / total) * 100); - bar.update(downloadedMB); + bar.update(downloadedPercents); }; return downloadProgressCallback; }; - return { register }; + const stop = (): void => { + progressBar.stop(); + }; + + process.once("exit", stop); + + return { register, stop }; }; diff --git a/src/browser-installer/registry/index.ts b/src/browser-installer/registry/index.ts index bdd0122fc..d063987d8 100644 --- a/src/browser-installer/registry/index.ts +++ b/src/browser-installer/registry/index.ts @@ -1,8 +1,10 @@ import type { BrowserPlatform } from "@puppeteer/browsers"; -import { readJSONSync, outputJSONSync, existsSync } from "fs-extra"; +import _ from "lodash"; +import { outputJSONSync } from "fs-extra"; import path from "path"; import { getRegistryPath, + readRegistry, browserInstallerDebug, Driver, Browser, @@ -12,87 +14,123 @@ import { type SupportedBrowser, type SupportedDriver, type DownloadProgressCallback, + type BinaryKey, + type BinaryName, + type OsName, + type OsVersion, + type OsPackagesKey, } from "../utils"; import { getFirefoxBuildId } from "../firefox/utils"; import logger from "../../utils/logger"; -import type { createBrowserDownloadProgressBar } from "./cli-progress-bar"; - -type VersionToPathMap = Record>; -type BinaryName = Exclude; -type RegistryKey = `${BinaryName}_${BrowserPlatform}`; -type Registry = Record; const registryPath = getRegistryPath(); -const registry: Registry = existsSync(registryPath) ? readJSONSync(registryPath) : {}; +const registry = readRegistry(registryPath); + +const getRegistryBinaryKey = (name: BinaryName, platform: BrowserPlatform): BinaryKey => `${name}_${platform}`; +const getRegistryOsPackagesKey = (name: OsName, version: OsVersion): OsPackagesKey => `${name}_${version}`; -let cliProgressBar: ReturnType | null = null; -let warnedFirstTimeInstall = false; +const saveRegistry = (): void => { + const replacer = (_: string, value: unknown): unknown | undefined => { + if ((value as Promise).then) { + return; + } -const getRegistryKey = (name: BinaryName, platform: BrowserPlatform): RegistryKey => `${name}_${platform}`; + return value; + }; + + outputJSONSync(registryPath, registry, { replacer }); +}; + +const getCliProgressBar = _.once(async () => { + const { createBrowserDownloadProgressBar } = await import("./cli-progress-bar"); + + return createBrowserDownloadProgressBar(); +}); export const getBinaryPath = async (name: BinaryName, platform: BrowserPlatform, version: string): Promise => { - const registryKey = getRegistryKey(name, platform); + const registryKey = getRegistryBinaryKey(name, platform); - if (!registry[registryKey]) { + if (!registry.binaries[registryKey]) { throw new Error(`Binary '${name}' on '${platform}' is not installed`); } - if (!registry[registryKey][version]) { + if (!registry.binaries[registryKey][version]) { throw new Error(`Version '${version}' of driver '${name}' on '${platform}' is not installed`); } - const binaryRelativePath = await registry[registryKey][version]; + const binaryRelativePath = await registry.binaries[registryKey][version]; browserInstallerDebug(`resolved '${name}@${version}' on ${platform} to ${binaryRelativePath}`); return path.resolve(registryPath, binaryRelativePath); }; +export const getOsPackagesPath = async (name: OsName, version: OsVersion): Promise => { + const registryKey = getRegistryOsPackagesKey(name, version); + + if (!registry.osPackages[registryKey]) { + throw new Error(`Packages for ${name}@${version} are not installed`); + } + + const osPackagesRelativePath = await registry.osPackages[registryKey]; + + browserInstallerDebug(`resolved os packages for '${name}@${version}' to ${osPackagesRelativePath}`); + + return path.resolve(registryPath, osPackagesRelativePath); +}; + const addBinaryToRegistry = ( name: BinaryName, platform: BrowserPlatform, version: string, absoluteBinaryPath: string, ): void => { - const registryKey = getRegistryKey(name, platform); + const registryKey = getRegistryBinaryKey(name, platform); const relativePath = path.relative(registryPath, absoluteBinaryPath); - registry[registryKey] ||= {}; - registry[registryKey][version] = relativePath; + registry.binaries[registryKey] ||= {}; + registry.binaries[registryKey][version] = relativePath; - const replacer = (_: string, value: unknown): unknown | undefined => { - if ((value as Promise).then) { - return; - } + browserInstallerDebug(`adding '${name}@${version}' on '${platform}' to registry at ${relativePath}`); - return value; - }; + saveRegistry(); +}; - browserInstallerDebug(`adding '${name}@${version}' on '${platform}' to registry at ${relativePath}`); - outputJSONSync(registryPath, registry, { replacer }); +const addOsPackageToRegistry = (name: OsName, version: OsVersion, absolutePackagesDirPath: string): void => { + const registryKey = getRegistryOsPackagesKey(name, version); + const relativePath = path.relative(registryPath, absolutePackagesDirPath); + + registry.osPackages[registryKey] = relativePath; + + browserInstallerDebug(`adding os packages for '${name}@${version}' to registry at ${relativePath}`); + + saveRegistry(); }; const getBinaryVersions = (name: BinaryName, platform: BrowserPlatform): string[] => { - const registryKey = getRegistryKey(name, platform); + const registryKey = getRegistryBinaryKey(name, platform); - if (!registry[registryKey]) { + if (!registry.binaries[registryKey]) { return []; } - return Object.keys(registry[registryKey]); + return Object.keys(registry.binaries[registryKey]); }; const hasBinaryVersion = (name: BinaryName, platform: BrowserPlatform, version: string): boolean => getBinaryVersions(name, platform).includes(version); +export const hasOsPackages = (name: OsName, version: OsVersion): boolean => + Boolean(registry.osPackages[getRegistryOsPackagesKey(name, version)]); + export const getMatchedDriverVersion = ( driverName: SupportedDriver, platform: BrowserPlatform, browserVersion: string, ): string | null => { - const registryKey = getRegistryKey(driverName, platform); + const registryKey = getRegistryBinaryKey(driverName, platform); - if (!registry[registryKey]) { + if (!registry.binaries[registryKey]) { return null; } @@ -109,7 +147,7 @@ export const getMatchedDriverVersion = ( } if (driverName === Driver.GECKODRIVER) { - const buildIds = Object.keys(registry[registryKey]); + const buildIds = Object.keys(registry.binaries[registryKey]); const buildIdsSorted = buildIds.sort(semverVersionsComparator); return buildIdsSorted.length ? buildIdsSorted[buildIdsSorted.length - 1] : null; @@ -123,9 +161,9 @@ export const getMatchedBrowserVersion = ( platform: BrowserPlatform, browserVersion: string, ): string | null => { - const registryKey = getRegistryKey(browserName, platform); + const registryKey = getRegistryBinaryKey(browserName, platform); - if (!registry[registryKey]) { + if (!registry.binaries[registryKey]) { return null; } @@ -170,13 +208,22 @@ export const getMatchedBrowserVersion = ( return suitableBuildIdsSorted[suitableBuildIdsSorted.length - 1]; }; +const logDownloadingOsPackagesWarningOnce = _.once((osName: string) => { + logger.warn(`Downloading extra ${osName} packages`); +}); + +const logDownloadingBrowsersWarningOnce = _.once(() => { + logger.warn("Downloading Testplane browsers"); + logger.warn("Note: this is one-time action. It may take a while..."); +}); + export const installBinary = async ( name: BinaryName, platform: BrowserPlatform, version: string, installFn: (downloadProgressCallback: DownloadProgressCallback) => Promise, ): Promise => { - const registryKey = getRegistryKey(name, platform); + const registryKey = getRegistryBinaryKey(name, platform); if (hasBinaryVersion(name, platform, version)) { return getBinaryPath(name, platform, version); @@ -184,32 +231,65 @@ export const installBinary = async ( browserInstallerDebug(`installing '${name}@${version}' on '${platform}'`); - if (!cliProgressBar) { - const { createBrowserDownloadProgressBar } = await import("./cli-progress-bar"); + const progressBar = await getCliProgressBar(); - cliProgressBar = createBrowserDownloadProgressBar(); - } - - const originalDownloadProgressCallback = cliProgressBar.register(name, version); + const originalDownloadProgressCallback = progressBar.register(name, version); const downloadProgressCallback: DownloadProgressCallback = (...args) => { - if (!warnedFirstTimeInstall) { - logger.warn("Downloading Testplane browsers"); - logger.warn("Note: this is one-time action. It may take a while..."); - - warnedFirstTimeInstall = true; - } + logDownloadingBrowsersWarningOnce(); return originalDownloadProgressCallback(...args); }; - const installPromise = installFn(downloadProgressCallback).then(executablePath => { - addBinaryToRegistry(name, platform, version, executablePath); + const installPromise = installFn(downloadProgressCallback) + .then(executablePath => { + addBinaryToRegistry(name, platform, version, executablePath); + + return executablePath; + }) + .catch(err => { + progressBar?.stop(); + + throw err; + }); + + registry.binaries[registryKey] ||= {}; + registry.binaries[registryKey][version] = installPromise; + + return installPromise; +}; + +export const installOsPackages = async ( + osName: OsName, + version: OsVersion, + installFn: (downloadProgressCallback: DownloadProgressCallback) => Promise, +): Promise => { + const registryKey = getRegistryOsPackagesKey(osName, version); + + if (hasOsPackages(osName, version)) { + return getOsPackagesPath(osName, version); + } + + browserInstallerDebug(`installing os packages for '${osName}@${version}'`); + + logDownloadingOsPackagesWarningOnce(osName); + + const progressBar = await getCliProgressBar(); + + const downloadProgressCallback = progressBar.register(`extra packages for ${osName}`, version); + + const installPromise = installFn(downloadProgressCallback) + .then(packagesPath => { + addOsPackageToRegistry(osName, version, packagesPath); + + return packagesPath; + }) + .catch(err => { + progressBar.stop(); - return executablePath; - }); + throw err; + }); - registry[registryKey] ||= {}; - registry[registryKey][version] = installPromise; + registry.osPackages[registryKey] = installPromise; return installPromise; }; diff --git a/src/browser-installer/ubuntu-packages/apt.ts b/src/browser-installer/ubuntu-packages/apt.ts index 3be0085e9..64c88239c 100644 --- a/src/browser-installer/ubuntu-packages/apt.ts +++ b/src/browser-installer/ubuntu-packages/apt.ts @@ -4,7 +4,7 @@ import path from "path"; import fs from "fs-extra"; import { exec } from "child_process"; import { ensureUnixBinaryExists } from "./utils"; -import { browserInstallerDebug } from "../utils"; +import { browserInstallerDebug, type DownloadProgressCallback } from "../utils"; import { MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED } from "../constants"; /** @link https://manpages.org/apt-cache/8 */ @@ -96,7 +96,7 @@ const downloadUbuntuPackages = async (dependencies: string[], targetDir: string) /** @link https://manpages.org/dpkg */ const unpackUbuntuPackages = async (packagesDir: string, destination: string): Promise => { - await ensureUnixBinaryExists("dpkg"); + await Promise.all([ensureUnixBinaryExists("dpkg"), fs.ensureDir(destination)]); return new Promise((resolve, reject) => { exec(`for pkg in *.deb; do dpkg -x $pkg ${destination}; done`, { cwd: packagesDir }, err => { @@ -109,43 +109,54 @@ const unpackUbuntuPackages = async (packagesDir: string, destination: string): P }); }; -export const installUbuntuPackages = async (packages: string[], destination: string): Promise => { +export const installUbuntuPackages = async ( + packages: string[], + destination: string, + { downloadProgressCallback }: { downloadProgressCallback: DownloadProgressCallback }, +): Promise => { if (!packages) { browserInstallerDebug(`There are no ubuntu packages to install`); - return fs.ensureDir(destination); + return; } const withRecursiveDependencies = await resolveTransitiveDependencies(packages); + downloadProgressCallback(40); + browserInstallerDebug(`Resolved direct packages to ${withRecursiveDependencies.length} dependencies`); const dependenciesToDownload = await filterNotExistingDependencies(withRecursiveDependencies); + downloadProgressCallback(70); + + const missingPkgs = MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED.filter(pkg => dependenciesToDownload.includes(pkg)); + + if (missingPkgs.length) { + throw new Error( + [ + "Missing some packages, which needs to be installed manually", + `Use \`apt-get install ${missingPkgs.join(" ")}\` to install them`, + `Then run "testplane install-deps" again\n`, + ].join("\n"), + ); + } + browserInstallerDebug(`There are ${dependenciesToDownload.length} deb packages to download`); if (!dependenciesToDownload.length) { - return fs.ensureDir(destination); + return; } const tmpPackagesDir = await fs.mkdtemp(path.join(os.tmpdir(), "testplane-ubuntu-apt-packages")); await downloadUbuntuPackages(dependenciesToDownload, tmpPackagesDir); + downloadProgressCallback(100); + browserInstallerDebug(`Downloaded ${dependenciesToDownload.length} deb packages`); await unpackUbuntuPackages(tmpPackagesDir, destination); browserInstallerDebug(`Unpacked ${dependenciesToDownload.length} deb packages`); - - const missingPkgs = MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED.filter(pkg => dependenciesToDownload.includes(pkg)); - - if (missingPkgs.length) { - throw new Error( - [ - "Missing some packages, which needs to be installed manually", - `Use \`apt-get install ${missingPkgs.join(" ")}\` to install them`, - ].join("\n"), - ); - } }; diff --git a/src/browser-installer/ubuntu-packages/collect-dependencies/browser-downloader.ts b/src/browser-installer/ubuntu-packages/collect-dependencies/browser-downloader.ts index 2d98c9e81..529722ca2 100644 --- a/src/browser-installer/ubuntu-packages/collect-dependencies/browser-downloader.ts +++ b/src/browser-installer/ubuntu-packages/collect-dependencies/browser-downloader.ts @@ -2,17 +2,11 @@ import path from "path"; import fs from "fs"; import _ from "lodash"; import { installBrowser } from "../.."; -import { getRegistryPath } from "../../utils"; +import { getRegistryPath, type Registry } from "../../utils"; import type { BrowserWithVersion } from "./utils"; -type BinaryNameWithArchPrefix = string; -type BinaryVersion = string; -type BinaryPath = string; - -type Registry = Record>; - const getRegistryBinaryPaths = (registry: Registry): string[] => { - const versionToPathMap = Object.values(registry); + const versionToPathMap = Object.values(registry.binaries); const binaryPaths = _.flatMap(versionToPathMap, Object.values); const registryPath = getRegistryPath(); diff --git a/src/browser-installer/ubuntu-packages/index.ts b/src/browser-installer/ubuntu-packages/index.ts index c2c0d1bd5..f0c4a9b1e 100644 --- a/src/browser-installer/ubuntu-packages/index.ts +++ b/src/browser-installer/ubuntu-packages/index.ts @@ -1,11 +1,12 @@ import _ from "lodash"; import fs from "fs-extra"; import path from "path"; -import { browserInstallerDebug, getUbuntuPackagesDir } from "../utils"; +import { getOsPackagesDir, type DownloadProgressCallback, browserInstallerDebug } from "../utils"; import { installUbuntuPackages } from "./apt"; import { getUbuntuMilestone } from "./utils"; import logger from "../../utils/logger"; -import { LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME } from "../constants"; +import { LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME, LINUX_UBUNTU_RELEASE_ID } from "../constants"; +import { getOsPackagesPath, hasOsPackages, installOsPackages } from "../registry"; export { isUbuntu, getUbuntuMilestone, ensureUnixBinaryExists } from "./utils"; @@ -35,32 +36,27 @@ export const writeUbuntuPackageDependencies = async (ubuntuMilestone: string, de await fs.outputJSON(getDependenciesArrayFilePath(ubuntuMilestone), packagesToWrite, { spaces: 4 }); }; -let installUbuntuPackageDependenciesPromise: Promise; +export const installUbuntuPackageDependencies = async (): Promise => { + const ubuntuMilestone = await getUbuntuMilestone(); -export const installUbuntuPackageDependencies = async (): Promise => { - if (installUbuntuPackageDependenciesPromise) { - return installUbuntuPackageDependenciesPromise; - } + browserInstallerDebug(`installing ubuntu${ubuntuMilestone} dependencies`); + + if (hasOsPackages(LINUX_UBUNTU_RELEASE_ID, ubuntuMilestone)) { + browserInstallerDebug(`installing ubuntu${ubuntuMilestone} dependencies`); - installUbuntuPackageDependenciesPromise = new Promise((resolve, reject) => { - const ubuntuPackagesDir = getUbuntuPackagesDir(); + return getOsPackagesPath(LINUX_UBUNTU_RELEASE_ID, ubuntuMilestone); + } - if (fs.existsSync(ubuntuPackagesDir)) { - browserInstallerDebug("Skip installing ubuntu packages, as they are installed already"); + const downloadFn = async (downloadProgressCallback: DownloadProgressCallback): Promise => { + const ubuntuPackageDependencies = await readUbuntuPackageDependencies(ubuntuMilestone); + const ubuntuPackagesDir = getOsPackagesDir(LINUX_UBUNTU_RELEASE_ID, ubuntuMilestone); - resolve(); - } else { - logger.log("Downloading extra deb packages to local browsers execution..."); + await installUbuntuPackages(ubuntuPackageDependencies, ubuntuPackagesDir, { downloadProgressCallback }); - getUbuntuMilestone() - .then(ubuntuMilestone => readUbuntuPackageDependencies(ubuntuMilestone)) - .then(dependencies => installUbuntuPackages(dependencies, ubuntuPackagesDir)) - .then(resolve) - .catch(reject); - } - }); + return ubuntuPackagesDir; + }; - return installUbuntuPackageDependenciesPromise; + return installOsPackages(LINUX_UBUNTU_RELEASE_ID, ubuntuMilestone, downloadFn); }; const listDirsAbsolutePath = async (dirBasePath: string, ...prefix: string[]): Promise => { @@ -88,34 +84,25 @@ const listDirsAbsolutePath = async (dirBasePath: string, ...prefix: string[]): P return directories; }; -let getUbuntuLinkerEnvPromise: Promise>; +const getUbuntuLinkerEnvRaw = async (): Promise> => { + const ubuntuMilestone = await getUbuntuMilestone(); -export const getUbuntuLinkerEnv = async (): Promise> => { - if (getUbuntuLinkerEnvPromise) { - return getUbuntuLinkerEnvPromise; + if (!hasOsPackages(LINUX_UBUNTU_RELEASE_ID, ubuntuMilestone)) { + return {}; } - getUbuntuLinkerEnvPromise = new Promise>((resolve, reject) => { - const ubuntuPackagesDir = getUbuntuPackagesDir(); + const ubuntuPackagesDir = await getOsPackagesPath(LINUX_UBUNTU_RELEASE_ID, ubuntuMilestone); - if (!fs.existsSync(ubuntuPackagesDir)) { - return resolve({}); - } + const currentRuntimeLibrariesEnvValue = process.env[LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME]; - const currentRuntimeLibrariesEnvValue = process.env[LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME]; + const [libDirs, usrLibDirs] = await Promise.all([ + listDirsAbsolutePath(ubuntuPackagesDir, "lib"), + listDirsAbsolutePath(ubuntuPackagesDir, "usr", "lib"), + ]); - Promise.all([ - listDirsAbsolutePath(ubuntuPackagesDir, "lib"), - listDirsAbsolutePath(ubuntuPackagesDir, "usr", "lib"), - ]) - .then(([libDirs, usrLibDirs]) => { - const libraryPaths = [...libDirs, ...usrLibDirs, currentRuntimeLibrariesEnvValue].filter(Boolean); + const libraryPaths = [...libDirs, ...usrLibDirs, currentRuntimeLibrariesEnvValue].filter(Boolean); - return { [LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME]: libraryPaths.join(":") }; - }) - .then(resolve) - .catch(reject); - }); - - return getUbuntuLinkerEnvPromise; + return { [LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME]: libraryPaths.join(":") }; }; + +export const getUbuntuLinkerEnv = _.once(getUbuntuLinkerEnvRaw); diff --git a/src/browser-installer/ubuntu-packages/utils.ts b/src/browser-installer/ubuntu-packages/utils.ts index 70381a439..a236c93d7 100644 --- a/src/browser-installer/ubuntu-packages/utils.ts +++ b/src/browser-installer/ubuntu-packages/utils.ts @@ -1,6 +1,8 @@ +import _ from "lodash"; import { exec } from "child_process"; import fs from "fs"; import { browserInstallerDebug } from "../utils"; +import { LINUX_UBUNTU_RELEASE_ID } from "../constants"; /** @link https://manpages.org/os-release/5 */ const OS_RELEASE_PATH = "/etc/os-release"; @@ -61,22 +63,16 @@ const osRelease = async (): Promise => { return result; }; -let isUbuntuCached: boolean | null = null; +const osReleaseCached = _.once(osRelease); export const isUbuntu = async (): Promise => { - if (isUbuntuCached !== null) { - return isUbuntuCached; - } - - isUbuntuCached = await osRelease() - .then(release => release.ID === "ubuntu") - .catch(() => false); + const osReleaseContents = await osReleaseCached(); - return isUbuntuCached; + return osReleaseContents.ID === LINUX_UBUNTU_RELEASE_ID; }; export const getUbuntuMilestone = async (): Promise => { - const release = await osRelease(); + const release = await osReleaseCached(); if (!release.VERSION_ID) { throw new Error(`VERSION_ID is missing in ${OS_RELEASE_PATH}. Probably its not Ubuntu`); diff --git a/src/browser-installer/utils.ts b/src/browser-installer/utils.ts index e6a0a20d9..d6629c9a4 100644 --- a/src/browser-installer/utils.ts +++ b/src/browser-installer/utils.ts @@ -1,13 +1,14 @@ import { detectBrowserPlatform, BrowserPlatform, Browser as PuppeteerBrowser } from "@puppeteer/browsers"; import extractZip from "extract-zip"; +import _ from "lodash"; import os from "os"; import path from "path"; -import { createWriteStream } from "fs"; +import fs from "fs-extra"; import { Readable } from "stream"; import debug from "debug"; import { MIN_CHROMIUM_MAC_ARM_VERSION } from "./constants"; -export type DownloadProgressCallback = (downloadedBytes: number, totalBytes: number) => void; +export type DownloadProgressCallback = (done: number, total?: number) => void; export const browserInstallerDebug = debug("testplane:browser-installer"); @@ -29,6 +30,18 @@ export const Driver = { export type SupportedBrowser = (typeof Browser)[keyof typeof Browser]; export type SupportedDriver = (typeof Driver)[keyof typeof Driver]; +export type VersionToPathMap = Record>; +export type BinaryName = Exclude; +export type BinaryKey = `${BinaryName}_${BrowserPlatform}`; +export type OsName = string; +export type OsVersion = string; +export type OsPackagesKey = `${OsName}_${OsVersion}`; +export type Registry = { + binaries: Record; + osPackages: Record>; + meta: { version: number }; +}; + export const getNormalizedBrowserName = ( browserName?: string, ): Exclude | null => { @@ -129,13 +142,42 @@ const getCacheDir = (envValueOverride = process.env.TESTPLANE_BROWSERS_PATH): st export const getRegistryPath = (envValueOverride?: string): string => path.join(getCacheDir(envValueOverride), "registry.json"); +export const readRegistry = (registryPath: string): Registry => { + const registry: Registry = { + binaries: {} as Record, + osPackages: {} as Record, + meta: { version: 1 }, + }; + + let fsData: Record; + + if (fs.existsSync(registryPath)) { + fsData = fs.readJSONSync(registryPath); + + const isRegistryV0 = fsData && !fsData.meta; + const isRegistryWithVersion = typeof _.get(fsData, "meta.version") === "number"; + + if (isRegistryWithVersion) { + return fsData as Registry; + } + + if (isRegistryV0) { + registry.binaries = fsData as Record; + } + } + + return registry; +}; + export const getBrowsersDir = (): string => path.join(getCacheDir(), "browsers"); -export const getUbuntuPackagesDir = (): string => path.join(getCacheDir(), "packages"); const getDriversDir = (): string => path.join(getCacheDir(), "drivers"); const getDriverDir = (driverName: string, driverVersion: string): string => path.join(getDriversDir(), driverName, driverVersion); +export const getOsPackagesDir = (osName: OsName, osVersion: OsVersion): string => + path.join(getCacheDir(), "packages", osName, osVersion); + export const getGeckoDriverDir = (driverVersion: string): string => getDriverDir("geckodriver", getBrowserPlatform() + "-" + driverVersion); export const getEdgeDriverDir = (driverVersion: string): string => @@ -168,7 +210,7 @@ export const retryFetch = async ( }; export const downloadFile = async (url: string, filePath: string): Promise => { - const writeStream = createWriteStream(filePath); + const writeStream = fs.createWriteStream(filePath); const response = await fetch(url); if (!response.ok || !response.body) { diff --git a/test/src/browser-installer/install.ts b/test/src/browser-installer/install.ts index 9dd3177bc..2778e5d88 100644 --- a/test/src/browser-installer/install.ts +++ b/test/src/browser-installer/install.ts @@ -174,6 +174,22 @@ describe("browser-installer/install", () => { assert.calledOnceWith(installChromeDriverStub, "115", { force: true }); }); + it("should install ubuntu packages on ubuntu", async () => { + isUbuntuStub.resolves(true); + + await installBrowsersWithDrivers([{ browserName: "chrome", browserVersion: "115" }]); + + assert.calledOnceWith(installUbuntuPackageDependenciesStub); + }); + + it("should not install ubuntu packages if its not ubuntu", async () => { + isUbuntuStub.resolves(false); + + await installBrowsersWithDrivers([{ browserName: "chrome", browserVersion: "115" }]); + + assert.notCalled(installUbuntuPackageDependenciesStub); + }); + it("should return result with browsers install status", async () => { installChromeStub.rejects(new Error("test chrome install error")); installFirefoxStub.resolves("/browser/path"); diff --git a/test/src/browser-installer/registry.ts b/test/src/browser-installer/registry.ts index 1e6d2ada3..6cff01d6d 100644 --- a/test/src/browser-installer/registry.ts +++ b/test/src/browser-installer/registry.ts @@ -1,44 +1,42 @@ import proxyquire from "proxyquire"; import sinon, { type SinonStub } from "sinon"; -import type * as Registry from "../../../src/browser-installer/registry"; -import { Browser, Driver, type DownloadProgressCallback } from "../../../src/browser-installer/utils"; +import type * as RegistryType from "../../../src/browser-installer/registry"; +import { Browser, Driver, type DownloadProgressCallback, type Registry } from "../../../src/browser-installer/utils"; import { BrowserPlatform } from "@puppeteer/browsers"; +import type { PartialDeep } from "type-fest"; describe("browser-installer/registry", () => { const sandbox = sinon.createSandbox(); - let registry: typeof Registry; + let registry: typeof RegistryType; - let readJSONSyncStub: SinonStub; + let readRegistryStub: SinonStub; let outputJSONSyncStub: SinonStub; - let existsSyncStub: SinonStub; let progressBarRegisterStub: SinonStub; let loggerWarnStub: SinonStub; - const createRegistry_ = (contents: Record> = {}): typeof Registry => { + const createRegistry_ = (contents: PartialDeep = {} as Registry): typeof RegistryType => { + contents.binaries ||= {}; + contents.osPackages ||= {}; + contents.meta ||= { version: 1 }; + return proxyquire("../../../src/browser-installer/registry", { - "../utils": { getRegistryPath: () => "/testplane/registry/registry.json" }, - "fs-extra": { readJSONSync: () => contents, existsSync: () => true }, + "../utils": { getRegistryPath: () => "/testplane/registry/registry.json", readRegistry: () => contents }, "../../utils/logger": { warn: loggerWarnStub }, }); }; beforeEach(() => { - readJSONSyncStub = sandbox.stub().returns({}); + readRegistryStub = sandbox.stub().returns({ binaries: {}, osPackages: {}, meta: { version: 1 } }); outputJSONSyncStub = sandbox.stub(); - existsSyncStub = sandbox.stub().returns(false); progressBarRegisterStub = sandbox.stub(); loggerWarnStub = sandbox.stub(); registry = proxyquire("../../../src/browser-installer/registry", { "./cli-progress-bar": { createBrowserDownloadProgressBar: () => ({ register: progressBarRegisterStub }) }, - "../utils": { getRegistryPath: () => "/testplane/registry/registry.json" }, + "../utils": { getRegistryPath: () => "/testplane/registry/registry.json", readRegistry: readRegistryStub }, "../../utils/logger": { warn: loggerWarnStub }, - "fs-extra": { - readJSONSync: readJSONSyncStub, - outputJSONSync: outputJSONSyncStub, - existsSync: existsSyncStub, - }, + "fs-extra": { outputJSONSync: outputJSONSyncStub }, }); }); @@ -47,9 +45,11 @@ describe("browser-installer/registry", () => { describe("getBinaryPath", () => { it("should return binary path", async () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - chrome_mac_arm: { - "115.0.5790.170": "../browsers/chrome", + binaries: { + // eslint-disable-next-line camelcase + chrome_mac_arm: { + "115.0.5790.170": "../browsers/chrome", + }, }, }); @@ -68,7 +68,7 @@ describe("browser-installer/registry", () => { it("should throw an error if browser version is not installed", async () => { // eslint-disable-next-line camelcase - registry = createRegistry_({ chrome_mac_arm: {} }); + registry = createRegistry_({ binaries: { chrome_mac_arm: {} } }); const fn = (): Promise => registry.getBinaryPath(Browser.CHROME, BrowserPlatform.MAC_ARM, "120"); @@ -79,11 +79,13 @@ describe("browser-installer/registry", () => { describe("getMatchedBrowserVersion", () => { it("should return matching latest chrome browser version", () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - chrome_mac_arm: { - "115.0.5790.170": "../browsers/chrome-115-0-5790-170", - "114.0.6980.170": "../browsers/chrome-114-0-6980-170", - "115.0.5320.180": "../browsers/chrome-115-0-5230-180", + binaries: { + // eslint-disable-next-line camelcase + chrome_mac_arm: { + "115.0.5790.170": "../browsers/chrome-115-0-5790-170", + "114.0.6980.170": "../browsers/chrome-114-0-6980-170", + "115.0.5320.180": "../browsers/chrome-115-0-5230-180", + }, }, }); @@ -96,11 +98,13 @@ describe("browser-installer/registry", () => { it("should return matching latest firefox browser version", () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - firefox_mac_arm: { - "stable_117.0b2": "../browsers/chrome-117-0b2", - "stable_118.0": "../browsers/firefox-118-0", - "stable_117.0b9": "../browsers/firefox-117-0b9", + binaries: { + // eslint-disable-next-line camelcase + firefox_mac_arm: { + "stable_117.0b2": "../browsers/chrome-117-0b2", + "stable_118.0": "../browsers/firefox-118-0", + "stable_117.0b9": "../browsers/firefox-117-0b9", + }, }, }); @@ -113,11 +117,13 @@ describe("browser-installer/registry", () => { it("should return null if no installed browser matching requirements", () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - chrome_mac_arm: { - "115.0.5790.170": "../browsers/chrome-115-0-5790-170", - "114.0.6980.170": "../browsers/chrome-114-0-6980-170", - "115.0.5320.180": "../browsers/chrome-115-0-5230-180", + binaries: { + // eslint-disable-next-line camelcase + chrome_mac_arm: { + "115.0.5790.170": "../browsers/chrome-115-0-5790-170", + "114.0.6980.170": "../browsers/chrome-114-0-6980-170", + "115.0.5320.180": "../browsers/chrome-115-0-5230-180", + }, }, }); @@ -132,11 +138,13 @@ describe("browser-installer/registry", () => { describe("getMatchedDriverVersion", () => { it("should return matching chromedriver version", () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - chromedriver_mac_arm: { - "115.0.5790.170": "../drivers/chromedriver-115-0-5790-170", - "114.0.6980.170": "../drivers/chromedriver-114-0-6980-170", - "115.0.5320.180": "../drivers/chromedriver-115-0-5230-180", + binaries: { + // eslint-disable-next-line camelcase + chromedriver_mac_arm: { + "115.0.5790.170": "../drivers/chromedriver-115-0-5790-170", + "114.0.6980.170": "../drivers/chromedriver-114-0-6980-170", + "115.0.5320.180": "../drivers/chromedriver-115-0-5230-180", + }, }, }); @@ -149,11 +157,13 @@ describe("browser-installer/registry", () => { it("should return matching chromedriver version", () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - edgedriver_mac_arm: { - "115.0.5790.170": "../drivers/edgedriver-115-0-5790-170", - "114.0.6980.170": "../drivers/edgedriver-114-0-6980-170", - "115.0.5320.180": "../drivers/edgedriver-115-0-5230-180", + binaries: { + // eslint-disable-next-line camelcase + edgedriver_mac_arm: { + "115.0.5790.170": "../drivers/edgedriver-115-0-5790-170", + "114.0.6980.170": "../drivers/edgedriver-114-0-6980-170", + "115.0.5320.180": "../drivers/edgedriver-115-0-5230-180", + }, }, }); @@ -166,11 +176,13 @@ describe("browser-installer/registry", () => { it("should return latest version for geckodriver", () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - geckodriver_mac_arm: { - "0.33.0": "../drivers/geckodriver-33", - "0.35.0": "../drivers/geckodriver-35", - "0.34.0": "../drivers/geckodriver-34", + binaries: { + // eslint-disable-next-line camelcase + geckodriver_mac_arm: { + "0.33.0": "../drivers/geckodriver-33", + "0.35.0": "../drivers/geckodriver-35", + "0.34.0": "../drivers/geckodriver-34", + }, }, }); @@ -183,8 +195,10 @@ describe("browser-installer/registry", () => { it("should return null if matching version is not found", () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - chromedriver_mac_arm: {}, + binaries: { + // eslint-disable-next-line camelcase + chromedriver_mac_arm: {}, + }, }); const version = registry.getMatchedDriverVersion(Driver.GECKODRIVER, BrowserPlatform.MAC_ARM, "115"); @@ -206,9 +220,11 @@ describe("browser-installer/registry", () => { it("should not install binary if it is already installed", async () => { registry = createRegistry_({ - // eslint-disable-next-line camelcase - chrome_mac_arm: { - "115.0.5320.180": "../browser/path", + binaries: { + // eslint-disable-next-line camelcase + chrome_mac_arm: { + "115.0.5320.180": "../browser/path", + }, }, }); @@ -235,8 +251,12 @@ describe("browser-installer/registry", () => { outputJSONSyncStub, "/testplane/registry/registry.json", { - // eslint-disable-next-line camelcase - chrome_mac_arm: { "115.0.5320.180": "../browser/path" }, + binaries: { + // eslint-disable-next-line camelcase + chrome_mac_arm: { "115.0.5320.180": "../browser/path" }, + }, + osPackages: {}, + meta: { version: 1 }, }, { replacer: sinon.match.func }, ); diff --git a/test/src/browser-installer/ubuntu-packages/index.ts b/test/src/browser-installer/ubuntu-packages/index.ts index aa29f26d6..4bcb3e0e9 100644 --- a/test/src/browser-installer/ubuntu-packages/index.ts +++ b/test/src/browser-installer/ubuntu-packages/index.ts @@ -5,6 +5,7 @@ import type { installUbuntuPackageDependencies as InstallUbuntuPackageDependencies, getUbuntuLinkerEnv as GetUbuntuLinkerEnv, } from "../../../../src/browser-installer/ubuntu-packages"; +import type { DownloadProgressCallback } from "../../../../src/browser-installer/utils"; describe("browser-installer/ubuntu-packages", () => { const sandbox = sinon.createSandbox(); @@ -18,6 +19,9 @@ describe("browser-installer/ubuntu-packages", () => { let loggerWarnStub: SinonStub; let installUbuntuPackagesStub: SinonStub; let getUbuntuMilestoneStub: SinonStub; + let hasOsPackagesStub: SinonStub; + let getOsPackagesPathStub: SinonStub; + let installOsPackagesStub: SinonStub; beforeEach(() => { fsStub = { @@ -32,11 +36,29 @@ describe("browser-installer/ubuntu-packages", () => { loggerWarnStub = sandbox.stub(); installUbuntuPackagesStub = sandbox.stub(); getUbuntuMilestoneStub = sandbox.stub().resolves("20"); + hasOsPackagesStub = sandbox.stub().returns(false); + getOsPackagesPathStub = sandbox.stub().resolves("/.testplane/packages/ubuntu/20"); + installOsPackagesStub = sandbox + .stub() + .callsFake( + async ( + _, + __, + installFn: (downloadProgressCallback: DownloadProgressCallback) => Promise, + ): Promise => { + return installFn(sinon.stub()); + }, + ); const ubuntuPackages = proxyquire("../../../../src/browser-installer/ubuntu-packages", { "fs-extra": fsStub, "./apt": { installUbuntuPackages: installUbuntuPackagesStub }, "./utils": { getUbuntuMilestone: getUbuntuMilestoneStub }, + "../registry": { + hasOsPackages: hasOsPackagesStub, + getOsPackagesPath: getOsPackagesPathStub, + installOsPackages: installOsPackagesStub, + }, "../../utils/logger": { log: loggerLogStub, warn: loggerWarnStub }, }); @@ -72,34 +94,9 @@ describe("browser-installer/ubuntu-packages", () => { await installUbuntuPackageDependencies(); - assert.calledOnceWith(loggerLogStub, "Downloading extra deb packages to local browsers execution..."); - assert.calledOnceWith(installUbuntuPackagesStub, ["foo", "bar"], sinon.match("packages")); - }); - - it("should read dependencies and install packages only once per multiple function calls", async () => { - getUbuntuMilestoneStub.resolves("20"); - fsStub.existsSync.withArgs(sinon.match("packages")).returns(false); - fsStub.readJSON.withArgs(sinon.match("ubuntu-20-dependencies.json")).resolves(["foo", "bar"]); - - const promise1 = await installUbuntuPackageDependencies(); - const promise2 = await installUbuntuPackageDependencies(); - - assert.equal(promise1, promise2); - assert.calledOnce(fsStub.readJSON); assert.calledOnceWith(installUbuntuPackagesStub, ["foo", "bar"], sinon.match("packages")); }); - it("should skip installation if directory with packages exists", async () => { - getUbuntuMilestoneStub.resolves("20"); - fsStub.existsSync.withArgs(sinon.match("packages")).returns(true); - - await installUbuntuPackageDependencies(); - - assert.notCalled(fsStub.readJSON.withArgs(sinon.match("ubuntu-20-dependencies.json"))); - assert.notCalled(loggerLogStub); - assert.notCalled(installUbuntuPackagesStub); - }); - it("should log warning if current ubuntu version is not supported", async () => { getUbuntuMilestoneStub.resolves("100500"); fsStub.readJSON.withArgs(sinon.match("ubuntu-100500-dependencies.json")).rejects(new Error("No such file")); @@ -119,6 +116,7 @@ describe("browser-installer/ubuntu-packages", () => { describe("getUbuntuLinkerEnv", () => { beforeEach(() => { + hasOsPackagesStub.returns(true); fsStub.existsSync.withArgs(sinon.match("packages")).returns(true); fsStub.readdir.withArgs(sinon.match("/lib")).resolves(["foo", "bar"]); fsStub.readdir.withArgs(sinon.match("/usr/lib")).resolves(["baz", "qux"]); @@ -128,10 +126,10 @@ describe("browser-installer/ubuntu-packages", () => { it("should resolve ubuntu linker env", async () => { const env = await getUbuntuLinkerEnv(); - assert.match(env.LD_LIBRARY_PATH, "/packages/lib/foo"); - assert.match(env.LD_LIBRARY_PATH, "/packages/lib/bar"); - assert.match(env.LD_LIBRARY_PATH, "/packages/usr/lib/baz"); - assert.match(env.LD_LIBRARY_PATH, "/packages/usr/lib/qux"); + assert.match(env.LD_LIBRARY_PATH, "/packages/ubuntu/20/lib/foo"); + assert.match(env.LD_LIBRARY_PATH, "/packages/ubuntu/20/lib/bar"); + assert.match(env.LD_LIBRARY_PATH, "/packages/ubuntu/20/usr/lib/baz"); + assert.match(env.LD_LIBRARY_PATH, "/packages/ubuntu/20/usr/lib/qux"); }); it("should concat existing LD_LIBRARY_PATH", async () => {