From dbacdc972d3cd7a5ecef1b4897ec7e9c197c62f8 Mon Sep 17 00:00:00 2001 From: Roman Kuznetsov Date: Fri, 29 Nov 2024 03:13:27 +0300 Subject: [PATCH] feat: auto ubuntu packages download for local browsers --- .github/workflows/collect-deps.yml | 56 ++ .npmignore | 1 + package-lock.json | 614 ++++++++++++++---- package.json | 6 +- src/browser-installer/chrome/index.ts | 18 +- .../chromium/revisions/linux.ts | 1 - .../chromium/revisions/mac.ts | 1 - .../chromium/revisions/win32.ts | 1 - .../chromium/revisions/win64.ts | 1 - src/browser-installer/constants.ts | 3 + src/browser-installer/firefox/index.ts | 13 + src/browser-installer/install.ts | 32 +- src/browser-installer/registry/index.ts | 4 +- src/browser-installer/ubuntu-packages/apt.ts | 150 +++++ .../autogenerated/ubuntu-20-dependencies.json | 42 ++ .../autogenerated/ubuntu-22-dependencies.json | 42 ++ .../autogenerated/ubuntu-24-dependencies.json | 42 ++ .../ubuntu-packages/index.ts | 116 ++++ .../ubuntu-packages/utils.ts | 86 +++ src/browser-installer/utils.ts | 1 + .../browser-downloader.ts | 42 ++ .../browser-versions/chrome.ts | 20 + .../browser-versions/chromium.ts | 13 + .../browser-versions/firefox.ts | 34 + .../browser-versions/index.ts | 17 + .../processed-browsers-linux.json | 192 ++++++ .../shared-objects-map-ubuntu-20.json | 52 ++ .../shared-objects-map-ubuntu-22.json | 52 ++ .../shared-objects-map-ubuntu-24.json | 52 ++ .../cache/index.ts | 133 ++++ .../constants.ts | 5 + .../index.ts | 69 ++ .../shared-object.ts | 34 + .../ubuntu/apt-file.ts | 23 + .../ubuntu/index.ts | 2 + .../ubuntu/readelf.ts | 21 + .../ubuntu/utils.ts | 9 + .../utils.ts | 12 + test/src/browser-installer/chrome/index.ts | 61 ++ test/src/browser-installer/firefox/index.ts | 69 ++ test/src/browser-installer/install.ts | 160 +++-- test/src/browser-installer/registry.ts | 8 +- test/src/browser-installer/ubuntu-packages.ts | 145 +++++ .../cache.ts | 95 +++ .../shared-object.ts | 81 +++ .../utils.ts | 17 + tsconfig.json | 1 + 47 files changed, 2457 insertions(+), 192 deletions(-) create mode 100644 .github/workflows/collect-deps.yml create mode 100644 src/browser-installer/ubuntu-packages/apt.ts create mode 100644 src/browser-installer/ubuntu-packages/autogenerated/ubuntu-20-dependencies.json create mode 100644 src/browser-installer/ubuntu-packages/autogenerated/ubuntu-22-dependencies.json create mode 100644 src/browser-installer/ubuntu-packages/autogenerated/ubuntu-24-dependencies.json create mode 100644 src/browser-installer/ubuntu-packages/index.ts create mode 100644 src/browser-installer/ubuntu-packages/utils.ts create mode 100644 src/collect-ubuntu-browser-dependencies/browser-downloader.ts create mode 100644 src/collect-ubuntu-browser-dependencies/browser-versions/chrome.ts create mode 100644 src/collect-ubuntu-browser-dependencies/browser-versions/chromium.ts create mode 100644 src/collect-ubuntu-browser-dependencies/browser-versions/firefox.ts create mode 100644 src/collect-ubuntu-browser-dependencies/browser-versions/index.ts create mode 100644 src/collect-ubuntu-browser-dependencies/cache/autogenerated/processed-browsers-linux.json create mode 100644 src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-20.json create mode 100644 src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-22.json create mode 100644 src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-24.json create mode 100644 src/collect-ubuntu-browser-dependencies/cache/index.ts create mode 100644 src/collect-ubuntu-browser-dependencies/constants.ts create mode 100644 src/collect-ubuntu-browser-dependencies/index.ts create mode 100644 src/collect-ubuntu-browser-dependencies/shared-object.ts create mode 100644 src/collect-ubuntu-browser-dependencies/ubuntu/apt-file.ts create mode 100644 src/collect-ubuntu-browser-dependencies/ubuntu/index.ts create mode 100644 src/collect-ubuntu-browser-dependencies/ubuntu/readelf.ts create mode 100644 src/collect-ubuntu-browser-dependencies/ubuntu/utils.ts create mode 100644 src/collect-ubuntu-browser-dependencies/utils.ts create mode 100644 test/src/browser-installer/ubuntu-packages.ts create mode 100644 test/src/collect-ubuntu-browser-dependencies/cache.ts create mode 100644 test/src/collect-ubuntu-browser-dependencies/shared-object.ts create mode 100644 test/src/collect-ubuntu-browser-dependencies/utils.ts diff --git a/.github/workflows/collect-deps.yml b/.github/workflows/collect-deps.yml new file mode 100644 index 000000000..7a7c9f07f --- /dev/null +++ b/.github/workflows/collect-deps.yml @@ -0,0 +1,56 @@ +name: Collect ubuntu browser dependencies +on: + schedule: + - cron: 0 0 1 * * +permissions: + pull-requests: write +jobs: + collect: + name: Collect browser dependencies + runs-on: ${{ matrix.os }} + env: + BRANCH_NAME: resolve-ubuntu-dependencies-${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + token: ${{ secrets.GH_ACCESS_TOKEN }} + - name: Setup Node JS + uses: actions/setup-node@v2 + with: + node-version: 18 + registry-url: https://registry.npmjs.org + - name: Prepare Ubuntu + run: sudo apt-get update && sudo apt-get install -y apt-file && sudo apt-file update + - run: npm ci + - name: Config git + run: git config --global user.name "y-infra" && git config --global user.email "y-infra@yandex.ru" + - name: Fetch branches + run: git fetch --all + - name: Checkout to branch + run: | + if git ls-remote --heads origin "$BRANCH_NAME" | grep -q "$BRANCH_NAME"; then + git checkout ${{ env.BRANCH_NAME }} + git pull + else + git checkout -b ${{ env.BRANCH_NAME }} + fi + - run: npm run resolve-ubuntu-dependencies + - run: git add src + - name: Commit changes + run: | + if git diff-index --quiet HEAD src; then + echo 'No changes' + else + echo 'Committing changes' + git commit src -m 'chore: update local browser dependencies for ${{ matrix.os }}' + git push origin ${{ env.BRANCH_NAME }} + gh pr create -B master -H ${{ env.BRANCH_NAME }} --title "Auto update local browser deps for ${{ matrix.os }}" --body "Created by Github action" || echo "Could not create PR. Seems like it already exists" + fi + env: + GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.npmignore b/.npmignore index 62f321a30..ced1e924d 100644 --- a/.npmignore +++ b/.npmignore @@ -3,3 +3,4 @@ .sublime-project* test/ examples/ +tsconfig.json diff --git a/package-lock.json b/package-lock.json index 43a1001c8..8865b194d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -84,6 +84,7 @@ "@types/debug": "4.1.12", "@types/escape-string-regexp": "2.0.1", "@types/fs-extra": "11.0.4", + "@types/js-levenshtein": "1.1.3", "@types/lodash": "4.14.191", "@types/micromatch": "4.0.9", "@types/mocha": "10.0.1", @@ -111,8 +112,10 @@ "eslint": "8.25.0", "eslint-config-gemini-testing": "2.8.0", "eslint-config-prettier": "8.7.0", + "execa": "5.1.1", "glob-extra": "5.0.2", "husky": "0.11.4", + "js-levenshtein": "1.1.6", "jsdom": "^24.0.0", "jsdom-global": "3.0.2", "onchange": "7.1.0", @@ -812,6 +815,128 @@ "node": ">=v18" } }, + "node_modules/@commitlint/cli/node_modules/execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/@commitlint/cli/node_modules/get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "dev": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/cli/node_modules/human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "dev": true, + "engines": { + "node": ">=16.17.0" + } + }, + "node_modules/@commitlint/cli/node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/cli/node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/cli/node_modules/npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "dev": true, + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/cli/node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/cli/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/cli/node_modules/strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@commitlint/cli/node_modules/yargs": { "version": "17.5.1", "dev": true, @@ -1039,6 +1164,128 @@ "node": ">=v18" } }, + "node_modules/@commitlint/rules/node_modules/execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/@commitlint/rules/node_modules/get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "dev": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/rules/node_modules/human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "dev": true, + "engines": { + "node": ">=16.17.0" + } + }, + "node_modules/@commitlint/rules/node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/rules/node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/rules/node_modules/npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "dev": true, + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/rules/node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/rules/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@commitlint/rules/node_modules/strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@commitlint/to-lines": { "version": "19.0.0", "dev": true, @@ -2290,6 +2537,12 @@ "@types/istanbul-lib-report": "*" } }, + "node_modules/@types/js-levenshtein": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@types/js-levenshtein/-/js-levenshtein-1.1.3.tgz", + "integrity": "sha512-jd+Q+sD20Qfu9e2aEXogiO3vpOC1PYJOUdyN9gvs4Qrvkg4wF43L5OhqrPeokdv8TL0/mXoYfpkcoGZMNN2pkQ==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.15", "dev": true, @@ -3485,17 +3738,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/archiver-utils/node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/archiver-utils/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -4714,17 +4956,6 @@ "node": ">=0.8.x" } }, - "node_modules/compress-commons/node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/compress-commons/node_modules/readable-stream": { "version": "4.5.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", @@ -7223,38 +7454,46 @@ } }, "node_modules/execa": { - "version": "8.0.1", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", "dev": true, - "license": "MIT", "dependencies": { "cross-spawn": "^7.0.3", - "get-stream": "^8.0.1", - "human-signals": "^5.0.0", - "is-stream": "^3.0.0", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", "merge-stream": "^2.0.0", - "npm-run-path": "^5.1.0", - "onetime": "^6.0.0", - "signal-exit": "^4.1.0", - "strip-final-newline": "^3.0.0" + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" }, "engines": { - "node": ">=16.17" + "node": ">=10" }, "funding": { "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, "node_modules/execa/node_modules/get-stream": { - "version": "8.0.1", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", "dev": true, - "license": "MIT", "engines": { - "node": ">=16" + "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/execa/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, "node_modules/expand-template": { "version": "2.0.3", "license": "(MIT OR WTFPL)", @@ -8432,11 +8671,12 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/human-signals": { - "version": "5.0.0", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true, - "license": "Apache-2.0", "engines": { - "node": ">=16.17.0" + "node": ">=10.17.0" } }, "node_modules/husky": { @@ -8756,11 +8996,11 @@ "license": "MIT" }, "node_modules/is-stream": { - "version": "3.0.0", - "dev": true, - "license": "MIT", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -8957,6 +9197,15 @@ "js-graphs": "src/jsgraphs.js" } }, + "node_modules/js-levenshtein": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", + "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/js-sdsl": { "version": "4.1.5", "dev": true, @@ -9942,14 +10191,12 @@ } }, "node_modules/mimic-fn": { - "version": "4.0.0", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true, - "license": "MIT", "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=6" } }, "node_modules/mimic-response": { @@ -10495,28 +10742,15 @@ } }, "node_modules/npm-run-path": { - "version": "5.3.0", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", "dev": true, - "license": "MIT", "dependencies": { - "path-key": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + "path-key": "^3.0.0" }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/npm-run-path/node_modules/path-key": { - "version": "4.0.0", - "dev": true, - "license": "MIT", "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=8" } }, "node_modules/nwsapi": { @@ -10564,14 +10798,15 @@ } }, "node_modules/onetime": { - "version": "6.0.0", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", "dev": true, - "license": "MIT", "dependencies": { - "mimic-fn": "^4.0.0" + "mimic-fn": "^2.1.0" }, "engines": { - "node": ">=12" + "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -12707,14 +12942,12 @@ } }, "node_modules/strip-final-newline": { - "version": "3.0.0", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "dev": true, - "license": "MIT", "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=6" } }, "node_modules/strip-indent": { @@ -15174,6 +15407,77 @@ "yargs": "^17.0.0" }, "dependencies": { + "execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + } + }, + "get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "dev": true + }, + "human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "dev": true + }, + "is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true + }, + "mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true + }, + "npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "dev": true, + "requires": { + "path-key": "^4.0.0" + } + }, + "onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "requires": { + "mimic-fn": "^4.0.0" + } + }, + "path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true + }, + "strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true + }, "yargs": { "version": "17.5.1", "dev": true, @@ -15329,6 +15633,79 @@ "@commitlint/to-lines": "^19.0.0", "@commitlint/types": "^19.0.3", "execa": "^8.0.1" + }, + "dependencies": { + "execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + } + }, + "get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "dev": true + }, + "human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "dev": true + }, + "is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true + }, + "mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true + }, + "npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "dev": true, + "requires": { + "path-key": "^4.0.0" + } + }, + "onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "requires": { + "mimic-fn": "^4.0.0" + } + }, + "path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true + }, + "strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true + } } }, "@commitlint/to-lines": { @@ -16154,6 +16531,12 @@ "@types/istanbul-lib-report": "*" } }, + "@types/js-levenshtein": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@types/js-levenshtein/-/js-levenshtein-1.1.3.tgz", + "integrity": "sha512-jd+Q+sD20Qfu9e2aEXogiO3vpOC1PYJOUdyN9gvs4Qrvkg4wF43L5OhqrPeokdv8TL0/mXoYfpkcoGZMNN2pkQ==", + "dev": true + }, "@types/json-schema": { "version": "7.0.15", "dev": true @@ -17013,11 +17396,6 @@ "path-scurry": "^1.11.1" } }, - "is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" - }, "minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -17780,11 +18158,6 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" }, - "is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" - }, "readable-stream": { "version": "4.5.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", @@ -19412,22 +19785,32 @@ } }, "execa": { - "version": "8.0.1", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", "dev": true, "requires": { "cross-spawn": "^7.0.3", - "get-stream": "^8.0.1", - "human-signals": "^5.0.0", - "is-stream": "^3.0.0", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", "merge-stream": "^2.0.0", - "npm-run-path": "^5.1.0", - "onetime": "^6.0.0", - "signal-exit": "^4.1.0", - "strip-final-newline": "^3.0.0" + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" }, "dependencies": { "get-stream": { - "version": "8.0.1", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true + }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true } } @@ -20185,7 +20568,9 @@ } }, "human-signals": { - "version": "5.0.0", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true }, "husky": { @@ -20378,8 +20763,9 @@ "dev": true }, "is-stream": { - "version": "3.0.0", - "dev": true + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" }, "is-text-path": { "version": "1.0.1", @@ -20503,6 +20889,12 @@ "js-graph-algorithms": { "version": "1.0.18" }, + "js-levenshtein": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", + "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", + "dev": true + }, "js-sdsl": { "version": "4.1.5", "dev": true @@ -21154,7 +21546,9 @@ } }, "mimic-fn": { - "version": "4.0.0", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, "mimic-response": { @@ -21497,16 +21891,12 @@ "dev": true }, "npm-run-path": { - "version": "5.3.0", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", "dev": true, "requires": { - "path-key": "^4.0.0" - }, - "dependencies": { - "path-key": { - "version": "4.0.0", - "dev": true - } + "path-key": "^3.0.0" } }, "nwsapi": { @@ -21540,10 +21930,12 @@ } }, "onetime": { - "version": "6.0.0", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", "dev": true, "requires": { - "mimic-fn": "^4.0.0" + "mimic-fn": "^2.1.0" } }, "os-browserify": { @@ -22929,7 +23321,9 @@ } }, "strip-final-newline": { - "version": "3.0.0", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "dev": true }, "strip-indent": { diff --git a/package.json b/package.json index fbc8fed65..a47abe158 100644 --- a/package.json +++ b/package.json @@ -9,10 +9,11 @@ ], "scripts": { "build": "tsc --build && npm run copy-static && npm run build-bundles", - "copy-static": "copyfiles 'src/browser/client-scripts/*' build", + "copy-static": "copyfiles 'src/browser/client-scripts/*' 'src/**/*.json' build", "build-node-bundle": "esbuild ./src/bundle/cjs/index.ts --outdir=./build/src/bundle/cjs --bundle --format=cjs --platform=node --target=ES2021", "build-browser-bundle": "node ./src/browser/client-scripts/build.js", "build-bundles": "concurrently -c 'auto' 'npm:build-browser-bundle' 'npm:build-node-bundle --minify'", + "resolve-ubuntu-dependencies": "ts-node ./src/collect-ubuntu-browser-dependencies", "check-types": "tsc --project tsconfig.spec.json", "clean": "rimraf build/ *.tsbuildinfo", "lint": "eslint --cache . && prettier --check .", @@ -122,6 +123,7 @@ "@types/debug": "4.1.12", "@types/escape-string-regexp": "2.0.1", "@types/fs-extra": "11.0.4", + "@types/js-levenshtein": "1.1.3", "@types/lodash": "4.14.191", "@types/micromatch": "4.0.9", "@types/mocha": "10.0.1", @@ -150,7 +152,9 @@ "eslint-config-gemini-testing": "2.8.0", "eslint-config-prettier": "8.7.0", "glob-extra": "5.0.2", + "execa": "5.1.1", "husky": "0.11.4", + "js-levenshtein": "1.1.6", "jsdom": "^24.0.0", "jsdom-global": "3.0.2", "onchange": "7.1.0", diff --git a/src/browser-installer/chrome/index.ts b/src/browser-installer/chrome/index.ts index 9ebb4061d..ee8089d63 100644 --- a/src/browser-installer/chrome/index.ts +++ b/src/browser-installer/chrome/index.ts @@ -6,6 +6,7 @@ import { DRIVER_WAIT_TIMEOUT } from "../constants"; import { getMilestone } from "../utils"; import { installChrome } from "./browser"; import { installChromeDriver } from "./driver"; +import { isUbuntu, getUbuntuLinkerEnv, installUbuntuPackageDependencies } from "../ubuntu-packages"; export { installChrome, installChromeDriver }; @@ -13,14 +14,29 @@ export const runChromeDriver = async ( chromeVersion: string, { debug = false } = {}, ): Promise<{ gridUrl: string; process: ChildProcess; port: number }> => { - const [chromeDriverPath] = await Promise.all([installChromeDriver(chromeVersion), installChrome(chromeVersion)]); + const shouldInstallUbuntuPackageDependencies = await isUbuntu(); + + const [chromeDriverPath] = await Promise.all([ + installChromeDriver(chromeVersion), + installChrome(chromeVersion), + shouldInstallUbuntuPackageDependencies ? installUbuntuPackageDependencies() : null, + ]); const milestone = getMilestone(chromeVersion); const randomPort = await getPort(); + const extraSpawnOpts = shouldInstallUbuntuPackageDependencies + ? { + env: { + ...process.env, + ...(await getUbuntuLinkerEnv()), + }, + } + : {}; const chromeDriver = spawn(chromeDriverPath, [`--port=${randomPort}`, debug ? `--verbose` : "--silent"], { windowsHide: true, detached: false, + ...extraSpawnOpts, }); if (debug) { diff --git a/src/browser-installer/chromium/revisions/linux.ts b/src/browser-installer/chromium/revisions/linux.ts index 27817229c..936a022e1 100644 --- a/src/browser-installer/chromium/revisions/linux.ts +++ b/src/browser-installer/chromium/revisions/linux.ts @@ -8,7 +8,6 @@ export default { 79: 707231, 80: 722374, 81: 737198, - 82: 750023, 83: 756143, 84: 769125, 85: 782822, diff --git a/src/browser-installer/chromium/revisions/mac.ts b/src/browser-installer/chromium/revisions/mac.ts index 400c85312..f3a11fb0c 100644 --- a/src/browser-installer/chromium/revisions/mac.ts +++ b/src/browser-installer/chromium/revisions/mac.ts @@ -8,7 +8,6 @@ export default { 79: 707225, 80: 722372, 81: 737194, - 82: 749986, 83: 756141, 84: 769122, 85: 782819, diff --git a/src/browser-installer/chromium/revisions/win32.ts b/src/browser-installer/chromium/revisions/win32.ts index 763cc6f2a..77abda40d 100644 --- a/src/browser-installer/chromium/revisions/win32.ts +++ b/src/browser-installer/chromium/revisions/win32.ts @@ -8,7 +8,6 @@ export default { 79: 707225, 80: 722370, 81: 737194, - 82: 750023, 83: 756143, 84: 769121, 85: 782817, diff --git a/src/browser-installer/chromium/revisions/win64.ts b/src/browser-installer/chromium/revisions/win64.ts index 7bf9e27e7..15376c8be 100644 --- a/src/browser-installer/chromium/revisions/win64.ts +++ b/src/browser-installer/chromium/revisions/win64.ts @@ -8,7 +8,6 @@ export default { 79: 707229, 80: 722374, 81: 737198, - 82: 750000, 83: 756141, 84: 769133, 85: 782823, diff --git a/src/browser-installer/constants.ts b/src/browser-installer/constants.ts index 68cc14b3b..3fd81ffa5 100644 --- a/src/browser-installer/constants.ts +++ b/src/browser-installer/constants.ts @@ -7,7 +7,10 @@ export const MIN_CHROMEDRIVER_FOR_TESTING_VERSION = 115; export const MIN_CHROMEDRIVER_MAC_ARM_NEW_ARCHIVE_NAME = 106; export const MIN_CHROMIUM_MAC_ARM_VERSION = 93; export const MIN_CHROMIUM_VERSION = 73; +export const MIN_FIREFOX_VERSION = 60; export const MIN_EDGEDRIVER_VERSION = 94; export const DRIVER_WAIT_TIMEOUT = 10 * 1000; // 10s export const BYTES_PER_KILOBYTE = 1 << 10; // eslint-disable-line no-bitwise export const BYTES_PER_MEGABYTE = BYTES_PER_KILOBYTE << 10; // eslint-disable-line no-bitwise +export const LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME = "LD_LIBRARY_PATH"; +export const MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED = ["fontconfig"]; diff --git a/src/browser-installer/firefox/index.ts b/src/browser-installer/firefox/index.ts index 7d6d42bf9..06a5b33cc 100644 --- a/src/browser-installer/firefox/index.ts +++ b/src/browser-installer/firefox/index.ts @@ -6,6 +6,7 @@ import { installFirefox } from "./browser"; import { installLatestGeckoDriver } from "./driver"; import { pipeLogsWithPrefix } from "../../dev-server/utils"; import { DRIVER_WAIT_TIMEOUT } from "../constants"; +import { getUbuntuLinkerEnv, installUbuntuPackageDependencies, isUbuntu } from "../ubuntu-packages"; export { installFirefox, installLatestGeckoDriver }; @@ -13,12 +14,23 @@ export const runGeckoDriver = async ( firefoxVersion: string, { debug = false } = {}, ): Promise<{ gridUrl: string; process: ChildProcess; port: number }> => { + const shouldInstallUbuntuPackageDependencies = await isUbuntu(); + const [geckoDriverPath] = await Promise.all([ installLatestGeckoDriver(firefoxVersion), installFirefox(firefoxVersion), + shouldInstallUbuntuPackageDependencies ? installUbuntuPackageDependencies() : null, ]); const randomPort = await getPort(); + const extraSpawnOpts = shouldInstallUbuntuPackageDependencies + ? { + env: { + ...process.env, + ...(await getUbuntuLinkerEnv()), + }, + } + : {}; const geckoDriver = await startGeckoDriver({ customGeckoDriverPath: geckoDriverPath, @@ -27,6 +39,7 @@ export const runGeckoDriver = async ( spawnOpts: { windowsHide: true, detached: false, + ...extraSpawnOpts, }, }); diff --git a/src/browser-installer/install.ts b/src/browser-installer/install.ts index b6f101d98..ae9f88543 100644 --- a/src/browser-installer/install.ts +++ b/src/browser-installer/install.ts @@ -6,7 +6,7 @@ import _ from "lodash"; export const installBrowser = async ( browserName?: string, browserVersion?: string, - { force = false, installWebDriver = false } = {}, + { force = false, shouldInstallWebDriver = false, shouldInstallUbuntuPackages = true } = {}, ): Promise => { const unsupportedBrowserError = new Error( [ @@ -25,28 +25,30 @@ export const installBrowser = async ( ); } + const { isUbuntu, installUbuntuPackageDependencies } = await import("./ubuntu-packages"); + + const needToInstallUbuntuPackages = shouldInstallUbuntuPackages && (await isUbuntu()); + if (/chrome/i.test(browserName)) { const { installChrome, installChromeDriver } = await import("./chrome"); - return installWebDriver - ? await Promise.all([ - installChrome(browserVersion, { force }), - installChromeDriver(browserVersion, { force }), - ]).then(binaries => binaries[0]) - : installChrome(browserVersion, { force }); + return await Promise.all([ + installChrome(browserVersion, { force }), + shouldInstallWebDriver && installChromeDriver(browserVersion, { force }), + needToInstallUbuntuPackages && installUbuntuPackageDependencies(), + ]).then(result => result[0]); } else if (/firefox/i.test(browserName)) { const { installFirefox, installLatestGeckoDriver } = await import("./firefox"); - return installWebDriver - ? await Promise.all([ - installFirefox(browserVersion, { force }), - installLatestGeckoDriver(browserVersion, { force }), - ]).then(binaries => binaries[0]) - : installFirefox(browserVersion, { force }); + return await Promise.all([ + installFirefox(browserVersion, { force }), + shouldInstallWebDriver && installLatestGeckoDriver(browserVersion, { force }), + needToInstallUbuntuPackages && installUbuntuPackageDependencies(), + ]).then(result => result[0]); } else if (/edge/i.test(browserName)) { const { installEdgeDriver } = await import("./edge"); - if (installWebDriver) { + if (shouldInstallWebDriver) { await installEdgeDriver(browserVersion, { force }); } @@ -79,7 +81,7 @@ const forceInstallBinaries = async ( browserName?: string, browserVersion?: string, ): ForceInstallBinaryResult => { - return installFn(browserName, browserVersion, { force: true, installWebDriver: true }) + return installFn(browserName, browserVersion, { force: true, shouldInstallWebDriver: true }) .then(successResult => { return successResult ? { status: BrowserInstallStatus.Ok } diff --git a/src/browser-installer/registry/index.ts b/src/browser-installer/registry/index.ts index e5b600e04..bdd0122fc 100644 --- a/src/browser-installer/registry/index.ts +++ b/src/browser-installer/registry/index.ts @@ -1,5 +1,5 @@ import type { BrowserPlatform } from "@puppeteer/browsers"; -import { readJsonSync, outputJSONSync, existsSync } from "fs-extra"; +import { readJSONSync, outputJSONSync, existsSync } from "fs-extra"; import path from "path"; import { getRegistryPath, @@ -23,7 +23,7 @@ type RegistryKey = `${BinaryName}_${BrowserPlatform}`; type Registry = Record; const registryPath = getRegistryPath(); -const registry: Registry = existsSync(registryPath) ? readJsonSync(registryPath) : {}; +const registry: Registry = existsSync(registryPath) ? readJSONSync(registryPath) : {}; let cliProgressBar: ReturnType | null = null; let warnedFirstTimeInstall = false; diff --git a/src/browser-installer/ubuntu-packages/apt.ts b/src/browser-installer/ubuntu-packages/apt.ts new file mode 100644 index 000000000..a957c44a3 --- /dev/null +++ b/src/browser-installer/ubuntu-packages/apt.ts @@ -0,0 +1,150 @@ +import os from "os"; +import path from "path"; +import fs from "fs-extra"; +import { exec } from "child_process"; +import { ensureUnixBinaryExists } from "./utils"; +import { browserInstallerDebug } from "../utils"; +import { MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED } from "../constants"; + +/** @link https://manpages.org/apt-cache/8 */ +const resolveTransitiveDependencies = async (directDependencies: string[]): Promise => { + await Promise.all(["apt-cache", "grep", "sort"].map(ensureUnixBinaryExists)); + + const aptDependsArgs = [ + "recurse", + "no-recommends", + "no-suggests", + "no-conflicts", + "no-breaks", + "no-replaces", + "no-enhances", + ] + .map(arg => `--${arg}`) + .join(" "); + + const listDependencies = (dependencyName: string): Promise => + new Promise((resolve, reject) => { + exec(`apt-cache depends ${aptDependsArgs} "${dependencyName}" | grep "^\\w" | sort -u`, (err, result) => { + if (err) { + reject(err); + } else { + resolve(result.split(/\s+/).filter(Boolean)); + } + }); + }); + + const fullDependencies = await Promise.all(directDependencies.map(listDependencies)); + + const dependenciesSet = new Set(); + + fullDependencies.forEach(depsArray => depsArray.forEach(dependency => dependenciesSet.add(dependency))); + + return Array.from(dependenciesSet); +}; + +/** @link https://manpages.org/apt/8 */ +const filterNotExistingDependencies = async (dependencies: string[]): Promise => { + if (!dependencies.length) { + return []; + } + + await ensureUnixBinaryExists("apt"); + + const existingDependencies = await new Promise((resolve, reject) => { + exec(`apt list ${dependencies.join(" ")} --installed`, (err, result) => { + if (err) { + reject(err); + } else { + const lines = result.split("\n"); + const existingDependencies = lines + .map(line => { + const slashIndex = line.indexOf("/"); + + if (slashIndex === -1) { + return ""; + } + + return line.slice(0, slashIndex); + }) + .filter(Boolean); + + resolve(existingDependencies); + } + }); + }); + + const dependenciesSet = new Set(dependencies); + + existingDependencies.forEach(existingDependency => dependenciesSet.delete(existingDependency)); + + return Array.from(dependenciesSet); +}; + +/** @link https://manpages.org/apt-get/8 */ +const downloadUbuntuPackages = async (dependencies: string[], targetDir: string): Promise => { + if (!dependencies.length) { + return; + } + + await ensureUnixBinaryExists("apt-get"); + + return new Promise((resolve, reject) => { + exec(`apt-get download ${dependencies.join(" ")}`, { cwd: targetDir }, err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +}; + +/** @link https://manpages.org/dpkg */ +const unpackUbuntuPackages = async (packagesDir: string, destination: string): Promise => { + await ensureUnixBinaryExists("dpkg"); + + return new Promise((resolve, reject) => { + exec(`for pkg in *.deb; do dpkg -x $pkg ${destination}; done`, { cwd: packagesDir }, err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +}; + +export const installUbuntuPackages = async (packages: string[], destination: string): Promise => { + const withRecursiveDependencies = await resolveTransitiveDependencies(packages); + + browserInstallerDebug(`Resolved direct packages to ${withRecursiveDependencies.length} dependencies`); + + const dependenciesToDownload = await filterNotExistingDependencies(withRecursiveDependencies); + + browserInstallerDebug(`There are ${dependenciesToDownload.length} deb packages to download`); + + if (!dependenciesToDownload.length) { + return fs.ensureDir(destination); + } + + const tmpPackagesDir = await fs.mkdtemp(path.join(os.tmpdir(), "testplane-ubuntu-apt-packages")); + + await downloadUbuntuPackages(dependenciesToDownload, tmpPackagesDir); + + browserInstallerDebug(`Downloaded ${dependenciesToDownload.length} deb packages`); + + await unpackUbuntuPackages(tmpPackagesDir, destination); + + browserInstallerDebug(`Unpacked ${dependenciesToDownload.length} deb packages`); + + const missingPkgs = MANDATORY_UBUNTU_PACKAGES_TO_BE_INSTALLED.filter(pkg => dependenciesToDownload.includes(pkg)); + + if (missingPkgs.length) { + throw new Error( + [ + "Missing some packages, which needs to be installed manually", + `Use \`apt-get install ${missingPkgs.join(" ")}\` to install them`, + ].join("\n"), + ); + } +}; diff --git a/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-20-dependencies.json b/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-20-dependencies.json new file mode 100644 index 000000000..3df0d3b05 --- /dev/null +++ b/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-20-dependencies.json @@ -0,0 +1,42 @@ +[ + "libasound2", + "libatk-bridge2.0-0", + "libatk1.0-0", + "libatspi2.0-0", + "libc6", + "libcairo2", + "libcups2", + "libdbus-1-3", + "libdbus-glib-1-2", + "libdrm2", + "libexpat1", + "libgbm1", + "libgcc-s1", + "libgdk-pixbuf2.0-0", + "libglib2.0-0", + "libgtk-3-0", + "libnspr4", + "libnss3", + "libpango-1.0-0", + "libpangocairo-1.0-0", + "libstdc++6", + "libudev1", + "libuuid1", + "libx11-6", + "libx11-xcb1", + "libxcb-dri3-0", + "libxcb1", + "libxcomposite1", + "libxcursor1", + "libxdamage1", + "libxext6", + "libxfixes3", + "libxi6", + "libxkbcommon0", + "libxrandr2", + "libxrender1", + "libxshmfence1", + "libxss1", + "libxt6", + "libxtst6" +] diff --git a/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-22-dependencies.json b/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-22-dependencies.json new file mode 100644 index 000000000..20f5d7e60 --- /dev/null +++ b/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-22-dependencies.json @@ -0,0 +1,42 @@ +[ + "libasound2", + "libatk-bridge2.0-0", + "libatk1.0-0", + "libatspi2.0-0", + "libc6", + "libcairo2", + "libcups2", + "libdbus-1-3", + "libdbus-glib-1-2", + "libdrm2", + "libexpat1", + "libgbm1", + "libgcc-s1", + "libgdk-pixbuf-2.0-0", + "libglib2.0-0", + "libgtk-3-0", + "libnspr4", + "libnss3", + "libpango-1.0-0", + "libpangocairo-1.0-0", + "libstdc++6", + "libudev1", + "libuuid1", + "libx11-6", + "libx11-xcb1", + "libxcb-dri3-0", + "libxcb1", + "libxcomposite1", + "libxcursor1", + "libxdamage1", + "libxext6", + "libxfixes3", + "libxi6", + "libxkbcommon0", + "libxrandr2", + "libxrender1", + "libxshmfence1", + "libxss1", + "libxt6", + "libxtst6" +] diff --git a/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-24-dependencies.json b/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-24-dependencies.json new file mode 100644 index 000000000..19f37be03 --- /dev/null +++ b/src/browser-installer/ubuntu-packages/autogenerated/ubuntu-24-dependencies.json @@ -0,0 +1,42 @@ +[ + "libasound2t64", + "libatk-bridge2.0-0t64", + "libatk1.0-0t64", + "libatspi2.0-0t64", + "libc6", + "libcairo2", + "libcups2t64", + "libdbus-1-3", + "libdbus-glib-1-2", + "libdrm2", + "libexpat1", + "libgbm1", + "libgcc-s1", + "libgdk-pixbuf-2.0-0", + "libglib2.0-0t64", + "libgtk-3-0t64", + "libnspr4", + "libnss3", + "libpango-1.0-0", + "libpangocairo-1.0-0", + "libstdc++6", + "libudev1", + "libuuid1", + "libx11-6", + "libx11-xcb1", + "libxcb-dri3-0", + "libxcb1", + "libxcomposite1", + "libxcursor1", + "libxdamage1", + "libxext6", + "libxfixes3", + "libxi6", + "libxkbcommon0", + "libxrandr2", + "libxrender1", + "libxshmfence1", + "libxss1", + "libxt6t64", + "libxtst6" +] diff --git a/src/browser-installer/ubuntu-packages/index.ts b/src/browser-installer/ubuntu-packages/index.ts new file mode 100644 index 000000000..8d64aa62c --- /dev/null +++ b/src/browser-installer/ubuntu-packages/index.ts @@ -0,0 +1,116 @@ +import _ from "lodash"; +import fs from "fs-extra"; +import path from "path"; +import { browserInstallerDebug, getUbuntuPackagesDir } from "../utils"; +import { installUbuntuPackages } from "./apt"; +import { getUbuntuMilestone } from "./utils"; +import logger from "../../utils/logger"; +import { LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME } from "../constants"; + +export { isUbuntu, getUbuntuMilestone, ensureUnixBinaryExists } from "./utils"; + +const getDependenciesArrayFilePath = (ubuntuMilestone: string): string => + path.join(__dirname, "autogenerated", `ubuntu-${ubuntuMilestone}-dependencies.json`); + +const readUbuntuPackageDependencies = async (ubuntuMilestone: string): Promise => { + try { + return await fs.readJSON(getDependenciesArrayFilePath(ubuntuMilestone)); + } catch (_) { + throw new Error( + `Unable to read ubuntu dependencies for Ubuntu@${ubuntuMilestone}, as this version currently not supported`, + ); + } +}; + +export const writeUbuntuPackageDependencies = async (ubuntuMilestone: string, deps: string[]): Promise => { + const currentPackages = await readUbuntuPackageDependencies(ubuntuMilestone).catch(() => [] as string[]); + + const packagesToWrite = _.uniq(currentPackages.concat(deps)).sort(); + + await fs.outputJSON(getDependenciesArrayFilePath(ubuntuMilestone), packagesToWrite, { spaces: 4 }); +}; + +let installUbuntuPackageDependenciesPromise: Promise; + +export const installUbuntuPackageDependencies = async (): Promise => { + if (installUbuntuPackageDependenciesPromise) { + return installUbuntuPackageDependenciesPromise; + } + + installUbuntuPackageDependenciesPromise = new Promise((resolve, reject) => { + const ubuntuPackagesDir = getUbuntuPackagesDir(); + + if (fs.existsSync(ubuntuPackagesDir)) { + browserInstallerDebug("Skip installing ubuntu packages, as they are installed already"); + + resolve(); + } else { + logger.log("Downloading extra deb packages to local browsers execution..."); + + getUbuntuMilestone() + .then(ubuntuMilestone => readUbuntuPackageDependencies(ubuntuMilestone)) + .then(dependencies => installUbuntuPackages(dependencies, ubuntuPackagesDir)) + .then(resolve) + .catch(reject); + } + }); + + return installUbuntuPackageDependenciesPromise; +}; + +const listDirsAbsolutePath = async (dirBasePath: string, ...prefix: string[]): Promise => { + const fullDirPath = path.join(dirBasePath, ...prefix); + + if (!fs.existsSync(fullDirPath)) { + return []; + } + + const dirContents = await fs.readdir(fullDirPath); + const dirContentsAbsPaths = dirContents.map(obj => path.join(fullDirPath, obj)); + + const directories = [] as string[]; + + await Promise.all( + dirContentsAbsPaths.map(obj => + fs.stat(obj).then(stat => { + if (stat.isDirectory()) { + directories.push(obj); + } + }), + ), + ); + + return directories; +}; + +let getUbuntuLinkerEnvPromise: Promise>; + +export const getUbuntuLinkerEnv = async (): Promise> => { + if (getUbuntuLinkerEnvPromise) { + return getUbuntuLinkerEnvPromise; + } + + getUbuntuLinkerEnvPromise = new Promise>((resolve, reject) => { + const ubuntuPackagesDir = getUbuntuPackagesDir(); + + if (!fs.existsSync(ubuntuPackagesDir)) { + return resolve({}); + } + + const currentRuntimeLibrariesEnvValue = process.env[LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME]; + + Promise.all([ + listDirsAbsolutePath(ubuntuPackagesDir, "lib"), + listDirsAbsolutePath(ubuntuPackagesDir, "usr", "lib"), + ]) + .then(([libDirs, usrLibDirs]) => { + const libraryPaths = [...libDirs, ...usrLibDirs, currentRuntimeLibrariesEnvValue].filter(Boolean); + + return { [LINUX_RUNTIME_LIBRARIES_PATH_ENV_NAME]: libraryPaths.join(":") }; + }) + .then(resolve) + .catch(reject); + }); + + return getUbuntuLinkerEnvPromise; +}; diff --git a/src/browser-installer/ubuntu-packages/utils.ts b/src/browser-installer/ubuntu-packages/utils.ts new file mode 100644 index 000000000..70381a439 --- /dev/null +++ b/src/browser-installer/ubuntu-packages/utils.ts @@ -0,0 +1,86 @@ +import { exec } from "child_process"; +import fs from "fs"; +import { browserInstallerDebug } from "../utils"; + +/** @link https://manpages.org/os-release/5 */ +const OS_RELEASE_PATH = "/etc/os-release"; + +type OsRelease = { + // General OS identification + NAME: string; + ID: string; + PRETTY_NAME: string; + ID_LIKE?: string; + CPE_NAME?: string; + VARIANT?: string; + VARIANT_ID?: string; + // Version identification + VERSION?: string; + VERSION_ID?: string; + VERSION_CODENAME?: string; + BUILD_ID?: string; + IMAGE_ID?: string; +}; + +/** @link https://manpages.org/which */ +export const ensureUnixBinaryExists = (binaryName: string): Promise => + new Promise((resolve, reject) => + exec(`which "${binaryName}"`, err => { + browserInstallerDebug(`Checking binary "${binaryName}" is installed: ${!err}`); + + if (err) { + reject(new Error(`Binary "${binaryName}" does not exist`)); + } else { + resolve(); + } + }), + ); + +/** @link https://manpages.org/os-release/5 */ +const osRelease = async (): Promise => { + if (!fs.existsSync(OS_RELEASE_PATH)) { + throw new Error(`"${OS_RELEASE_PATH}" is missing. Probably its not Linux`); + } + + const fileContents = await fs.promises.readFile(OS_RELEASE_PATH, "utf8"); + const result = {} as OsRelease; + + for (const line of fileContents.split("\n")) { + if (!line.includes("=")) { + continue; + } + + const splitPosition = line.indexOf("="); + const key = line.slice(0, splitPosition) as keyof OsRelease; + const value = line.slice(splitPosition + 1); + const valueIsWrappedWithQuotes = value.startsWith('"') && value.endsWith('"'); + + result[key] = valueIsWrappedWithQuotes ? value.slice(1, -1) : value; + } + + return result; +}; + +let isUbuntuCached: boolean | null = null; + +export const isUbuntu = async (): Promise => { + if (isUbuntuCached !== null) { + return isUbuntuCached; + } + + isUbuntuCached = await osRelease() + .then(release => release.ID === "ubuntu") + .catch(() => false); + + return isUbuntuCached; +}; + +export const getUbuntuMilestone = async (): Promise => { + const release = await osRelease(); + + if (!release.VERSION_ID) { + throw new Error(`VERSION_ID is missing in ${OS_RELEASE_PATH}. Probably its not Ubuntu`); + } + + return release.VERSION_ID.split(".")[0] as string; +}; diff --git a/src/browser-installer/utils.ts b/src/browser-installer/utils.ts index c15947fa6..fbd41a03e 100644 --- a/src/browser-installer/utils.ts +++ b/src/browser-installer/utils.ts @@ -124,6 +124,7 @@ export const getRegistryPath = (envValueOverride?: string): string => path.join(getCacheDir(envValueOverride), "registry.json"); export const getBrowsersDir = (): string => path.join(getCacheDir(), "browsers"); +export const getUbuntuPackagesDir = (): string => path.join(getCacheDir(), "packages"); const getDriversDir = (): string => path.join(getCacheDir(), "drivers"); const getDriverDir = (driverName: string, driverVersion: string): string => diff --git a/src/collect-ubuntu-browser-dependencies/browser-downloader.ts b/src/collect-ubuntu-browser-dependencies/browser-downloader.ts new file mode 100644 index 000000000..4f9b9c996 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/browser-downloader.ts @@ -0,0 +1,42 @@ +import path from "path"; +import fs from "fs"; +import _ from "lodash"; +import { installBrowser } from "../browser-installer"; +import { getRegistryPath } from "../browser-installer/utils"; +import type { BrowserWithVersion } from "./utils"; + +type BinaryNameWithArchPrefix = string; +type BinaryVersion = string; +type BinaryPath = string; + +type Registry = Record>; + +const getRegistryBinaryPaths = (registry: Registry): string[] => { + const versionToPathMap = Object.values(registry); + const binaryPaths = _.flatMap(versionToPathMap, Object.values); + const registryPath = getRegistryPath(); + + return binaryPaths.map(relativePath => path.resolve(registryPath, relativePath)); +}; + +/** @returns array of binary absolute paths */ +export const downloadBrowserVersions = async (browsers: BrowserWithVersion[]): Promise => { + if (!browsers.length) { + return []; + } + + const registryPath = getRegistryPath(); + + const installBinaries = ({ browserName, browserVersion }: BrowserWithVersion): Promise => + installBrowser(browserName, browserVersion, { + shouldInstallWebDriver: true, + shouldInstallUbuntuPackages: false, + }); + + await Promise.all(browsers.map(installBinaries)); + + const registryJson = await fs.promises.readFile(registryPath, { encoding: "utf8" }); + const registry = JSON.parse(registryJson); + + return getRegistryBinaryPaths(registry); +}; diff --git a/src/collect-ubuntu-browser-dependencies/browser-versions/chrome.ts b/src/collect-ubuntu-browser-dependencies/browser-versions/chrome.ts new file mode 100644 index 000000000..fc6e3d683 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/browser-versions/chrome.ts @@ -0,0 +1,20 @@ +import { retryFetch } from "../../browser-installer/utils"; +import { CHROME_FOR_TESTING_VERSIONS_API_URL } from "../constants"; + +type ChromeVersionInfo = { + milestone: `${number}`; + version: `${number}.${number}.${number}.${number}`; + revision: `${number}`; +}; + +type ChromeVersionsApiResponse = { milestones: Record<`${number}`, ChromeVersionInfo> }; + +export const fetchChromeMilestoneVersions = async (): Promise => { + try { + const response = await retryFetch(CHROME_FOR_TESTING_VERSIONS_API_URL); + const data = (await response.json()) as ChromeVersionsApiResponse; + return Object.values(data.milestones).map(({ version }) => version); + } catch (err) { + throw new Error(`Couldn't get chrome versions: ${err}`); + } +}; diff --git a/src/collect-ubuntu-browser-dependencies/browser-versions/chromium.ts b/src/collect-ubuntu-browser-dependencies/browser-versions/chromium.ts new file mode 100644 index 000000000..891dcd7f7 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/browser-versions/chromium.ts @@ -0,0 +1,13 @@ +import { getBrowserPlatform } from "../../browser-installer/utils"; + +export const fetchChromiumMilestoneVersions = async (): Promise => { + try { + const platform = getBrowserPlatform(); + + const { default: versions } = await import(`../../browser-installer/chromium/revisions/${platform}`); + + return Object.keys(versions); + } catch (err) { + throw new Error(`Couldn't get chromium versions: ${err}`); + } +}; diff --git a/src/collect-ubuntu-browser-dependencies/browser-versions/firefox.ts b/src/collect-ubuntu-browser-dependencies/browser-versions/firefox.ts new file mode 100644 index 000000000..763935a39 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/browser-versions/firefox.ts @@ -0,0 +1,34 @@ +import _ from "lodash"; +import { getMilestone, retryFetch } from "../../browser-installer/utils"; +import { FIREFOX_VERSIONS_API_URL } from "../constants"; +import { MIN_FIREFOX_VERSION } from "../../browser-installer/constants"; + +type FirefoxVersionInfo = { + category: "major" | "esr" | "stability" | "dev"; + date: `${number}-${number}-${number}`; + version: string; +}; + +type FirefoxVersionsApiResponse = { releases: Record }; + +export const fetchFirefoxMilestoneVersions = async (): Promise => { + try { + const response = await retryFetch(FIREFOX_VERSIONS_API_URL); + const data = (await response.json()) as FirefoxVersionsApiResponse; + const stableVersions = Object.values(data.releases) + .filter(data => ["stability", "esr"].includes(data.category)) + .filter(data => Number(getMilestone(data.version)) >= MIN_FIREFOX_VERSION); + + const majorGrouped = _.groupBy(stableVersions, data => data.version.split(".")[0]); + + return Object.keys(majorGrouped).map(groupName => { + const versionsSorted = majorGrouped[groupName].sort((a, b) => { + return parseInt(a.version.replace(".", ""), 16) - parseInt(b.version.replace(".", ""), 16); + }); + + return versionsSorted.pop()?.version as string; + }); + } catch (err) { + throw new Error(`Couldn't get firefox versions: ${err}`); + } +}; diff --git a/src/collect-ubuntu-browser-dependencies/browser-versions/index.ts b/src/collect-ubuntu-browser-dependencies/browser-versions/index.ts new file mode 100644 index 000000000..6cccca3f5 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/browser-versions/index.ts @@ -0,0 +1,17 @@ +import { fetchChromiumMilestoneVersions } from "./chromium"; +import { fetchChromeMilestoneVersions } from "./chrome"; +import { fetchFirefoxMilestoneVersions } from "./firefox"; +import type { BrowserWithVersion } from "../utils"; + +export const fetchBrowsersMilestones = async (): Promise => { + const createMapToBrowser = (browserName: string) => (data: string[]) => + data.map(browserVersion => ({ browserName, browserVersion })); + + const [chromiumVersions, chromeVersions, firefoxVersions] = await Promise.all([ + fetchChromiumMilestoneVersions().then(createMapToBrowser("chrome")), + fetchChromeMilestoneVersions().then(createMapToBrowser("chrome")), + fetchFirefoxMilestoneVersions().then(createMapToBrowser("firefox")), + ]); + + return [...chromiumVersions, ...chromeVersions, ...firefoxVersions]; +}; diff --git a/src/collect-ubuntu-browser-dependencies/cache/autogenerated/processed-browsers-linux.json b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/processed-browsers-linux.json new file mode 100644 index 000000000..b066ab001 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/processed-browsers-linux.json @@ -0,0 +1,192 @@ +{ + "downloadedBrowsers": { + "chrome": [ + "100", + "101", + "102", + "103", + "104", + "105", + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114", + "115", + "116", + "117", + "118", + "119", + "120", + "121", + "122", + "123", + "124", + "125", + "126", + "127", + "128", + "129", + "130", + "131", + "132", + "133", + "73", + "74", + "75", + "76", + "77", + "78", + "79", + "80", + "81", + "83", + "84", + "85", + "86", + "87", + "88", + "89", + "90", + "91", + "92", + "93", + "94", + "95", + "96", + "97", + "98", + "99" + ], + "firefox": [ + "100", + "101", + "102", + "103", + "104", + "105", + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114", + "115", + "115", + "116", + "117", + "118", + "119", + "120", + "121", + "122", + "123", + "124", + "125", + "126", + "127", + "128", + "128", + "129", + "130", + "131", + "132", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + "64", + "65", + "66", + "67", + "68", + "69", + "70", + "72", + "73", + "74", + "76", + "77", + "78", + "80", + "81", + "82", + "84", + "85", + "86", + "88", + "89", + "90", + "91", + "92", + "94", + "95", + "96", + "97", + "98", + "99" + ] + }, + "sharedObjects": [ + "ld-linux-x86-64.so.2", + "libX11-xcb.so.1", + "libX11.so.6", + "libXcomposite.so.1", + "libXcursor.so.1", + "libXdamage.so.1", + "libXext.so.6", + "libXfixes.so.3", + "libXi.so.6", + "libXrandr.so.2", + "libXrender.so.1", + "libXss.so.1", + "libXt.so.6", + "libXtst.so.6", + "libasound.so.2", + "libatk-1.0.so.0", + "libatk-bridge-2.0.so.0", + "libatspi.so.0", + "libc.so.6", + "libcairo.so.2", + "libcups.so.2", + "libdbus-1.so.3", + "libdbus-glib-1.so.2", + "libdl.so.2", + "libdrm.so.2", + "libexpat.so.1", + "libgbm.so.1", + "libgcc_s.so.1", + "libgdk-3.so.0", + "libgdk_pixbuf-2.0.so.0", + "libgio-2.0.so.0", + "libglib-2.0.so.0", + "libgobject-2.0.so.0", + "libgtk-3.so.0", + "libm.so.6", + "libnspr4.so", + "libnss3.so", + "libnssutil3.so", + "libpango-1.0.so.0", + "libpangocairo-1.0.so.0", + "libpthread.so.0", + "librt.so.1", + "libsmime3.so", + "libstdc++.so.6", + "libudev.so.1", + "libuuid.so.1", + "libxcb-dri3.so.0", + "libxcb.so.1", + "libxkbcommon.so.0", + "libxshmfence.so.1" + ] +} diff --git a/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-20.json b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-20.json new file mode 100644 index 000000000..98a01b903 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-20.json @@ -0,0 +1,52 @@ +{ + "ld-linux-x86-64.so.2": "libc6", + "libX11-xcb.so.1": "libx11-xcb1", + "libX11.so.6": "libx11-6", + "libXcomposite.so.1": "libxcomposite1", + "libXcursor.so.1": "libxcursor1", + "libXdamage.so.1": "libxdamage1", + "libXext.so.6": "libxext6", + "libXfixes.so.3": "libxfixes3", + "libXi.so.6": "libxi6", + "libXrandr.so.2": "libxrandr2", + "libXrender.so.1": "libxrender1", + "libXss.so.1": "libxss1", + "libXt.so.6": "libxt6", + "libXtst.so.6": "libxtst6", + "libasound.so.2": "libasound2", + "libatk-1.0.so.0": "libatk1.0-0", + "libatk-bridge-2.0.so.0": "libatk-bridge2.0-0", + "libatspi.so.0": "libatspi2.0-0", + "libc.so.6": "libc6", + "libcairo.so.2": "libcairo2", + "libcups.so.2": "libcups2", + "libdbus-1.so.3": "libdbus-1-3", + "libdbus-glib-1.so.2": "libdbus-glib-1-2", + "libdl.so.2": "libc6", + "libdrm.so.2": "libdrm2", + "libexpat.so.1": "libexpat1", + "libgbm.so.1": "libgbm1", + "libgcc_s.so.1": "libgcc-s1", + "libgdk-3.so.0": "libgtk-3-0", + "libgdk_pixbuf-2.0.so.0": "libgdk-pixbuf2.0-0", + "libgio-2.0.so.0": "libglib2.0-0", + "libglib-2.0.so.0": "libglib2.0-0", + "libgobject-2.0.so.0": "libglib2.0-0", + "libgtk-3.so.0": "libgtk-3-0", + "libm.so.6": "libc6", + "libnspr4.so": "libnspr4", + "libnss3.so": "libnss3", + "libnssutil3.so": "libnss3", + "libpango-1.0.so.0": "libpango-1.0-0", + "libpangocairo-1.0.so.0": "libpangocairo-1.0-0", + "libpthread.so.0": "libc6", + "librt.so.1": "libc6", + "libsmime3.so": "libnss3", + "libstdc++.so.6": "libstdc++6", + "libudev.so.1": "libudev1", + "libuuid.so.1": "libuuid1", + "libxcb-dri3.so.0": "libxcb-dri3-0", + "libxcb.so.1": "libxcb1", + "libxkbcommon.so.0": "libxkbcommon0", + "libxshmfence.so.1": "libxshmfence1" +} diff --git a/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-22.json b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-22.json new file mode 100644 index 000000000..f4d5b4268 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-22.json @@ -0,0 +1,52 @@ +{ + "ld-linux-x86-64.so.2": "libc6", + "libX11-xcb.so.1": "libx11-xcb1", + "libX11.so.6": "libx11-6", + "libXcomposite.so.1": "libxcomposite1", + "libXcursor.so.1": "libxcursor1", + "libXdamage.so.1": "libxdamage1", + "libXext.so.6": "libxext6", + "libXfixes.so.3": "libxfixes3", + "libXi.so.6": "libxi6", + "libXrandr.so.2": "libxrandr2", + "libXrender.so.1": "libxrender1", + "libXss.so.1": "libxss1", + "libXt.so.6": "libxt6", + "libXtst.so.6": "libxtst6", + "libasound.so.2": "libasound2", + "libatk-1.0.so.0": "libatk1.0-0", + "libatk-bridge-2.0.so.0": "libatk-bridge2.0-0", + "libatspi.so.0": "libatspi2.0-0", + "libc.so.6": "libc6", + "libcairo.so.2": "libcairo2", + "libcups.so.2": "libcups2", + "libdbus-1.so.3": "libdbus-1-3", + "libdbus-glib-1.so.2": "libdbus-glib-1-2", + "libdl.so.2": "libc6", + "libdrm.so.2": "libdrm2", + "libexpat.so.1": "libexpat1", + "libgbm.so.1": "libgbm1", + "libgcc_s.so.1": "libgcc-s1", + "libgdk-3.so.0": "libgtk-3-0", + "libgdk_pixbuf-2.0.so.0": "libgdk-pixbuf-2.0-0", + "libgio-2.0.so.0": "libglib2.0-0", + "libglib-2.0.so.0": "libglib2.0-0", + "libgobject-2.0.so.0": "libglib2.0-0", + "libgtk-3.so.0": "libgtk-3-0", + "libm.so.6": "libc6", + "libnspr4.so": "libnspr4", + "libnss3.so": "libnss3", + "libnssutil3.so": "libnss3", + "libpango-1.0.so.0": "libpango-1.0-0", + "libpangocairo-1.0.so.0": "libpangocairo-1.0-0", + "libpthread.so.0": "libc6", + "librt.so.1": "libc6", + "libsmime3.so": "libnss3", + "libstdc++.so.6": "libstdc++6", + "libudev.so.1": "libudev1", + "libuuid.so.1": "libuuid1", + "libxcb-dri3.so.0": "libxcb-dri3-0", + "libxcb.so.1": "libxcb1", + "libxkbcommon.so.0": "libxkbcommon0", + "libxshmfence.so.1": "libxshmfence1" +} diff --git a/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-24.json b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-24.json new file mode 100644 index 000000000..32d0cfcc5 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/cache/autogenerated/shared-objects-map-ubuntu-24.json @@ -0,0 +1,52 @@ +{ + "ld-linux-x86-64.so.2": "libc6", + "libX11-xcb.so.1": "libx11-xcb1", + "libX11.so.6": "libx11-6", + "libXcomposite.so.1": "libxcomposite1", + "libXcursor.so.1": "libxcursor1", + "libXdamage.so.1": "libxdamage1", + "libXext.so.6": "libxext6", + "libXfixes.so.3": "libxfixes3", + "libXi.so.6": "libxi6", + "libXrandr.so.2": "libxrandr2", + "libXrender.so.1": "libxrender1", + "libXss.so.1": "libxss1", + "libXt.so.6": "libxt6t64", + "libXtst.so.6": "libxtst6", + "libasound.so.2": "libasound2t64", + "libatk-1.0.so.0": "libatk1.0-0t64", + "libatk-bridge-2.0.so.0": "libatk-bridge2.0-0t64", + "libatspi.so.0": "libatspi2.0-0t64", + "libc.so.6": "libc6", + "libcairo.so.2": "libcairo2", + "libcups.so.2": "libcups2t64", + "libdbus-1.so.3": "libdbus-1-3", + "libdbus-glib-1.so.2": "libdbus-glib-1-2", + "libdl.so.2": "libc6", + "libdrm.so.2": "libdrm2", + "libexpat.so.1": "libexpat1", + "libgbm.so.1": "libgbm1", + "libgcc_s.so.1": "libgcc-s1", + "libgdk-3.so.0": "libgtk-3-0t64", + "libgdk_pixbuf-2.0.so.0": "libgdk-pixbuf-2.0-0", + "libgio-2.0.so.0": "libglib2.0-0t64", + "libglib-2.0.so.0": "libglib2.0-0t64", + "libgobject-2.0.so.0": "libglib2.0-0t64", + "libgtk-3.so.0": "libgtk-3-0t64", + "libm.so.6": "libc6", + "libnspr4.so": "libnspr4", + "libnss3.so": "libnss3", + "libnssutil3.so": "libnss3", + "libpango-1.0.so.0": "libpango-1.0-0", + "libpangocairo-1.0.so.0": "libpangocairo-1.0-0", + "libpthread.so.0": "libc6", + "librt.so.1": "libc6", + "libsmime3.so": "libnss3", + "libstdc++.so.6": "libstdc++6", + "libudev.so.1": "libudev1", + "libuuid.so.1": "libuuid1", + "libxcb-dri3.so.0": "libxcb-dri3-0", + "libxcb.so.1": "libxcb1", + "libxkbcommon.so.0": "libxkbcommon0", + "libxshmfence.so.1": "libxshmfence1" +} diff --git a/src/collect-ubuntu-browser-dependencies/cache/index.ts b/src/collect-ubuntu-browser-dependencies/cache/index.ts new file mode 100644 index 000000000..232977b58 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/cache/index.ts @@ -0,0 +1,133 @@ +import fs from "fs-extra"; +import _ from "lodash"; +import path from "path"; +import { getMilestone } from "../../browser-installer/utils"; +import type { BrowserWithVersion } from "../utils"; + +type SharedObjectName = string; +type PackageName = string; + +type BrowserName = string; +type BrowserVersion = string; + +export type CacheData = { + /** Different for each ubuntu version */ + sharedObjectsMap: Record; + + /** Mutual for all linux versions */ + processedBrowsers: { + downloadedBrowsers: Record; + sharedObjects: string[]; + }; +}; + +const sortObject = (obj: T): T => { + if (_.isArray(obj)) { + return obj.sort(); + } + + if (!_.isPlainObject(obj)) { + return obj; + } + + const sourceObj = obj as T & Record; + const result = {} as T; + + const sortedKeys = Object.keys(sourceObj).sort() as Array; + + for (const key of sortedKeys) { + result[key] = sortObject(sourceObj[key]); + } + + return result; +}; + +export class Cache { + private _sharedObjectsMapPath: string; + private _processedBrowsersCachePath: string; + private _cache: CacheData = { + sharedObjectsMap: {}, + processedBrowsers: { downloadedBrowsers: {}, sharedObjects: [] }, + }; + + constructor(osVersion: string) { + const autoGeneratedDirPath = path.join(__dirname, "autogenerated"); + + this._sharedObjectsMapPath = path.join(autoGeneratedDirPath, `shared-objects-map-ubuntu-${osVersion}.json`); + this._processedBrowsersCachePath = path.join(autoGeneratedDirPath, "processed-browsers-linux.json"); + } + + async read(): Promise { + const [sharedObjectsMap, processedBrowsers] = await Promise.all([ + fs.readJSON(this._sharedObjectsMapPath).catch(() => null), + fs.readJSON(this._processedBrowsersCachePath).catch(() => null), + ]); + + if (sharedObjectsMap) { + this._cache.sharedObjectsMap = sharedObjectsMap; + } + + if (processedBrowsers) { + this._cache.processedBrowsers = processedBrowsers; + } + + return this; + } + + async write(): Promise { + const { sharedObjectsMap, processedBrowsers } = this._cache; + + const uniqSharedObjects = _.uniq(Object.keys(sharedObjectsMap).concat(processedBrowsers.sharedObjects)); + + processedBrowsers.sharedObjects = uniqSharedObjects; + + await fs.outputJSON(this._sharedObjectsMapPath, sortObject(sharedObjectsMap), { spaces: 4 }); + await fs.outputJSON(this._processedBrowsersCachePath, sortObject(processedBrowsers), { spaces: 4 }); + } + + private hasProcessedBrowser({ browserName, browserVersion }: BrowserWithVersion): boolean { + const processedBrowserVersions = this._cache.processedBrowsers.downloadedBrowsers[browserName]; + const milestone = getMilestone(browserVersion); + + return Boolean(processedBrowserVersions && processedBrowserVersions.includes(milestone)); + } + + filterProcessedBrowsers(browsers: BrowserWithVersion[]): BrowserWithVersion[] { + return browsers.filter(browser => !this.hasProcessedBrowser(browser)); + } + + private saveProcessedBrowser({ browserName, browserVersion }: BrowserWithVersion): void { + const browserCache = (this._cache.processedBrowsers.downloadedBrowsers[browserName] ||= []); + const milestone = getMilestone(browserVersion); + + if (!browserCache.includes(milestone)) { + browserCache.push(milestone); + } + } + + saveProcessedBrowsers(browsers: BrowserWithVersion[]): void { + browsers.forEach(browser => this.saveProcessedBrowser(browser)); + } + + hasResolvedPackageName(sharedObjectName: string): boolean { + return Boolean(this._cache.sharedObjectsMap[sharedObjectName]); + } + + getResolvedPackageName(sharedObjectName: string): string { + if (!this.hasResolvedPackageName(sharedObjectName)) { + throw new Error(`shared object [${sharedObjectName}] is not cached`); + } + + return this._cache.sharedObjectsMap[sharedObjectName]; + } + + savePackageName(sharedObjectName: string, packageName: string): void { + this._cache.sharedObjectsMap[sharedObjectName] = packageName; + } + + getUnresolvedSharedObjects(): string[] { + const resolvedSharedObjects = Object.keys(this._cache.sharedObjectsMap); + + return _.difference(this._cache.processedBrowsers.sharedObjects, resolvedSharedObjects); + } +} diff --git a/src/collect-ubuntu-browser-dependencies/constants.ts b/src/collect-ubuntu-browser-dependencies/constants.ts new file mode 100644 index 000000000..56d8b0d49 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/constants.ts @@ -0,0 +1,5 @@ +export const FIREFOX_VERSIONS_API_URL = "https://product-details.mozilla.org/1.0/firefox.json"; +export const CHROME_FOR_TESTING_VERSIONS_API_URL = + "https://googlechromelabs.github.io/chrome-for-testing/latest-versions-per-milestone.json"; +// Those are couldn't be seen with readelf -d +export const EXTRA_FIREFOX_SHARED_OBJECTS = ["libdbus-glib-1.so.2", "libXt.so.6"]; diff --git a/src/collect-ubuntu-browser-dependencies/index.ts b/src/collect-ubuntu-browser-dependencies/index.ts new file mode 100644 index 000000000..16e7c7127 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/index.ts @@ -0,0 +1,69 @@ +import _ from "lodash"; +import { EXTRA_FIREFOX_SHARED_OBJECTS } from "./constants"; +import { getBinarySharedObjectDependencies, searchSharedObjectPackage } from "./shared-object"; +import { Cache } from "./cache"; +import { fetchBrowsersMilestones } from "./browser-versions/index"; +import { downloadBrowserVersions } from "./browser-downloader"; +import { getUbuntuMilestone, writeUbuntuPackageDependencies } from "../browser-installer/ubuntu-packages"; +import logger from "../utils/logger"; + +const createResolveSharedObjectToPackageName = + (cache: Cache) => + async (sharedObject: string): Promise => { + if (cache.hasResolvedPackageName(sharedObject)) { + return cache.getResolvedPackageName(sharedObject); + } + + const packageName = await searchSharedObjectPackage(sharedObject); + + cache.savePackageName(sharedObject, packageName); + + return packageName; + }; + +async function main(): Promise { + const ubuntuMilestone = await getUbuntuMilestone(); + + logger.log(`Detected ubuntu release: "${ubuntuMilestone}"`); + + const cache = await new Cache(ubuntuMilestone).read(); + + const browserVersions = await fetchBrowsersMilestones(); + + logger.log(`Fetched ${browserVersions.length} browser milestones`); + + const browsersToDownload = cache.filterProcessedBrowsers(browserVersions); + + logger.log(`There are ${browsersToDownload.length} browsers to download`); + + const binaryPaths = await downloadBrowserVersions(browsersToDownload); + + logger.log(`There are ${binaryPaths.length} binaries in registry (browsers with drivers)`); + + const downloadedBinarySharedObjectsArrays = await Promise.all(binaryPaths.map(getBinarySharedObjectDependencies)); + const downloadedBinarySharedObjects = _.flatten(downloadedBinarySharedObjectsArrays); + + const extraBinarySharedObjects = cache.getUnresolvedSharedObjects().concat(EXTRA_FIREFOX_SHARED_OBJECTS); + + const uniqSharedObjects = _.uniq(downloadedBinarySharedObjects.concat(extraBinarySharedObjects)); + + logger.log(`There are ${uniqSharedObjects.length} shared objects to resolve`); + + const resolveSharedObjectToPackageName = createResolveSharedObjectToPackageName(cache); + const ubuntuPackages = await Promise.all(uniqSharedObjects.map(resolveSharedObjectToPackageName)); + const uniqUbuntuPackages = _.uniq(ubuntuPackages).filter(Boolean); + + logger.log(`Resolved ${uniqSharedObjects.length} shared objects to ${uniqUbuntuPackages.length} packages`); + + cache.saveProcessedBrowsers(browsersToDownload); + + await cache.write(); + + logger.log("Saved cache to file system"); + + await writeUbuntuPackageDependencies(ubuntuMilestone, uniqUbuntuPackages); + + logger.log(`Saved ubuntu package direct dependencies for Ubuntu@${ubuntuMilestone}`); +} + +main(); diff --git a/src/collect-ubuntu-browser-dependencies/shared-object.ts b/src/collect-ubuntu-browser-dependencies/shared-object.ts new file mode 100644 index 000000000..6feb9cf62 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/shared-object.ts @@ -0,0 +1,34 @@ +import _ from "lodash"; +import calcLevenshtein from "js-levenshtein"; +import { readElf, aptFileSearch } from "./ubuntu"; + +export const searchSharedObjectPackage = async (sharedObject: string): Promise => { + const aptFileResult = await aptFileSearch(sharedObject); + + const packages = aptFileResult.split("\n").filter(Boolean); + + if (packages.includes("libc6")) { + return "libc6"; + } + + const relevantPackageName = _.minBy(packages, packageName => calcLevenshtein(sharedObject, packageName)) as string; + + return relevantPackageName; +}; + +export const getBinarySharedObjectDependencies = async (binaryPath: string): Promise => { + const sharedObjectRegExp = /^\s*\dx\d+\s\(NEEDED\)\s*Shared library: \[(.*)\]/gm; + + const readElfResult = await readElf(binaryPath, { dynamic: true }); + + let regExpResult = sharedObjectRegExp.exec(readElfResult); + const sharedObjectDependencies: string[] = []; + + while (regExpResult && regExpResult[1]) { + sharedObjectDependencies.push(regExpResult[1]); + + regExpResult = sharedObjectRegExp.exec(readElfResult); + } + + return sharedObjectDependencies; +}; diff --git a/src/collect-ubuntu-browser-dependencies/ubuntu/apt-file.ts b/src/collect-ubuntu-browser-dependencies/ubuntu/apt-file.ts new file mode 100644 index 000000000..68636c9db --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/ubuntu/apt-file.ts @@ -0,0 +1,23 @@ +import execa from "execa"; +import { getCliArgs } from "../utils"; +import { throwIfFailed } from "./utils"; +import { ensureUnixBinaryExists } from "../../browser-installer/ubuntu-packages"; + +const APT_FILE_BINARY_NAME = "apt-file"; + +/** + * @summary search in which package a file is included + * @returns name of the library, which can be downloaded via apt + * @link https://manpages.org/apt-file + */ +export const aptFileSearch = async (fileToSearch: string): Promise => { + await ensureUnixBinaryExists(APT_FILE_BINARY_NAME); + + const args = getCliArgs({ "package-only": true }); + + const result = await execa(APT_FILE_BINARY_NAME, ["search", fileToSearch, ...args]); + + throwIfFailed(result); + + return result.stdout; +}; diff --git a/src/collect-ubuntu-browser-dependencies/ubuntu/index.ts b/src/collect-ubuntu-browser-dependencies/ubuntu/index.ts new file mode 100644 index 000000000..2490649ea --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/ubuntu/index.ts @@ -0,0 +1,2 @@ +export * from "./readelf"; +export * from "./apt-file"; diff --git a/src/collect-ubuntu-browser-dependencies/ubuntu/readelf.ts b/src/collect-ubuntu-browser-dependencies/ubuntu/readelf.ts new file mode 100644 index 000000000..6dc244423 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/ubuntu/readelf.ts @@ -0,0 +1,21 @@ +import execa from "execa"; +import { getCliArgs } from "../utils"; +import { throwIfFailed } from "./utils"; +import { ensureUnixBinaryExists } from "../../browser-installer/ubuntu-packages"; + +const BINARY_NAME = "readelf"; +/** + * @summary get information about ELF files + * @link https://manpages.org/readelf + */ +export const readElf = async (filePath: string, opts?: { dynamic?: boolean }): Promise => { + await ensureUnixBinaryExists(BINARY_NAME); + + const args = getCliArgs({ ...opts, wide: true }); + + const result = await execa(BINARY_NAME, [filePath, ...args]); + + throwIfFailed(result); + + return result.stdout; +}; diff --git a/src/collect-ubuntu-browser-dependencies/ubuntu/utils.ts b/src/collect-ubuntu-browser-dependencies/ubuntu/utils.ts new file mode 100644 index 000000000..1bf8ac1c4 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/ubuntu/utils.ts @@ -0,0 +1,9 @@ +import type { ExecaReturnValue } from "execa"; + +export const throwIfFailed = (execaResult: ExecaReturnValue): void => { + const { exitCode, failed, command, stderr } = execaResult; + + if (failed) { + throw new Error(`Command "${command}" failed with exit code "${exitCode}". stderr:\n${stderr}`); + } +}; diff --git a/src/collect-ubuntu-browser-dependencies/utils.ts b/src/collect-ubuntu-browser-dependencies/utils.ts new file mode 100644 index 000000000..ee3985fa1 --- /dev/null +++ b/src/collect-ubuntu-browser-dependencies/utils.ts @@ -0,0 +1,12 @@ +export type BrowserWithVersion = { browserName: string; browserVersion: string }; + +export const getCliArgs = >(flags?: T): string[] => { + if (!flags) { + return []; + } + + const keys = Object.keys(flags).filter(Boolean); + const enabledFlags = keys.filter(key => Boolean(flags[key])); + + return enabledFlags.map(flag => (flag.length === 1 ? `-${flag}` : `--${flag}`)); +}; diff --git a/test/src/browser-installer/chrome/index.ts b/test/src/browser-installer/chrome/index.ts index 4944f884c..c61bf55c8 100644 --- a/test/src/browser-installer/chrome/index.ts +++ b/test/src/browser-installer/chrome/index.ts @@ -16,6 +16,10 @@ describe("browser-installer/chrome", () => { let getPortStub: SinonStub; let waitPortStub: SinonStub; + let isUbuntuStub: SinonStub; + let getUbuntuLinkerEnvStub: SinonStub; + let installUbuntuPackageDependenciesStub: SinonStub; + beforeEach(() => { pipeLogsWithPrefixStub = sandbox.stub(); installChromeStub = sandbox.stub().resolves("/browser/path"); @@ -24,10 +28,19 @@ describe("browser-installer/chrome", () => { getPortStub = sandbox.stub().resolves(12345); waitPortStub = sandbox.stub().resolves(); + isUbuntuStub = sandbox.stub().resolves(false); + getUbuntuLinkerEnvStub = sandbox.stub().resolves({ LD_LINKER_PATH: "foobar" }); + installUbuntuPackageDependenciesStub = sandbox.stub().resolves(); + runChromeDriver = proxyquire("../../../../src/browser-installer/chrome", { "../../dev-server/utils": { pipeLogsWithPrefix: pipeLogsWithPrefixStub }, "./driver": { installChromeDriver: installChromeDriverStub }, "./browser": { installChrome: installChromeStub }, + "../ubuntu-packages": { + isUbuntu: isUbuntuStub, + getUbuntuLinkerEnv: getUbuntuLinkerEnvStub, + installUbuntuPackageDependencies: installUbuntuPackageDependenciesStub, + }, child_process: { spawn: spawnStub }, // eslint-disable-line camelcase "wait-port": waitPortStub, "get-port": getPortStub, @@ -84,4 +97,52 @@ describe("browser-installer/chrome", () => { assert.notCalled(pipeLogsWithPrefixStub); }); + + describe("ubuntu", () => { + it(`should not try to install ubuntu packages if its not ubuntu`, async () => { + isUbuntuStub.resolves(false); + + await runChromeDriver("130"); + + assert.notCalled(installUbuntuPackageDependenciesStub); + }); + + it(`should try to install ubuntu packages if its ubuntu`, async () => { + isUbuntuStub.resolves(true); + + await runChromeDriver("130"); + + assert.calledOnce(installUbuntuPackageDependenciesStub); + }); + + it(`should not set ubuntu linker env variables if its not ubuntu`, async () => { + installChromeDriverStub.resolves("/driver/path"); + getPortStub.resolves(10050); + isUbuntuStub.resolves(false); + + await runChromeDriver("130"); + + assert.notCalled(getUbuntuLinkerEnvStub); + assert.calledOnceWith(spawnStub, sinon.match.string, sinon.match.array, { + windowsHide: true, + detached: false, + }); + }); + + it(`should set ubuntu linker env variables if its ubuntu`, async () => { + isUbuntuStub.resolves(true); + getUbuntuLinkerEnvStub.resolves({ foo: "bar" }); + + await runChromeDriver("130"); + + assert.calledOnceWith(spawnStub, sinon.match.string, sinon.match.array, { + windowsHide: true, + detached: false, + env: { + ...process.env, + foo: "bar", + }, + }); + }); + }); }); diff --git a/test/src/browser-installer/firefox/index.ts b/test/src/browser-installer/firefox/index.ts index 2c457bc97..ce5a71269 100644 --- a/test/src/browser-installer/firefox/index.ts +++ b/test/src/browser-installer/firefox/index.ts @@ -16,6 +16,10 @@ describe("browser-installer/firefox", () => { let getPortStub: SinonStub; let waitPortStub: SinonStub; + let isUbuntuStub: SinonStub; + let getUbuntuLinkerEnvStub: SinonStub; + let installUbuntuPackageDependenciesStub: SinonStub; + beforeEach(() => { pipeLogsWithPrefixStub = sandbox.stub(); installFirefoxStub = sandbox.stub().resolves("/browser/path"); @@ -24,10 +28,19 @@ describe("browser-installer/firefox", () => { getPortStub = sandbox.stub().resolves(12345); waitPortStub = sandbox.stub().resolves(); + isUbuntuStub = sandbox.stub().resolves(false); + getUbuntuLinkerEnvStub = sandbox.stub().resolves({ LD_LINKER_PATH: "foobar" }); + installUbuntuPackageDependenciesStub = sandbox.stub().resolves(); + runGeckoDriver = proxyquire("../../../../src/browser-installer/firefox", { "../../dev-server/utils": { pipeLogsWithPrefix: pipeLogsWithPrefixStub }, "./browser": { installFirefox: installFirefoxStub }, "./driver": { installLatestGeckoDriver: installLatestGeckoDriverStub }, + "../ubuntu-packages": { + isUbuntu: isUbuntuStub, + getUbuntuLinkerEnv: getUbuntuLinkerEnvStub, + installUbuntuPackageDependencies: installUbuntuPackageDependenciesStub, + }, geckodriver: { start: startGeckoDriverStub }, "wait-port": waitPortStub, "get-port": getPortStub, @@ -96,4 +109,60 @@ describe("browser-installer/firefox", () => { assert.notCalled(pipeLogsWithPrefixStub); }); + + describe("ubuntu", () => { + it(`should not try to install ubuntu packages if its not ubuntu`, async () => { + isUbuntuStub.resolves(false); + + await runGeckoDriver("130"); + + assert.notCalled(installUbuntuPackageDependenciesStub); + }); + + it(`should try to install ubuntu packages if its ubuntu`, async () => { + isUbuntuStub.resolves(true); + + await runGeckoDriver("130"); + + assert.calledOnce(installUbuntuPackageDependenciesStub); + }); + + it(`should not set ubuntu linker env variables if its not ubuntu`, async () => { + installLatestGeckoDriverStub.resolves("/driver/path"); + getPortStub.resolves(10050); + isUbuntuStub.resolves(false); + + await runGeckoDriver("130"); + + assert.notCalled(getUbuntuLinkerEnvStub); + assert.calledOnceWith(startGeckoDriverStub, { + customGeckoDriverPath: "/driver/path", + port: 10050, + log: "fatal", + spawnOpts: { + windowsHide: true, + detached: false, + }, + }); + }); + + it(`should set ubuntu linker env variables if its ubuntu`, async () => { + isUbuntuStub.resolves(true); + getUbuntuLinkerEnvStub.resolves({ foo: "bar" }); + + await runGeckoDriver("130"); + + assert.calledOnceWith( + startGeckoDriverStub, + sinon.match({ + spawnOpts: { + env: { + ...process.env, + foo: "bar", + }, + }, + }), + ); + }); + }); }); diff --git a/test/src/browser-installer/install.ts b/test/src/browser-installer/install.ts index cff6f5435..1bc00052b 100644 --- a/test/src/browser-installer/install.ts +++ b/test/src/browser-installer/install.ts @@ -17,6 +17,9 @@ describe("browser-installer/install", () => { let installLatestGeckoDriverStub: SinonStub; let installEdgeDriverStub: SinonStub; + let isUbuntuStub: SinonStub; + let installUbuntuPackageDependenciesStub: SinonStub; + beforeEach(() => { installChromeStub = sandbox.stub(); installChromeDriverStub = sandbox.stub(); @@ -24,10 +27,17 @@ describe("browser-installer/install", () => { installLatestGeckoDriverStub = sandbox.stub(); installEdgeDriverStub = sandbox.stub(); + isUbuntuStub = sandbox.stub().resolves(false); + installUbuntuPackageDependenciesStub = sandbox.stub().resolves(); + const installer = proxyquire("../../../src/browser-installer/install", { "./chrome": { installChrome: installChromeStub, installChromeDriver: installChromeDriverStub }, "./edge": { installEdgeDriver: installEdgeDriverStub }, "./firefox": { installFirefox: installFirefoxStub, installLatestGeckoDriver: installLatestGeckoDriverStub }, + "./ubuntu-packages": { + isUbuntu: isUbuntuStub, + installUbuntuPackageDependencies: installUbuntuPackageDependenciesStub, + }, }); installBrowser = installer.installBrowser; @@ -36,88 +46,128 @@ describe("browser-installer/install", () => { afterEach(() => sandbox.restore()); - [true, false].forEach(force => { - describe(`installBrowser, force: ${force}`, () => { - describe("chrome", () => { - it("should install browser", async () => { - installChromeStub.withArgs("115").resolves("/browser/path"); + describe(`installBrowser`, () => { + [true, false].forEach(force => { + describe(`force: ${force}`, () => { + describe("chrome", () => { + it("should install browser", async () => { + installChromeStub.withArgs("115").resolves("/browser/path"); - const binaryPath = await installBrowser("chrome", "115", { force }); + const binaryPath = await installBrowser("chrome", "115", { force }); - assert.equal(binaryPath, "/browser/path"); - assert.calledOnceWith(installChromeStub, "115", { force }); - assert.notCalled(installChromeDriverStub); - }); + assert.equal(binaryPath, "/browser/path"); + assert.calledOnceWith(installChromeStub, "115", { force }); + assert.notCalled(installChromeDriverStub); + }); - it("should install browser with webdriver", async () => { - installChromeStub.withArgs("115").resolves("/browser/path"); + it("should install browser with webdriver", async () => { + installChromeStub.withArgs("115").resolves("/browser/path"); - const binaryPath = await installBrowser("chrome", "115", { force, installWebDriver: true }); + const binaryPath = await installBrowser("chrome", "115", { + force, + shouldInstallWebDriver: true, + }); - assert.equal(binaryPath, "/browser/path"); - assert.calledOnceWith(installChromeStub, "115", { force }); - assert.calledOnceWith(installChromeDriverStub, "115", { force }); + assert.equal(binaryPath, "/browser/path"); + assert.calledOnceWith(installChromeStub, "115", { force }); + assert.calledOnceWith(installChromeDriverStub, "115", { force }); + }); }); - }); - describe("firefox", () => { - it("should install browser", async () => { - installFirefoxStub.withArgs("115").resolves("/browser/path"); + describe("firefox", () => { + it("should install browser", async () => { + installFirefoxStub.withArgs("115").resolves("/browser/path"); + + const binaryPath = await installBrowser("firefox", "115", { force }); - const binaryPath = await installBrowser("firefox", "115", { force }); + assert.equal(binaryPath, "/browser/path"); + assert.calledOnceWith(installFirefoxStub, "115", { force }); + assert.notCalled(installLatestGeckoDriverStub); + }); - assert.equal(binaryPath, "/browser/path"); - assert.calledOnceWith(installFirefoxStub, "115", { force }); - assert.notCalled(installLatestGeckoDriverStub); + it("should install browser with webdriver", async () => { + installFirefoxStub.withArgs("115").resolves("/browser/path"); + + const binaryPath = await installBrowser("firefox", "115", { + force, + shouldInstallWebDriver: true, + }); + + assert.equal(binaryPath, "/browser/path"); + assert.calledOnceWith(installFirefoxStub, "115", { force }); + assert.calledOnceWith(installLatestGeckoDriverStub, "115", { force }); + }); }); - it("should install browser with webdriver", async () => { - installFirefoxStub.withArgs("115").resolves("/browser/path"); + describe("edge", () => { + it("should return null", async () => { + const binaryPath = await installBrowser("MicrosoftEdge", "115", { force }); - const binaryPath = await installBrowser("firefox", "115", { force, installWebDriver: true }); + assert.equal(binaryPath, null); + assert.notCalled(installEdgeDriverStub); + }); - assert.equal(binaryPath, "/browser/path"); - assert.calledOnceWith(installFirefoxStub, "115", { force }); - assert.calledOnceWith(installLatestGeckoDriverStub, "115", { force }); + it("should install webdriver", async () => { + const binaryPath = await installBrowser("MicrosoftEdge", "115", { + force, + shouldInstallWebDriver: true, + }); + + assert.equal(binaryPath, null); + assert.calledOnceWith(installEdgeDriverStub, "115", { force }); + }); }); - }); - describe("edge", () => { - it("should return null", async () => { - const binaryPath = await installBrowser("MicrosoftEdge", "115", { force }); + describe("safari", () => { + it("should return null", async () => { + const binaryPath = await installBrowser("safari", "115", { + force, + shouldInstallWebDriver: true, + }); - assert.equal(binaryPath, null); - assert.notCalled(installEdgeDriverStub); + assert.equal(binaryPath, null); + }); }); - it("should install webdriver", async () => { - const binaryPath = await installBrowser("MicrosoftEdge", "115", { force, installWebDriver: true }); + it("should throw exception on unsupported browser name", async () => { + await assert.isRejected( + installBrowser("foobar", "115", { force }), + /Couldn't install browser 'foobar', as it is not supported/, + ); + }); - assert.equal(binaryPath, null); - assert.calledOnceWith(installEdgeDriverStub, "115", { force }); + it("should throw exception on empty browser version", async () => { + await assert.isRejected( + installBrowser("chrome", "", { force }), + /Couldn't install browser 'chrome' because it has invalid version: ''/, + ); }); }); + }); - describe("safari", () => { - it("should return null", async () => { - const binaryPath = await installBrowser("safari", "115", { force, installWebDriver: true }); + ["chrome", "firefox"].forEach(browser => { + it(`should not install ubuntu dependencies if flag is unset for ${browser}`, async () => { + isUbuntuStub.resolves(true); - assert.equal(binaryPath, null); - }); + await installBrowser(browser, "115", { shouldInstallUbuntuPackages: false }); + + assert.notCalled(installUbuntuPackageDependenciesStub); }); - it("should throw exception on unsupported browser name", async () => { - await assert.isRejected( - installBrowser("foobar", "115", { force }), - /Couldn't install browser 'foobar', as it is not supported/, - ); + it(`should not install ubuntu dependencies if its not ubuntu for ${browser}`, async () => { + isUbuntuStub.resolves(false); + + await installBrowser(browser, "115"); + + assert.notCalled(installUbuntuPackageDependenciesStub); }); - it("should throw exception on empty browser version", async () => { - await assert.isRejected( - installBrowser("chrome", "", { force }), - /Couldn't install browser 'chrome' because it has invalid version: ''/, - ); + it(`should install ubuntu dependencies by default if its ubuntu for ${browser}`, async () => { + isUbuntuStub.resolves(true); + + await installBrowser(browser, "115"); + + assert.calledOnce(installUbuntuPackageDependenciesStub); }); }); }); diff --git a/test/src/browser-installer/registry.ts b/test/src/browser-installer/registry.ts index db900d5c3..1e6d2ada3 100644 --- a/test/src/browser-installer/registry.ts +++ b/test/src/browser-installer/registry.ts @@ -9,7 +9,7 @@ describe("browser-installer/registry", () => { let registry: typeof Registry; - let readJsonSyncStub: SinonStub; + let readJSONSyncStub: SinonStub; let outputJSONSyncStub: SinonStub; let existsSyncStub: SinonStub; let progressBarRegisterStub: SinonStub; @@ -18,13 +18,13 @@ describe("browser-installer/registry", () => { const createRegistry_ = (contents: Record> = {}): typeof Registry => { return proxyquire("../../../src/browser-installer/registry", { "../utils": { getRegistryPath: () => "/testplane/registry/registry.json" }, - "fs-extra": { readJsonSync: () => contents, existsSync: () => true }, + "fs-extra": { readJSONSync: () => contents, existsSync: () => true }, "../../utils/logger": { warn: loggerWarnStub }, }); }; beforeEach(() => { - readJsonSyncStub = sandbox.stub().returns({}); + readJSONSyncStub = sandbox.stub().returns({}); outputJSONSyncStub = sandbox.stub(); existsSyncStub = sandbox.stub().returns(false); progressBarRegisterStub = sandbox.stub(); @@ -35,7 +35,7 @@ describe("browser-installer/registry", () => { "../utils": { getRegistryPath: () => "/testplane/registry/registry.json" }, "../../utils/logger": { warn: loggerWarnStub }, "fs-extra": { - readJsonSync: readJsonSyncStub, + readJSONSync: readJSONSyncStub, outputJSONSync: outputJSONSyncStub, existsSync: existsSyncStub, }, diff --git a/test/src/browser-installer/ubuntu-packages.ts b/test/src/browser-installer/ubuntu-packages.ts new file mode 100644 index 000000000..543088a70 --- /dev/null +++ b/test/src/browser-installer/ubuntu-packages.ts @@ -0,0 +1,145 @@ +import proxyquire from "proxyquire"; +import sinon, { type SinonStub } from "sinon"; +import type { + writeUbuntuPackageDependencies as WriteUbuntuPackageDependencies, + installUbuntuPackageDependencies as InstallUbuntuPackageDependencies, + getUbuntuLinkerEnv as GetUbuntuLinkerEnv, +} from "../../../src/browser-installer/ubuntu-packages"; + +describe("browser-installer/ubuntu-packages", () => { + const sandbox = sinon.createSandbox(); + + let writeUbuntuPackageDependencies: typeof WriteUbuntuPackageDependencies; + let installUbuntuPackageDependencies: typeof InstallUbuntuPackageDependencies; + let getUbuntuLinkerEnv: typeof GetUbuntuLinkerEnv; + + let fsStub: Record; + let loggerLogStub: SinonStub; + let installUbuntuPackagesStub: SinonStub; + let getUbuntuMilestoneStub: SinonStub; + + beforeEach(() => { + fsStub = { + readJSON: sinon.stub().resolves({}), + existsSync: sinon.stub().returns(false), + readdir: sinon.stub().resolves([]), + stat: sinon.stub().resolves({ isDirectory: () => true }), + outputJSON: sinon.stub().resolves({}), + } as Record; + + loggerLogStub = sandbox.stub(); + installUbuntuPackagesStub = sandbox.stub(); + getUbuntuMilestoneStub = sandbox.stub().resolves("20"); + + const ubuntuPackages = proxyquire("../../../src/browser-installer/ubuntu-packages", { + "fs-extra": fsStub, + "./apt": { installUbuntuPackages: installUbuntuPackagesStub }, + "./utils": { getUbuntuMilestone: getUbuntuMilestoneStub }, + "../../utils/logger": { log: loggerLogStub }, + }); + + ({ writeUbuntuPackageDependencies, installUbuntuPackageDependencies, getUbuntuLinkerEnv } = ubuntuPackages); + }); + + afterEach(() => sandbox.restore()); + + describe("writeUbuntuPackageDependencies", () => { + it("should write sorted dependencies if file does not exist", async () => { + getUbuntuMilestoneStub.resolves("20"); + fsStub.readJSON.withArgs(sinon.match("ubuntu-20-dependencies.json")).rejects(new Error("No such file")); + + await writeUbuntuPackageDependencies("20", ["b", "a", "c"]); + + assert.calledOnceWith(fsStub.outputJSON, sinon.match.string, ["a", "b", "c"]); + }); + + it("should write uniq sorted dependencies with existing deps from file", async () => { + fsStub.readJSON.resolves(["a", "b", "d"]); + + await writeUbuntuPackageDependencies("20", ["e", "c", "d"]); + + assert.calledOnceWith(fsStub.outputJSON, sinon.match.string, ["a", "b", "c", "d", "e"]); + }); + }); + + describe("installUbuntuPackageDependencies", () => { + it("should install deps for current milestone", async () => { + getUbuntuMilestoneStub.resolves("20"); + fsStub.existsSync.withArgs(sinon.match("packages")).returns(false); + fsStub.readJSON.withArgs(sinon.match("ubuntu-20-dependencies.json")).resolves(["foo", "bar"]); + + await installUbuntuPackageDependencies(); + + assert.calledOnceWith(loggerLogStub, "Downloading extra deb packages to local browsers execution..."); + assert.calledOnceWith(installUbuntuPackagesStub, ["foo", "bar"], sinon.match("packages")); + }); + + it("should read dependencies and install packages only once per multiple function calls", async () => { + getUbuntuMilestoneStub.resolves("20"); + fsStub.existsSync.withArgs(sinon.match("packages")).returns(false); + fsStub.readJSON.withArgs(sinon.match("ubuntu-20-dependencies.json")).resolves(["foo", "bar"]); + + const promise1 = await installUbuntuPackageDependencies(); + const promise2 = await installUbuntuPackageDependencies(); + + assert.equal(promise1, promise2); + assert.calledOnce(fsStub.readJSON); + assert.calledOnceWith(installUbuntuPackagesStub, ["foo", "bar"], sinon.match("packages")); + }); + + it("should skip installation if directory with packages exists", async () => { + getUbuntuMilestoneStub.resolves("20"); + fsStub.existsSync.withArgs(sinon.match("packages")).returns(true); + + await installUbuntuPackageDependencies(); + + assert.notCalled(fsStub.readJSON.withArgs(sinon.match("ubuntu-20-dependencies.json"))); + assert.notCalled(loggerLogStub); + assert.notCalled(installUbuntuPackagesStub); + }); + }); + + describe("getUbuntuLinkerEnv", () => { + beforeEach(() => { + fsStub.existsSync.withArgs(sinon.match("packages")).returns(true); + fsStub.readdir.withArgs(sinon.match("/lib")).resolves(["foo", "bar"]); + fsStub.readdir.withArgs(sinon.match("/usr/lib")).resolves(["baz", "qux"]); + fsStub.stat.resolves({ isDirectory: () => true }); + }); + + it("should resolve ubuntu linker env", async () => { + const env = await getUbuntuLinkerEnv(); + + assert.match(env.LD_LIBRARY_PATH, "/packages/lib/foo"); + assert.match(env.LD_LIBRARY_PATH, "/packages/lib/bar"); + assert.match(env.LD_LIBRARY_PATH, "/packages/usr/lib/baz"); + assert.match(env.LD_LIBRARY_PATH, "/packages/usr/lib/qux"); + }); + + it("should concat existing LD_LIBRARY_PATH", async () => { + const envBack = process.env.LD_LIBRARY_PATH; + process.env.LD_LIBRARY_PATH = "foo/bar/baz"; + + const env = await getUbuntuLinkerEnv(); + + process.env.LD_LIBRARY_PATH = envBack; + assert.match(env.LD_LIBRARY_PATH, "foo/bar/baz"); + }); + + it("should cache env value", async () => { + await getUbuntuLinkerEnv(); + + const existsSyncCallCount = fsStub.existsSync.callCount; + const readDirCallCount = fsStub.readdir.callCount; + const statCallCount = fsStub.stat.callCount; + + await getUbuntuLinkerEnv(); + await getUbuntuLinkerEnv(); + await getUbuntuLinkerEnv(); + + assert.callCount(fsStub.existsSync, existsSyncCallCount); + assert.callCount(fsStub.readdir, readDirCallCount); + assert.callCount(fsStub.stat, statCallCount); + }); + }); +}); diff --git a/test/src/collect-ubuntu-browser-dependencies/cache.ts b/test/src/collect-ubuntu-browser-dependencies/cache.ts new file mode 100644 index 000000000..af9cb2da4 --- /dev/null +++ b/test/src/collect-ubuntu-browser-dependencies/cache.ts @@ -0,0 +1,95 @@ +import proxyquire from "proxyquire"; +import sinon, { type SinonStub } from "sinon"; +import type { Cache as CacheType, CacheData } from "../../../src/collect-ubuntu-browser-dependencies/cache"; + +describe("collect-ubuntu-browser-dependencies/shared-object", () => { + const sandbox = sinon.createSandbox(); + + let cache: CacheType; + + let fsStub: Record; + + const setCache_ = async (data: CacheData): Promise => { + fsStub.readJSON.withArgs(sinon.match("processed-browsers-linux.json")).resolves(data.processedBrowsers); + fsStub.readJSON.withArgs(sinon.match("ubuntu-os-version.json")).resolves(data.sharedObjectsMap); + + await cache.read(); + }; + + const getCache_ = async (): Promise => { + await cache.write(); + + const processedBrowsersCache = fsStub.outputJSON.withArgs(sinon.match("processed-browsers-linux.json")).args; + const sharedObjectsMapPath = fsStub.outputJSON.withArgs(sinon.match("ubuntu-os-version.json")).args; + + const result: CacheData = { + sharedObjectsMap: sharedObjectsMapPath[sharedObjectsMapPath.length - 1][1], + processedBrowsers: processedBrowsersCache[processedBrowsersCache.length - 1][1], + }; + + return result; + }; + + beforeEach(() => { + fsStub = { + readJSON: sinon.stub().resolves({}), + outputJSON: sinon.stub().resolves({}), + existsSync: sinon.stub().returns(false), + readdir: sinon.stub().resolves([]), + stat: sinon.stub().resolves({ isDirectory: false }), + } as Record; + + const Cache = proxyquire("../../../src/collect-ubuntu-browser-dependencies/cache", { + "fs-extra": fsStub, + }).Cache; + + cache = new Cache("os-version"); + }); + + afterEach(() => sandbox.restore()); + + it("should filter processed browsers", async () => { + await setCache_({ + processedBrowsers: { downloadedBrowsers: { chrome: ["80"] }, sharedObjects: ["libc.so.6"] }, + sharedObjectsMap: {}, + }); + + const filteredBrowsers = cache.filterProcessedBrowsers([ + { browserName: "chrome", browserVersion: "80.0.123.17" }, + { browserName: "chrome", browserVersion: "82.0.123.17" }, + ]); + + assert.deepEqual(filteredBrowsers, [{ browserName: "chrome", browserVersion: "82.0.123.17" }]); + }); + + it("should save processed browsers", async () => { + cache.saveProcessedBrowsers([ + { browserName: "chrome", browserVersion: "80.0.123.17" }, + { browserName: "chrome", browserVersion: "82.0.123.17" }, + ]); + + const cacheData = await getCache_(); + + assert.deepEqual(cacheData.processedBrowsers.downloadedBrowsers, { chrome: ["80", "82"] }); + }); + + it("should save resolved shared objects", async () => { + cache.savePackageName("libc.so.6", "libc6"); + + const cacheData = await getCache_(); + + assert.deepEqual(cacheData.processedBrowsers.sharedObjects, ["libc.so.6"]); + assert.deepEqual(cacheData.sharedObjectsMap, { "libc.so.6": "libc6" }); + }); + + it("should get unresolved shared objects", async () => { + await setCache_({ + processedBrowsers: { downloadedBrowsers: { chrome: ["80"] }, sharedObjects: ["libc.so.6", "libnss3.so"] }, + sharedObjectsMap: { "libc.so.6": "libc6" }, + }); + + const unresolvedSharedObjects = cache.getUnresolvedSharedObjects(); + + assert.deepEqual(unresolvedSharedObjects, ["libnss3.so"]); + }); +}); diff --git a/test/src/collect-ubuntu-browser-dependencies/shared-object.ts b/test/src/collect-ubuntu-browser-dependencies/shared-object.ts new file mode 100644 index 000000000..a6431afa3 --- /dev/null +++ b/test/src/collect-ubuntu-browser-dependencies/shared-object.ts @@ -0,0 +1,81 @@ +import proxyquire from "proxyquire"; +import sinon, { type SinonStub } from "sinon"; +import type { + searchSharedObjectPackage as SearchSharedObjectPackage, + getBinarySharedObjectDependencies as GetBinarySharedObjectDependencies, +} from "../../../src/collect-ubuntu-browser-dependencies/shared-object"; + +describe("collect-ubuntu-browser-dependencies/shared-object", () => { + const sandbox = sinon.createSandbox(); + + let searchSharedObjectPackage: typeof SearchSharedObjectPackage; + let getBinarySharedObjectDependencies: typeof GetBinarySharedObjectDependencies; + + let readElfStub: SinonStub; + let aptFileSearchStub: SinonStub; + + beforeEach(() => { + readElfStub = sandbox.stub().resolves(); + aptFileSearchStub = sandbox.stub().resolves(); + + const sharedObject = proxyquire("../../../src/collect-ubuntu-browser-dependencies/shared-object", { + "./ubuntu": { + readElf: readElfStub, + aptFileSearch: aptFileSearchStub, + }, + }); + + ({ searchSharedObjectPackage, getBinarySharedObjectDependencies } = sharedObject); + }); + + afterEach(() => sandbox.restore()); + + describe("searchSharedObjectPackage", () => { + it("should return package name closest to shared object name", async () => { + aptFileSearchStub.withArgs("libnss3.so").resolves(`firefox\nlibnss3\n`); + + const packageName = await searchSharedObjectPackage("libnss3.so"); + + assert.equal(packageName, "libnss3"); + }); + }); + + describe("getBinarySharedObjectDependencies", () => { + it("should return binary direct shared object deps", async () => { + readElfStub.resolves(` +Dynamic section at offset 0xb00 contains 26 entries: + Tag Type Name/Value + 0x0000000000000001 (NEEDED) Shared library: [libpthread.so.0] + 0x0000000000000001 (NEEDED) Shared library: [libdl.so.2] + 0x0000000000000001 (NEEDED) Shared library: [libc.so.6] + 0x0000000000000015 (DEBUG) 0x0 + 0x0000000000000007 (RELA) 0x2005b0 + 0x0000000000000008 (RELASZ) 48 (bytes) + 0x0000000000000009 (RELAENT) 24 (bytes) + 0x0000000000000017 (JMPREL) 0x2005e0 + 0x0000000000000002 (PLTRELSZ) 192 (bytes) + 0x0000000000000003 (PLTGOT) 0x203cc0 + 0x0000000000000014 (PLTREL) RELA + 0x0000000000000006 (SYMTAB) 0x200308 + 0x000000000000000b (SYMENT) 24 (bytes) + 0x0000000000000005 (STRTAB) 0x2004c0 + 0x000000000000000a (STRSZ) 238 (bytes) + 0x000000006ffffef5 (GNU_HASH) 0x2004a0 + 0x0000000000000019 (INIT_ARRAY) 0x202af8 + 0x000000000000001b (INIT_ARRAYSZ) 8 (bytes) + 0x000000000000001a (FINI_ARRAY) 0x202af0 + 0x000000000000001c (FINI_ARRAYSZ) 8 (bytes) + 0x000000000000000c (INIT) 0x201a34 + 0x000000000000000d (FINI) 0x201a50 + 0x000000006ffffff0 (VERSYM) 0x200428 + 0x000000006ffffffe (VERNEED) 0x200440 + 0x000000006fffffff (VERNEEDNUM) 2 + 0x0000000000000000 (NULL) 0x0 + `); + + const deps = await getBinarySharedObjectDependencies("binary/path"); + + assert.deepEqual(deps, ["libpthread.so.0", "libdl.so.2", "libc.so.6"]); + }); + }); +}); diff --git a/test/src/collect-ubuntu-browser-dependencies/utils.ts b/test/src/collect-ubuntu-browser-dependencies/utils.ts new file mode 100644 index 000000000..74d0947bc --- /dev/null +++ b/test/src/collect-ubuntu-browser-dependencies/utils.ts @@ -0,0 +1,17 @@ +import { getCliArgs } from "../../../src/collect-ubuntu-browser-dependencies/utils"; + +describe("collect-ubuntu-browser-dependencies/utils", () => { + describe("getCliArgs", () => { + it("should support long cli keys", () => { + assert.deepEqual(getCliArgs({ foo: true }), ["--foo"]); + }); + + it("should support short cli keys", () => { + assert.deepEqual(getCliArgs({ f: true }), ["-f"]); + }); + + it("should not return disabled cli keys", () => { + assert.deepEqual(getCliArgs({ f: false, foo: false }), []); + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index ebbbcd72b..32146370b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,7 @@ "src/**/*.test.ts", "src/browser/client-scripts", "src/bundle/cjs", + "src/collect-ubuntu-browser-dependencies", "src/runner/browser-env/vite/browser-modules" ], "compilerOptions": {