diff --git a/.github/workflows/all-tests.yml b/.github/workflows/all-tests.yml index 16a4c0b096..3791f8d6d6 100644 --- a/.github/workflows/all-tests.yml +++ b/.github/workflows/all-tests.yml @@ -61,10 +61,14 @@ env: PR_NUMBER: ${{ github.event.number }} SHA: ${{ github.event.pull_request.head.sha }} SYSTEM_INFO: ./tests/pw/playwright/systemInfo.json - API_TEST_RESULT: ./tests/pw/playwright-report/api/summary-report/results.json - E2E_TEST_RESULT: ./tests/pw/playwright-report/e2e/summary-report/results.json - API_COVERAGE: ./tests/pw/playwright-report/api/coverage-report/coverage.json - E2E_COVERAGE: ./tests/pw/playwright-report/e2e/coverage-report/coverage.json + API_TEST_RESULT: ./tests/pw/all-reports/test-artifact-api/playwright-report/api/summary-report/results.json + API_COVERAGE: ./tests/pw/all-reports/test-artifact-api/playwright-report/api/coverage-report/coverage.json + E2E_TEST_RESULT: ./tests/pw/all-reports/merged-summary.json + E2E_COVERAGE: ./tests/pw/all-reports/merged-coverage.json + # API_TEST_RESULT: ./tests/pw/playwright-report/api/summary-report/results.json + # E2E_TEST_RESULT: ./tests/pw/playwright-report/e2e/summary-report/results.json + # API_COVERAGE: ./tests/pw/playwright-report/api/coverage-report/coverage.json + # E2E_COVERAGE: ./tests/pw/playwright-report/e2e/coverage-report/coverage.json TEST_PUBLISH_KEY_STRIPE: ${{secrets.TEST_PUBLISH_KEY_STRIPE}} TEST_SECRET_KEY_STRIPE: ${{secrets.TEST_SECRET_KEY_STRIPE}} CLIENT_ID_STRIPE: ${{secrets.CLIENT_ID_STRIPE}} @@ -538,34 +542,6 @@ jobs: with: path: tests/pw/all-reports pattern: test-artifact-* - # merge-multiple: true - - # - name: Debug test-artifact-1 structure - # working-directory: tests/pw/all-reports - # run: | - # echo "Contents of test-artifact-1:" - # ls -R test-artifact-1 - - # - name: Check blob reports - # working-directory: tests/pw/all-reports - # run: | - # echo "Listing all directories and files:" - # ls -R - # echo "Navigating to blob-report:" - # cd test-artifact-1/playwright-report/e2e/blob-report - # ls - - # - name: Move all blob report folder contents into a single folder - # working-directory: tests/pw/all-reports - # run: | - # mkdir -p all-blob-reports - # echo "Finding all blob-report directories and moving contents..." - # find . -type d -name "blob-report" | while read -r dir; do - # echo "Processing $dir" - # mv "$dir"/* all-blob-reports/ || echo "No files in $dir" - # done - # echo "All blob report contents moved to all-blob-reports:" - # ls all-blob-reports - name: Move all blob report folder contents into a single folder working-directory: tests/pw/all-reports @@ -582,64 +558,68 @@ jobs: npx playwright merge-reports --reporter html ./all-reports/all-blob-reports mv playwright-report ./all-reports/html-report - - name: Generate summary report + - name: Generate merged summary report + id: summary-report working-directory: tests/pw run: | - node ./utils/mergeSummaryReport.js + npx ts-node ./utils/mergeSummaryReport.ts + + - name: Generate merged coverage report + id: summary-report + working-directory: tests/pw + run: | + npx ts-node ./utils/mergeCoverageSummary.ts + + # Prepare test summary + - name: Prepare test summary + id: prepare-test-summary + if: always() && steps.summary-report.outcome == 'success' + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + const script = require("./tests/pw/utils/gitTestSummary.ts") + return await script({github, context, core}) + + # Find PR comment + - name: Find PR comment by github-actions[bot] + uses: peter-evans/find-comment@v3 + id: find-comment + if: github.event_name == 'pull_request' + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Tests Summary + # Post test summary as PR comment + - name: Create or update PR comment + uses: peter-evans/create-or-update-comment@v4 + if: github.event_name == 'pull_request' + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: ${{ steps.prepare-test-summary.outputs.result }} + edit-mode: replace - name: Upload final test artifact uses: actions/upload-artifact@v4 with: name: final-test-artifact path: tests/pw/all-reports - retention-days: - 7 - - # - name: Publish HTML report to GH Pages - # if: always() # We want this to always run, even if test fail prior to this step running - # uses: peaceiris/actions-gh-pages@v4 - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # publish_branch: gh-pages - # publish_dir: tests/pw/playwright-report/e2e/html-report # This is where index.html will be output - # keep_files: true # Set this to true to make sure we don't wipe out the other content in GitHub Pages - # user_name: "github-actions[bot]" # This will be the username that gets stamped in the repo commit - # user_email: "github-actions[bot]@users.noreply.github.com" # This will be the user email that gets stamped in the repo commit - - # - name: Test results - # if: always() - # run: | - # echo "https://$GITHUB_REPOSITORY_OWNER.github.io/${GITHUB_REPOSITORY#*/}" -# publishTestResults: -# name: Publish Test Result -# needs: tests -# runs-on: ubuntu-latest -# if: always() -# steps: -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# id: download - -# - name: Publish HTML report to GH Pages -# if: always() # We want this to always run, even if test fail prior to this step running -# uses: peaceiris/actions-gh-pages@v3 -# with: -# github_token: ${{ secrets.GITHUB_TOKEN }} -# publish_dir: ${{steps.download.outputs.download-path}} # This is where index.html will be output -# # keep_files: true # Set this to true to make sure we don't wipe out the other content in GitHub Pages -# user_name: "github-actions[bot]" # This will be the username that gets stamped in the repo commit -# user_email: "github-actions[bot]@users.noreply.github.com" # This will be the user email that gets stamped in the repo commit - -# # redundant / in echo url remove before run -# - name: Test results -# run: | -# echo "url is https://$GITHUB_REPOSITORY_OWNER.github.io/${GITHUB_REPOSITORY#/*/}" -# iff: github.event_name == 'pull_request' - -# - name: Log github context & env variables -# env: -# GITHUB_CONTEXT: ${{ toJson(github) }} -# run: | -# echo "$GITHUB_CONTEXT" -# env + retention-days: 7 + + # - name: Publish HTML report to GH Pages + # if: always() # We want this to always run, even if test fail prior to this step running + # uses: peaceiris/actions-gh-pages@v4 + # with: + # github_token: ${{ secrets.GITHUB_TOKEN }} + # publish_branch: gh-pages + # publish_dir: tests/pw/playwright-report/e2e/html-report # This is where index.html will be output + # keep_files: true # Set this to true to make sure we don't wipe out the other content in GitHub Pages + # user_name: 'github-actions[bot]' # This will be the username that gets stamped in the repo commit + # user_email: 'github-actions[bot]@users.noreply.github.com' # This will be the user email that gets stamped in the repo com + + # - name: Test results + # if: always() + # run: | + # echo "https://$GITHUB_REPOSITORY_OWNER.github.io/${GITHUB_REPOSITORY#*/}" diff --git a/tests/pw/package-lock.json b/tests/pw/package-lock.json index 0983b847ef..39d5c7f0ff 100644 --- a/tests/pw/package-lock.json +++ b/tests/pw/package-lock.json @@ -28,6 +28,7 @@ "js-yaml": "^4.1.0", "npm-check-updates": "^17.1.11", "prettier": "^3.4.2", + "ts-node": "^10.9.2", "tslib": "^2.8.1", "typescript": "^5.7.2" } @@ -41,6 +42,18 @@ "node": ">=0.10.0" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -232,6 +245,31 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@kwsites/file-exists": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", @@ -316,6 +354,30 @@ "node": ">=10" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -667,6 +729,18 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -730,6 +804,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -960,6 +1040,12 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -1048,6 +1134,15 @@ "node": ">=0.10" } }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/docker-compose": { "version": "0.24.3", "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-0.24.3.tgz", @@ -1959,6 +2054,12 @@ "url": "https://github.com/sponsors/wellwelwel" } }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -2747,6 +2848,49 @@ "typescript": ">=4.2.0" } }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -2814,6 +2958,12 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, "node_modules/wcwidth": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", @@ -2892,6 +3042,15 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -2920,6 +3079,15 @@ "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", "dev": true }, + "@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "requires": { + "@jridgewell/trace-mapping": "0.3.9" + } + }, "@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -3042,6 +3210,28 @@ "integrity": "sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==", "dev": true }, + "@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "@kwsites/file-exists": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", @@ -3102,6 +3292,30 @@ "defer-to-connect": "^2.0.0" } }, + "@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true + }, + "@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true + }, "@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -3335,6 +3549,15 @@ "dev": true, "requires": {} }, + "acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "requires": { + "acorn": "^8.11.0" + } + }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -3375,6 +3598,12 @@ "color-convert": "^2.0.1" } }, + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, "argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -3550,6 +3779,12 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, + "create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, "cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3608,6 +3843,12 @@ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true + }, "docker-compose": { "version": "0.24.3", "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-0.24.3.tgz", @@ -4282,6 +4523,12 @@ "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.1.tgz", "integrity": "sha512-FbAj6lXil6t8z4z3j0E5mfRlPzxkySotzUHwRXjlpRh10vc6AI6WN62ehZj82VG7M20rqogJ0GLwar2Xa05a8Q==" }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -4828,6 +5075,27 @@ "dev": true, "requires": {} }, + "ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "requires": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + } + }, "tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -4879,6 +5147,12 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, "wcwidth": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", @@ -4939,6 +5213,12 @@ "fd-slicer": "~1.1.0" } }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true + }, "yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/tests/pw/package.json b/tests/pw/package.json index d410a1ff64..333bdd1b0e 100644 --- a/tests/pw/package.json +++ b/tests/pw/package.json @@ -59,6 +59,7 @@ "js-yaml": "^4.1.0", "npm-check-updates": "^17.1.11", "prettier": "^3.4.2", + "ts-node": "^10.9.2", "tslib": "^2.8.1", "typescript": "^5.7.2" }, diff --git a/tests/pw/utils/mergeCoverageSummary.ts b/tests/pw/utils/mergeCoverageSummary.ts new file mode 100644 index 0000000000..589bc9aedc --- /dev/null +++ b/tests/pw/utils/mergeCoverageSummary.ts @@ -0,0 +1,103 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +interface CoverageReport { + total_features: number; + total_covered_features: number; + coverage: string; + page_coverage: Record; + covered_features: string[]; + uncovered_features: string[]; +} + +// Helper function to merge page coverage +const mergePageCoverage = (existing: Record, newCoverage: Record): Record => { + const merged: Record = { ...existing }; + for (const page in newCoverage) { + if (newCoverage[page] !== undefined) { + merged[page] = (merged[page] ?? 0) + newCoverage[page]; + } + } + return merged; +}; + +// Main function to merge coverage files +const mergeCoverageReports = (reportPaths: string[]): CoverageReport => { + const mergedReport: CoverageReport = { + total_features: 0, + total_covered_features: 0, + coverage: '0', + page_coverage: {}, + covered_features: [], + uncovered_features: [], + }; + + reportPaths.forEach(reportPath => { + const report: CoverageReport = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + + // Add total features and covered features + mergedReport.total_features += report.total_features; + mergedReport.total_covered_features += report.total_covered_features; + + // Add coverage percentages (convert string to number, remove "%") + mergedReport.coverage = (parseFloat(mergedReport.coverage.replace('%', '')) + parseFloat(report.coverage.replace('%', ''))).toFixed(2) + '%'; + + // Merge page coverage + mergedReport.page_coverage = mergePageCoverage(mergedReport.page_coverage, report.page_coverage); + + // Append features (deduplication and sorting will be handled separately) + mergedReport.covered_features.push(...report.covered_features); + mergedReport.uncovered_features.push(...report.uncovered_features); + }); + + // Deduplicate and sort features after all reports are merged + mergedReport.covered_features = [...new Set(mergedReport.covered_features)].sort(); + mergedReport.uncovered_features = [...new Set(mergedReport.uncovered_features)].sort(); + + // Recalculate overall coverage percentage + mergedReport.coverage = ((mergedReport.total_covered_features / mergedReport.total_features) * 100).toFixed(2) + '%'; + + for (const page in mergedReport.page_coverage) { + if (mergedReport.page_coverage[page] !== undefined) { + mergedReport.page_coverage[page] = parseFloat((mergedReport.page_coverage[page]! / reportPaths.length).toFixed(2)); + } + } + + return mergedReport; +}; + +// Main script execution +const reportsFolder = './all-reports'; // Update to your artifacts folder location +const reportPaths: string[] = []; + +// Collect all coverage.json files +const findReports = (dir: string): void => { + const files = fs.readdirSync(dir); + files.forEach(file => { + const fullPath = path.join(dir, file); + if (fs.statSync(fullPath).isDirectory()) { + if (!fullPath.includes(path.join('api'))) { // todo: update if api suite is also run in matrix job + // Ignore directories containing 'api' + findReports(fullPath); // Recurse into subdirectories + } + } else if (file === 'coverage.json' && !fullPath.includes(path.join('api'))) { + reportPaths.push(fullPath); + } + }); +}; + +findReports(reportsFolder); + +if (reportPaths.length === 0) { + console.error('No coverage.json files found in artifacts.'); + process.exit(1); +} + +// Merge reports +const mergedCoverage = mergeCoverageReports(reportPaths); + +// Save the merged coverage report +const outputPath = './all-reports/merged-coverage.json'; +fs.writeFileSync(outputPath, JSON.stringify(mergedCoverage, null, 2), 'utf8'); + +console.log(`Merged coverage report saved to ${outputPath}`); diff --git a/tests/pw/utils/mergeSummaryReport.js b/tests/pw/utils/mergeSummaryReport.ts similarity index 63% rename from tests/pw/utils/mergeSummaryReport.js rename to tests/pw/utils/mergeSummaryReport.ts index e84b718012..ad3a03d464 100644 --- a/tests/pw/utils/mergeSummaryReport.js +++ b/tests/pw/utils/mergeSummaryReport.ts @@ -1,16 +1,33 @@ -const fs = require('fs'); -const path = require('path'); +import * as fs from 'fs'; +import * as path from 'path'; + +interface TestReport { + suite_name: string; + total_tests: number; + passed: number; + failed: number; + flaky: number; + skipped: number; + suite_duration: number; + suite_duration_formatted?: string; + tests: string[]; + passed_tests: string[]; + failed_tests: string[]; + flaky_tests: string[]; + skipped_tests: string[]; +} // Helper function to format duration -const getFormattedDuration = milliseconds => { +const getFormattedDuration = (milliseconds: number): string => { const hours = Math.floor(milliseconds / (1000 * 60 * 60)); - const min = Math.floor((milliseconds / (1000 * 60)) % 60); - const sec = Math.floor((milliseconds / 1000) % 60); - return `${hours < 1 ? '' : hours + 'h '}${min < 1 ? '' : min + 'm '}${sec < 1 ? '' : sec + 's'}`; + const minutes = Math.floor((milliseconds / (1000 * 60)) % 60); + const seconds = Math.floor((milliseconds / 1000) % 60); + return `${hours < 1 ? '' : hours + 'h '}${minutes < 1 ? '' : minutes + 'm '}${seconds < 1 ? '' : seconds + 's'}`; }; -const mergeReports = reportPaths => { - const mergedReport = { +// Main function to merge reports +const mergeReports = (reportPaths: string[]): TestReport => { + const mergedReport: TestReport = { suite_name: '', total_tests: 0, passed: 0, @@ -27,7 +44,8 @@ const mergeReports = reportPaths => { }; reportPaths.forEach(reportPath => { - const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + const report: TestReport = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + mergedReport.total_tests += report.total_tests; mergedReport.passed += report.passed; mergedReport.failed += report.failed; @@ -58,16 +76,19 @@ const mergeReports = reportPaths => { // Main script execution const reportsFolder = './all-reports'; // Change to your artifacts location -const reportPaths = []; +const reportPaths: string[] = []; -// Collect all result.json files -const findReports = dir => { +// Function to collect all results.json files +const findReports = (dir: string): void => { const files = fs.readdirSync(dir); files.forEach(file => { const fullPath = path.join(dir, file); if (fs.statSync(fullPath).isDirectory()) { - findReports(fullPath); // Recurse into subdirectories - } else if (file === 'results.json') { + if (!fullPath.includes(path.join('api'))) { // todo: update if api suite is also run in matrix job + // Ignore directories containing 'api' + findReports(fullPath); // Recurse into subdirectories + } + } else if (file === 'results.json' && !fullPath.includes(path.join('api'))) { reportPaths.push(fullPath); } });