From 1fd928e0bb54c49663450b5fb17e9173a931bf77 Mon Sep 17 00:00:00 2001 From: Adam Date: Sat, 11 Nov 2023 14:47:43 +0000 Subject: [PATCH 1/6] dev: work on nx integration github workflow --- .github/workflows/nx-integration-tests.yml | 186 ++++++++++++++++++ nx-integration-tests-setup/action.yml | 120 +++++++++++ .../action.yml | 63 ++++++ 3 files changed, 369 insertions(+) create mode 100644 .github/workflows/nx-integration-tests.yml create mode 100644 nx-integration-tests-setup/action.yml create mode 100644 supported-services-matrix-calculator/action.yml diff --git a/.github/workflows/nx-integration-tests.yml b/.github/workflows/nx-integration-tests.yml new file mode 100644 index 0000000..ede35fc --- /dev/null +++ b/.github/workflows/nx-integration-tests.yml @@ -0,0 +1,186 @@ +name: NX Integration Test +#description: Workflow to run NX assisted integration tests for Mage-OS projects. +on: + workflow_call: + inputs: + repository: + type: string + description: "Repository" + required: true + pr_head: + type: string + description: "head SHA" + required: true + pr_base: + type: string + description: "pr base SHA" + required: true + workflow_dispatch: + inputs: + repository: + type: string + description: "Repository" + required: true + pr_head: + type: string + description: "head SHA" + required: true + pr_base: + type: string + description: "pr base SHA" + required: true + +jobs: + debug: + name: Debug + runs-on: ubuntu-latest + steps: + - name: debug + shell: bash + env: + repository: ${{ inputs.repository }} + pr_head: ${{ inputs.pr_head }} + pr_base: ${{ inputs.pr_base }} + run: | + echo "input was" + echo "repository: ${repository}" + echo "pr_head: ${pr_head}" + echo "pr_base: ${pr_base}" + + matrix-calculator: + outputs: + php_versions: ${{ steps.calculate-matrix.outputs.php_versions }} + database_versions: ${{ steps.calculate-matrix.outputs.database_versions }} + search_versions: ${{ steps.calculate-matrix.outputs.search_versions }} + message_queue_versions: ${{ steps.calculate-matrix.outputs.message_queue_versions }} + cache_versions: ${{ steps.calculate-matrix.outputs.cache_versions }} + http_cache_versions: ${{ steps.calculate-matrix.outputs.http_cache_versions }} + + runs-on: ubuntu-latest + steps: + - name: Run Matrix Calulator + id: calculate-matrix + uses: mage-os/github-actions/supported-services-matrix-calculator@main + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.pr_head }} + + - name: Calculated Result + shell: bash + env: + php_versions: ${{ steps.calculate-matrix.outputs.php_versions }} + database_versions: ${{ steps.calculate-matrix.outputs.database_versions }} + search_versions: ${{ steps.calculate-matrix.outputs.search_versions }} + message_queue_versions: ${{ steps.calculate-matrix.outputs.message_queue_versions }} + cache_versions: ${{ steps.calculate-matrix.outputs.cache_versions }} + http_cache_versions: ${{ steps.calculate-matrix.outputs.http_cache_versions }} + run: | + echo "PHP Versions: $php_versions" + echo "database_versions: $database_versions" + echo "search_versions: $search_versions" + echo "message_queue_versions: $message_queue_versions" + echo "cache_versions: $cache_versions" + echo "http_cache_versions: $http_cache_versions" + + nx-project-setup: + runs-on: ubuntu-latest + steps: + - name: Run Nx Project Setup + uses: mage-os/github-actions/nx-integration-tests-setup@main + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.pr_head }} + pr_base: ${{ inputs.pr_base }} + + integration-tests: + needs: + - matrix-calculator + - nx-project-setup + strategy: + fail-fast: false + matrix: + php_version: ${{ fromJSON(needs.matrix-calculator.outputs.php_versions) }} + database_version: ${{ fromJSON(needs.matrix-calculator.outputs.database_versions) }} + search_version: ${{ fromJSON(needs.matrix-calculator.outputs.search_versions) }} + message_queue_version: ${{ fromJSON(needs.matrix-calculator.outputs.message_queue_versions) }} + cache_version: ${{ fromJSON(needs.matrix-calculator.outputs.cache_versions) }} + http_cache_version: ${{ fromJSON(needs.matrix-calculator.outputs.http_cache_versions) }} + runs-on: ubuntu-latest + steps: + # - name: Running With + # shell: bash + # run: | + # echo "PHP Versions: ${{ matrix.php_version }}" + # echo "database Versions: ${{ matrix.database_version }}" + # echo "search Versions: ${{ matrix.search_version }}" + # echo "message_queue Versions: ${{ matrix.message_queue_version }}" + # echo "cache Versions: ${{ matrix.cache_version }}" + # echo "http_cache Versions: ${{ matrix.http_cache_version }}" + + - name: Project Cache + uses: actions/cache/restore@v3 + with: + path: main + key: ${{ runner.os }}-project-${{ inputs.ref }} + + # could probably swap this to a docker container in future + - name: Install NX + working-directory: ./main + run: | + npm install -g nx@15.4.1 + + - name: Print Affected + working-directory: ./main + run: | + AFFECTED_OUTPUT=/tmp/affect.json + nx print-affected --head=HEAD --base=remotes/origin/${{ inputs.pr_base }} > ${AFFECTED_OUTPUT} + cat ${AFFECTED_OUTPUT} + echo "Affected Projects: $(jq .projects ${AFFECTED_OUTPUT})" + + - name: Setup Warden Environment + uses: mage-os/github-actions/warden/setup-environment@main + with: + php_version: ${{ matrix.php_version }} + database: ${{ matrix.database_version }} + search: ${{ matrix.search_version }} + rabbitmq: ${{ matrix.message_queue_version }} + redis: ${{ matrix.cache_version }} + varnish: ${{ matrix.http_cache_version }} + base_directory: "./main" + + - name: Setup config for Integration tests + uses: mage-os/github-actions/warden/integration-tests@main + with: + search: ${{ matrix.search_version }} + rabbitmq: ${{ matrix.message_queue_version }} + redis: ${{ matrix.cache_version }} + run_memory_test: 0 + run_magento_integration_tests: 0 + run_magento_integration_tests_real_suite: 0 + base_directory: "./main" + + # - name: Run Memory Tests + # uses: ./main/.github/action/warden/run-integration-tests + # with: + # search: ${{ matrix.search_version }} + # rabbitmq: ${{ matrix.message_queue_version }} + # redis: ${{ matrix.cache_version }} + # run_memory_test: 1 + # run_magento_integration_tests: 0 + # run_magento_integration_tests_real_suite: 0 + + # - name: Run Integration Tests + # uses: ./main/.github/action/warden/run-integration-tests + # with: + # search: ${{ matrix.search_version }} + # rabbitmq: ${{ matrix.message_queue_version }} + # redis: ${{ matrix.cache_version }} + # run_memory_test: 0 + # run_magento_integration_tests: 1 + # run_magento_integration_tests_real_suite: 0 + + - name: Run Integration Tests (Real) + working-directory: ./main + run: | + export WARDEN="$(dirname $(pwd))/warden/bin/warden" + nx affected --target=test:integration --head=HEAD --base=remotes/origin/${{ inputs.pr_base }} diff --git a/nx-integration-tests-setup/action.yml b/nx-integration-tests-setup/action.yml new file mode 100644 index 0000000..f521162 --- /dev/null +++ b/nx-integration-tests-setup/action.yml @@ -0,0 +1,120 @@ +name: "Nx Integration Tests Setup" +author: "Mage-OS" +description: "Setup and cache Nx project, this can then be reused with all service combinations." + +inputs: + repository: + type: string + description: "Repository" + required: true + ref: + type: string + description: "head SHA" + required: true + pr_base: + type: string + description: "pr base SHA" + required: true + +# permissions: +# contents: write + +runs: + using: "composite" + steps: + - name: Checkout PR commit + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + path: main + # Need to do this otherwise Nx cant determine diff + fetch-depth: 0 + + - name: Fetch base + working-directory: ./main + shell: bash + run: git fetch origin ${{ inputs.pr_base }} + + - name: Cache Composer dependencies + uses: actions/cache@v3 + with: + path: /tmp/composer-cache + key: ${{ runner.os }}-${{ hashFiles('**/composer.lock') }} + + - id: get-composer-and-php-version + name: Get Composer & PHP Version + working-directory: ./main + shell: bash + run: | + echo "php_version=$(jq -c .services.php[0] supported-services.json)" >> "$GITHUB_OUTPUT" + echo "composer_version=$(jq -rc .services.composer[0] supported-services.json)" >> "$GITHUB_OUTPUT" + - name: Composer Install + uses: php-actions/composer@v6 + with: + version: ${{ steps.get-composer-and-php-version.outputs.composer_version }} + php_version: ${{ steps.get-composer-and-php-version.outputs.php_version }} + args: "--ignore-platform-reqs --optimize-autoloader" + working_dir: main + + # could probably swap this to a docker container in future + - name: Install NX + working-directory: ./main + shell: bash + run: | + npm install -g nx@15.4.1 + + # should be able to cache this in future also + - name: Checkout Nx Repo + uses: actions/checkout@v3 + with: + repository: adamzero1/nx-for-php + ref: docker-wrapper-2 + path: nx + + - name: Copy in NX files + working-directory: ./main + shell: bash + run: | + NXDIR="../nx" + cp -r ${NXDIR}/nx ./ + cp ${NXDIR}/nx.json ./ + cp ${NXDIR}/package.json ./ + cp ${NXDIR}/package-lock.json ./ + + - name: Install NPM Deps + working-directory: ./main + shell: bash + run: | + npm ci + + - name: Generate Nx Workspace + working-directory: ./main + shell: bash + run: | + npm run generate-workspace -- --commands=test:unit,test:integration \ + --test:unit='if [ -d {{ MODULE_PATH }}Test/Unit ]; then ${WARDEN} env exec -T php-fpm ./vendor/bin/phpunit -c dev/tests/unit/phpunit.xml {{ MODULE_PATH }}Test/Unit; else echo "{{ MODULE_NAME }} has no unit test; fi' \ + --test:integration='${WARDEN} env exec -T --workdir /var/www/html/dev/tests/integration php-fpm ../../../vendor/bin/phpunit --configuration phpunit.xml.dist --testsuite '"'"'Magento Integration Tests Real Suite'"'"' --filter='"'"'/Magento/{{ MODULE_DIRECTORY }}/|Magento\\{{ MODULE_DIRECTORY }}'"'"' --log-junit=../../../phpunit-output/junit/{{ MODULE_DIRECTORY }}.xml --coverage-html=../../../phpunit-output/coverage-html/{{ MODULE_DIRECTORY }}' + + - name: Print Affected + working-directory: ./main + shell: bash + run: | + AFFECTED_OUTPUT=/tmp/affect.json + nx print-affected --head=HEAD --base=remotes/origin/${{ inputs.pr_base }} > ${AFFECTED_OUTPUT} + echo "Affected Projects: $(jq .projects ${AFFECTED_OUTPUT})" + + # just to get some timings + - name: Print Affected2 + working-directory: ./main + shell: bash + run: | + AFFECTED_OUTPUT=/tmp/affect.json + nx print-affected --head=HEAD --base=remotes/origin/${{ inputs.pr_base }} > ${AFFECTED_OUTPUT} + echo "Affected Projects: $(jq .projects ${AFFECTED_OUTPUT})" + + - name: Project Cache + uses: actions/cache/save@v3 + with: + path: main + key: ${{ runner.os }}-project-${{ inputs.ref }} \ No newline at end of file diff --git a/supported-services-matrix-calculator/action.yml b/supported-services-matrix-calculator/action.yml new file mode 100644 index 0000000..813ac53 --- /dev/null +++ b/supported-services-matrix-calculator/action.yml @@ -0,0 +1,63 @@ +name: "Supported Services Matrix Calculator" +author: "Mage-OS" +description: "Calulate a matrix of all supported services (based off supported-services.json)" + +inputs: + repository: + type: string + description: "Repository" + required: true + ref: + type: string + description: "head SHA" + required: true + +outputs: + php_versions: + description: The applicable PHP versions + value: ${{ steps.set-matrix.outputs.php_versions }} + database_versions: + description: The applicable DB versions + value: ${{ steps.set-matrix.outputs.database_versions }} + search_versions: + description: The applicable Search versions + value: ${{ steps.set-matrix.outputs.search_versions }} + message_queue_versions: + description: The applicable Message Queue versions + value: ${{ steps.set-matrix.outputs.message_queue_versions }} + cache_versions: + description: The applicable Cache versions + value: ${{ steps.set-matrix.outputs.cache_versions }} + http_cache_versions: + description: The applicable HTTP Cache versions + value: ${{ steps.set-matrix.outputs.http_cache_versions }} + +runs: + using: "composite" + steps: + - name: Checkout PR commit + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + + - id: set-matrix + name: Calculate Matrix + shell: bash + run: | + echo "php_versions=$(jq -c .services.php supported-services.json)" >> "$GITHUB_OUTPUT" + echo "database_versions=$(jq -c .services.database supported-services.json)" >> "$GITHUB_OUTPUT" + echo "search_versions=$(jq -c .services.search supported-services.json)" >> "$GITHUB_OUTPUT" + echo "message_queue_versions=$(jq -c .services.message_queue supported-services.json)" >> "$GITHUB_OUTPUT" + echo "cache_versions=$(jq -c .services.cache supported-services.json)" >> "$GITHUB_OUTPUT" + echo "http_cache_versions=$(jq -c .services.http_cache supported-services.json)" >> "$GITHUB_OUTPUT" + + - name: Debug output + shell: bash + run: | + echo "PHP Versions: ${{ steps.set-matrix.outputs.php_versions }}" + echo "database Versions: ${{ steps.set-matrix.outputs.database_versions }}" + echo "search Versions: ${{ steps.set-matrix.outputs.search_versions }}" + echo "message_queue Versions: ${{ steps.set-matrix.outputs.message_queue_versions }}" + echo "cache Versions: ${{ steps.set-matrix.outputs.cache_versions }}" + echo "http_cache Versions: ${{ steps.set-matrix.outputs.http_cache_versions }}" \ No newline at end of file From 7af06fcecf73bd1bf9622770fdecbff28e59482e Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 27 Nov 2023 16:23:36 +0000 Subject: [PATCH 2/6] dev: removing debug --- .github/workflows/nx-integration-tests.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/nx-integration-tests.yml b/.github/workflows/nx-integration-tests.yml index ede35fc..64ba23f 100644 --- a/.github/workflows/nx-integration-tests.yml +++ b/.github/workflows/nx-integration-tests.yml @@ -107,16 +107,6 @@ jobs: http_cache_version: ${{ fromJSON(needs.matrix-calculator.outputs.http_cache_versions) }} runs-on: ubuntu-latest steps: - # - name: Running With - # shell: bash - # run: | - # echo "PHP Versions: ${{ matrix.php_version }}" - # echo "database Versions: ${{ matrix.database_version }}" - # echo "search Versions: ${{ matrix.search_version }}" - # echo "message_queue Versions: ${{ matrix.message_queue_version }}" - # echo "cache Versions: ${{ matrix.cache_version }}" - # echo "http_cache Versions: ${{ matrix.http_cache_version }}" - - name: Project Cache uses: actions/cache/restore@v3 with: From 28524a8dc76901e43397efdb1e34f12f46c5c2ba Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 27 Nov 2023 16:26:35 +0000 Subject: [PATCH 3/6] dev: removing examples of other jobs --- .github/workflows/nx-integration-tests.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/nx-integration-tests.yml b/.github/workflows/nx-integration-tests.yml index 64ba23f..eb2e371 100644 --- a/.github/workflows/nx-integration-tests.yml +++ b/.github/workflows/nx-integration-tests.yml @@ -149,26 +149,6 @@ jobs: run_magento_integration_tests_real_suite: 0 base_directory: "./main" - # - name: Run Memory Tests - # uses: ./main/.github/action/warden/run-integration-tests - # with: - # search: ${{ matrix.search_version }} - # rabbitmq: ${{ matrix.message_queue_version }} - # redis: ${{ matrix.cache_version }} - # run_memory_test: 1 - # run_magento_integration_tests: 0 - # run_magento_integration_tests_real_suite: 0 - - # - name: Run Integration Tests - # uses: ./main/.github/action/warden/run-integration-tests - # with: - # search: ${{ matrix.search_version }} - # rabbitmq: ${{ matrix.message_queue_version }} - # redis: ${{ matrix.cache_version }} - # run_memory_test: 0 - # run_magento_integration_tests: 1 - # run_magento_integration_tests_real_suite: 0 - - name: Run Integration Tests (Real) working-directory: ./main run: | From a85b811eaba094bdd91a1a1d67e089078e47fbe0 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 27 Nov 2023 16:28:28 +0000 Subject: [PATCH 4/6] dev: implementing suggested changes --- nx-integration-tests-setup/action.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nx-integration-tests-setup/action.yml b/nx-integration-tests-setup/action.yml index f521162..fdd5893 100644 --- a/nx-integration-tests-setup/action.yml +++ b/nx-integration-tests-setup/action.yml @@ -16,14 +16,12 @@ inputs: description: "pr base SHA" required: true -# permissions: -# contents: write runs: using: "composite" steps: - name: Checkout PR commit - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} @@ -66,7 +64,7 @@ runs: # should be able to cache this in future also - name: Checkout Nx Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: adamzero1/nx-for-php ref: docker-wrapper-2 From 5bf3436207a6a12169d77dd6d8b7e7d13e8cec50 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 28 Nov 2023 09:49:27 +0000 Subject: [PATCH 5/6] dev: using range for Nx version --- nx-integration-tests-setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nx-integration-tests-setup/action.yml b/nx-integration-tests-setup/action.yml index fdd5893..8f6863a 100644 --- a/nx-integration-tests-setup/action.yml +++ b/nx-integration-tests-setup/action.yml @@ -60,7 +60,7 @@ runs: working-directory: ./main shell: bash run: | - npm install -g nx@15.4.1 + npm install -g nx@^15.4 # should be able to cache this in future also - name: Checkout Nx Repo From 6632e423328c1d2d9d65f317cf87af86b0384391 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 28 Nov 2023 13:43:45 +0000 Subject: [PATCH 6/6] dev: using range for Nx version --- .github/workflows/nx-integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nx-integration-tests.yml b/.github/workflows/nx-integration-tests.yml index eb2e371..3213d21 100644 --- a/.github/workflows/nx-integration-tests.yml +++ b/.github/workflows/nx-integration-tests.yml @@ -117,7 +117,7 @@ jobs: - name: Install NX working-directory: ./main run: | - npm install -g nx@15.4.1 + npm install -g nx@^15.4 - name: Print Affected working-directory: ./main