diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index d166b375..1064d09f 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -24,47 +24,47 @@ jobs: fail-fast: false matrix: entry: - - version: 1.3.17 - admin_password: admin - - version: 2.0.0 - admin_password: admin - - version: 2.18.0 - - version: 2.18.0 - tests: plugins/index_state_management - - version: 2.18.0 - tests: plugins/ml - - version: 2.18.0 - tests: routing + # - version: 1.3.17 + # admin_password: admin + # - version: 2.0.0 + # admin_password: admin + # - version: 2.18.0 + # - version: 2.18.0 + # tests: plugins/index_state_management + # - version: 2.18.0 + # tests: plugins/ml + # - version: 2.18.0 + # tests: routing - version: 2.18.0 tests: snapshot - - version: 2.18.0 - tests: remote_store - - version: 2.18.0 - tests: dangling - url: http://localhost:9200 - - version: 2.18.0 - tests: plugins/replication - url: http://localhost:9200 - - version: 2.18.0 - tests: plugins/streaming - - version: 2.18.0 - tests: plugins/notifications - - version: 2.18.0 - tests: plugins/query_insights - - version: 2.18.0 - tests: plugins/workload-management - - version: 2.18.0 - tests: plugins/analysis - - version: 2.18.0 - tests: plugins/security - cert: tests/plugins/security/.kirk.pem - key: tests/plugins/security/.kirk-key.pem - - version: 2.19.0 - hub: opensearchstaging - ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' - - version: 3.0.0 - hub: opensearchstaging - ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de' + # - version: 2.18.0 + # tests: remote_store + # - version: 2.18.0 + # tests: dangling + # url: http://localhost:9200 + # - version: 2.18.0 + # tests: plugins/replication + # url: http://localhost:9200 + # - version: 2.18.0 + # tests: plugins/streaming + # - version: 2.18.0 + # tests: plugins/notifications + # - version: 2.18.0 + # tests: plugins/query_insights + # - version: 2.18.0 + # tests: plugins/workload-management + # - version: 2.18.0 + # tests: plugins/analysis + # - version: 2.18.0 + # tests: plugins/security + # cert: tests/plugins/security/.kirk.pem + # key: tests/plugins/security/.kirk-key.pem + # - version: 2.19.0 + # hub: opensearchstaging + # ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' + # - version: 3.0.0 + # hub: opensearchstaging + # ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest @@ -83,7 +83,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: '20' + node-version: "20" - name: Install Dependencies run: npm ci @@ -105,7 +105,8 @@ jobs: --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \ --opensearch-cert=${{ matrix.entry.cert }} \ --opensearch-key=${{ matrix.entry.key }} \ - --tests=tests/${{ matrix.entry.tests || 'default' }} + --tests=tests/${{ matrix.entry.tests || 'default' }} \ + --verbose - name: Get Container Logs if: failure() @@ -120,83 +121,3 @@ jobs: with: name: coverage-${{ matrix.entry.version }}-${{ steps.tests.outputs.hash }} path: coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json - - merge-coverage: - runs-on: ubuntu-latest - needs: test-opensearch-spec - steps: - - uses: actions/checkout@v4 - - - name: Download Spec Coverage Data - uses: actions/download-artifact@v4 - with: - path: coverage - - - name: Collect Test File Names - run: | - find tests/. -name "*.yaml" | xargs realpath | jq -Rn '{ files: [inputs | "\(.)"] }' > test-files.json - - - name: Display Unused Test Files - run: | - jq -r -sc ' - (map(.stories) | add | unique) as $stories | - (map(.files) | add | unique) as $all | - $all-$stories | - .[] - ' $(find ./ -name "test-spec-coverage-*.json") test-files.json > ./coverage/files.txt - cat ./coverage/files.txt | sed -e 's/^/::error::/' - test ! -s ./coverage/files.txt || { echo "::error::Unused test files detected."; exit 1; } - - - name: Collect and Display Missing Test Paths - run: | - jq -r -sc ' - (map(.operations) | add | unique) as $all | - (map(.evaluated_operations) | add | unique) as $evaluated | - ($all-$evaluated | map({path: .path, method: .method, evaluated: false})) as $unevaluated | - ($evaluated | map({path: .path, method: .method, evaluated: true})) as $evaluated | - $evaluated+$unevaluated | - sort_by(.path) | - map(if .evaluated then "- [x] \(.method) \(.path)" else "- [ ] \(.method) \(.path)" end) | - .[] - ' $(find ./ -name "test-spec-coverage-*.json") - - - name: Combine Test Coverage Data - shell: bash -eo pipefail {0} - run: | - jq -sc ' - (map(.operations) | add | unique | length) as $total_operations_count | - (map(.evaluated_operations) | add | unique | length) as $evaluated_operations_count | - { - total_operations_count: $total_operations_count, - evaluated_operations_count: $evaluated_operations_count, - evaluated_paths_pct: $evaluated_operations_count | (10000 * . / $total_operations_count | round / 100) - } - ' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json - cat ./coverage/coverage.json - - - name: Construct Comment Data Payload - if: github.event_name == 'pull_request' - shell: bash -eo pipefail {0} - run: | - jq \ - --arg pr_number ${PR_NUMBER} \ - --slurpfile test_coverage ./coverage/coverage.json \ - --null-input ' - { - "pr_number": ($pr_number), - "comment_identifier": "## Spec Test Coverage Analysis", - "template_name": "pr-test-coverage-analysis", - "template_data": { - "test_coverage": ($test_coverage[0]) - } - } - ' | tee pr-comment.json - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - - - name: Upload PR Comment Payload - uses: actions/upload-artifact@v4 - if: github.event_name == 'pull_request' - with: - name: pr-comment - path: pr-comment.json