diff --git a/.cspell b/.cspell index f292a14d7..37a004c69 100644 --- a/.cspell +++ b/.cspell @@ -47,6 +47,7 @@ aarch actiongroup actiongroups aggregatable +argjson asciifolding authc authinfo diff --git a/.github/pr-comment-templates/pr-test-coverage-analysis.template b/.github/pr-comment-templates/pr-test-coverage-analysis.template new file mode 100644 index 000000000..2dcd0ae02 --- /dev/null +++ b/.github/pr-comment-templates/pr-test-coverage-analysis.template @@ -0,0 +1,8 @@ +## Spec Test Coverage Analysis +{{with .test_coverage}} + +| Total | Tested | +|-------------------|-----------------------------| +| {{.paths_count}} | {{.evaluated_paths_count}} | + +{{end}} \ No newline at end of file diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml index f66217f69..037cca862 100644 --- a/.github/workflows/pr-comment.yml +++ b/.github/workflows/pr-comment.yml @@ -4,6 +4,7 @@ on: workflow_run: workflows: - Analyze PR Changes + - Test Spec types: - completed @@ -71,4 +72,4 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ env.PR_NUMBER }} body: ${{ steps.render.outputs.result }} - edit-mode: replace \ No newline at end of file + edit-mode: replace diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 6686b8d9e..6395778a1 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -54,10 +54,61 @@ jobs: run: docker-compose up -d - name: Run Tests - run: npm run test:spec -- --opensearch-insecure --coverage test-spec-coverage-${{ matrix.entry.version }}.json + run: | + npm run test:spec -- --opensearch-insecure --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json - name: Upload Test Coverage Results uses: actions/upload-artifact@v4 with: - name: test-spec-coverage-${{ matrix.entry.version }} - path: test-spec-coverage-${{ matrix.entry.version }}.json \ No newline at end of file + name: coverage-${{ matrix.entry.version }} + path: coverage/test-spec-coverage-${{ matrix.entry.version }}.json + + merge-coverage: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + needs: test-opensearch-spec + steps: + - uses: actions/checkout@v3 + + - name: Download Spec Coverage Data + uses: actions/download-artifact@v4 + with: + path: coverage + + - name: Combine Test Coverage Data + shell: bash -eo pipefail {0} + run: | + jq -sc ' + map(to_entries) | + flatten | + group_by(.key) | + map({key: .[0].key, value: map(.value) | max}) | + from_entries | + .' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json + + cat ./coverage/coverage.json + + - name: Construct Comment Data Payload + shell: bash -eo pipefail {0} + run: | + jq \ + --arg pr_number ${PR_NUMBER} \ + --slurpfile test_coverage ./coverage/coverage.json \ + --null-input ' + { + "pr_number": ($pr_number), + "comment_identifier": "# Test Coverage Analysis", + "template_name": "pr-test-coverage-analysis", + "template_data": { + "test_coverage": ($test_coverage[0]) + } + } + ' | tee pr-comment.json + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + + - name: Upload PR Comment Payload + uses: actions/upload-artifact@v4 + with: + name: pr-comment + path: pr-comment.json \ No newline at end of file diff --git a/tools/src/tester/TestResults.ts b/tools/src/tester/TestResults.ts index 112a5bbf3..8740d8771 100644 --- a/tools/src/tester/TestResults.ts +++ b/tools/src/tester/TestResults.ts @@ -34,8 +34,10 @@ export default class TestResults { test_coverage(): SpecTestCoverage { return { - evaluated_paths_count: this.evaluated_paths_count(), - paths_count: this.spec_paths_count() + test_coverage: { + evaluated_paths_count: this.evaluated_paths_count(), + paths_count: this.spec_paths_count() + } } } diff --git a/tools/src/tester/types/test.types.ts b/tools/src/tester/types/test.types.ts index 535770099..9ce87f311 100644 --- a/tools/src/tester/types/test.types.ts +++ b/tools/src/tester/types/test.types.ts @@ -8,6 +8,8 @@ */ export interface SpecTestCoverage { - paths_count: number - evaluated_paths_count: number + test_coverage: { + paths_count: number + evaluated_paths_count: number + } } diff --git a/tools/tests/tester/TestResults.test.ts b/tools/tests/tester/TestResults.test.ts index d6a5b239d..21eed3c62 100644 --- a/tools/tests/tester/TestResults.test.ts +++ b/tools/tests/tester/TestResults.test.ts @@ -46,8 +46,10 @@ describe('TestResults', () => { const filename = 'coverage.json' test_results.write_coverage(filename) expect(JSON.parse(fs.readFileSync(filename, 'utf8'))).toEqual({ - evaluated_paths_count: 1, - paths_count: 4 + test_coverage: { + evaluated_paths_count: 1, + paths_count: 4 + } }) fs.unlinkSync(filename) })