Skip to content

Commit

Permalink
WIP: try to reproduce snapshot failure
Browse files Browse the repository at this point in the history 
Signed-off-by: dblock <[email protected]>
  • Loading branch information
@dblock
dblock committed Dec 19, 2024
1 parent 4231dad commit 3db4e74
Showing 1 changed file with 45 additions and 121 deletions.
166 changes: 45 additions & 121 deletions .github/workflows/test-spec.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,47 +24,57 @@ jobs:
fail-fast: false
matrix:
entry:
- version: 1.3.17
admin_password: admin
- version: 2.0.0
admin_password: admin
- version: 2.18.0
- version: 2.18.0
tests: plugins/index_state_management
- version: 2.18.0
tests: plugins/ml
- version: 2.18.0
tests: routing
# - version: 1.3.17
# admin_password: admin
# - version: 2.0.0
# admin_password: admin
# - version: 2.18.0
# - version: 2.18.0
# tests: plugins/index_state_management
# - version: 2.18.0
# tests: plugins/ml
# - version: 2.18.0
# tests: routing
- version: 2.18.0
tests: snapshot
- version: 2.18.0
tests: remote_store
- version: 2.18.0
tests: dangling
url: http://localhost:9200
- version: 2.18.0
tests: plugins/replication
url: http://localhost:9200
- version: 2.18.0
tests: plugins/streaming
- version: 2.18.0
tests: plugins/notifications
tests: snapshot
- version: 2.18.0
tests: plugins/query_insights
tests: snapshot
- version: 2.18.0
tests: plugins/workload-management
tests: snapshot
- version: 2.18.0
tests: plugins/analysis
tests: snapshot
- version: 2.18.0
tests: plugins/security
cert: tests/plugins/security/.kirk.pem
key: tests/plugins/security/.kirk-key.pem
- version: 2.19.0
hub: opensearchstaging
ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'
- version: 3.0.0
hub: opensearchstaging
ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de'
tests: snapshot
# - version: 2.18.0
# tests: remote_store
# - version: 2.18.0
# tests: dangling
# url: http://localhost:9200
# - version: 2.18.0
# tests: plugins/replication
# url: http://localhost:9200
# - version: 2.18.0
# tests: plugins/streaming
# - version: 2.18.0
# tests: plugins/notifications
# - version: 2.18.0
# tests: plugins/query_insights
# - version: 2.18.0
# tests: plugins/workload-management
# - version: 2.18.0
# tests: plugins/analysis
# - version: 2.18.0
# tests: plugins/security
# cert: tests/plugins/security/.kirk.pem
# key: tests/plugins/security/.kirk-key.pem
# - version: 2.19.0
# hub: opensearchstaging
# ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'
# - version: 3.0.0
# hub: opensearchstaging
# ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de'

name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }})
runs-on: ubuntu-latest
Expand All @@ -83,7 +93,7 @@ jobs:
- name: Setup Node.js
uses: actions/setup-node@v3
with:
node-version: '20'
node-version: "20"

Check failure on line 96 in .github/workflows/test-spec.yml

View workflow job for this annotation

GitHub Actions / lint 

Strings must use singlequote

- name: Install Dependencies
run: npm ci
Expand Down Expand Up @@ -114,89 +124,3 @@ jobs:
echo Dumping logs from $container_id ... && \
docker logs $container_id \
; done
- name: Upload Test Coverage Results
uses: actions/upload-artifact@v4
with:
name: coverage-${{ matrix.entry.version }}-${{ steps.tests.outputs.hash }}
path: coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json

merge-coverage:
runs-on: ubuntu-latest
needs: test-opensearch-spec
steps:
- uses: actions/checkout@v4

- name: Download Spec Coverage Data
uses: actions/download-artifact@v4
with:
path: coverage

- name: Collect Test File Names
run: |
find tests/. -name "*.yaml" | xargs realpath | jq -Rn '{ files: [inputs | "\(.)"] }' > test-files.json
- name: Display Unused Test Files
run: |
jq -r -sc '
(map(.stories) | add | unique) as $stories |
(map(.files) | add | unique) as $all |
$all-$stories |
.[]
' $(find ./ -name "test-spec-coverage-*.json") test-files.json > ./coverage/files.txt
cat ./coverage/files.txt | sed -e 's/^/::error::/'
test ! -s ./coverage/files.txt || { echo "::error::Unused test files detected."; exit 1; }
- name: Collect and Display Missing Test Paths
run: |
jq -r -sc '
(map(.operations) | add | unique) as $all |
(map(.evaluated_operations) | add | unique) as $evaluated |
($all-$evaluated | map({path: .path, method: .method, evaluated: false})) as $unevaluated |
($evaluated | map({path: .path, method: .method, evaluated: true})) as $evaluated |
$evaluated+$unevaluated |
sort_by(.path) |
map(if .evaluated then "- [x] \(.method) \(.path)" else "- [ ] \(.method) \(.path)" end) |
.[]
' $(find ./ -name "test-spec-coverage-*.json")
- name: Combine Test Coverage Data
shell: bash -eo pipefail {0}
run: |
jq -sc '
(map(.operations) | add | unique | length) as $total_operations_count |
(map(.evaluated_operations) | add | unique | length) as $evaluated_operations_count |
{
total_operations_count: $total_operations_count,
evaluated_operations_count: $evaluated_operations_count,
evaluated_paths_pct: $evaluated_operations_count | (10000 * . / $total_operations_count | round / 100)
}
' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json
cat ./coverage/coverage.json
- name: Construct Comment Data Payload
if: github.event_name == 'pull_request'
shell: bash -eo pipefail {0}
run: |
jq \
--arg pr_number ${PR_NUMBER} \
--slurpfile test_coverage ./coverage/coverage.json \
--null-input '
{
"pr_number": ($pr_number),
"comment_identifier": "## Spec Test Coverage Analysis",
"template_name": "pr-test-coverage-analysis",
"template_data": {
"test_coverage": ($test_coverage[0])
}
}
' | tee pr-comment.json
env:
PR_NUMBER: ${{ github.event.pull_request.number }}

- name: Upload PR Comment Payload
uses: actions/upload-artifact@v4
if: github.event_name == 'pull_request'
with:
name: pr-comment
path: pr-comment.json

0 comments on commit 3db4e74

Please sign in to comment.