Devnet Release Scenarios #11766
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Devnet Release Scenarios | |
| on: | |
| schedule: | |
| - cron: '0 */2 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| Run: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: | |
| - FourValidatorsReconnect | |
| - MultipleValidatorsDown | |
| - FourValidatorsReconnectRmDatabase | |
| - FourValidatorsReconnectSpammer | |
| - MacroBlockProduction | |
| - Validators90sDownFullNodes | |
| - LowBlockProducerTimeout | |
| - FiveValidatorsWithSpammer | |
| - ValidatorsStress | |
| include: | |
| - test: FourValidatorsReconnect | |
| devnet_args: -t .github/devnet_topologies/four_validators.toml -R | |
| - test: MultipleValidatorsDown | |
| devnet_args: -t .github/devnet_topologies/four_validators.toml -k 2 -R -as | |
| - test: FourValidatorsReconnectRmDatabase | |
| devnet_args: -t .github/devnet_topologies/four_validators.toml -d -R | |
| - test: FourValidatorsReconnectSpammer | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_3.toml -R | |
| - test: MacroBlockProduction | |
| pre: "sed -i 's/BLOCKS_PER_BATCH: u32 = 32;/BLOCKS_PER_BATCH: u32 = 2;/g' primitives/src/policy.rs" | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_2.toml -k 0 -R | |
| - test: Validators90sDownFullNodes | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_3.toml -dt 90 -R | |
| - test: LowBlockProducerTimeout | |
| pre: "sed -i 's/MIN_PRODUCER_TIMEOUT: u64 = 4 * 1000;/MIN_PRODUCER_TIMEOUT: u64 = 2 * 1000;/g' primitives/src/policy.rs" | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_2.toml -k 0 -R | |
| - test: FiveValidatorsWithSpammer | |
| devnet_args: -t .github/devnet_topologies/five_validators_multi_spammer.toml -R -k 0 | |
| - test: ValidatorsStress | |
| devnet_args: -t .github/devnet_topologies/validators_stress.toml -R -k 2 -as -dt 30 | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| large-packages: false | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.8' | |
| - name: Install python dependencies | |
| run: pip install scripts/devnet | |
| - name: Set up Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Optionally patch the source | |
| run: ${{ matrix.pre }} | |
| - name: Build the code | |
| run: cargo build --release | |
| - name: Retrieve initial timestamp | |
| id: initial_ts | |
| run: | | |
| echo "INITIAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT | |
| - name: Execute the test | |
| run: | | |
| python3 scripts/devnet/devnet.py --run-environment ci ${{ matrix.devnet_args }} | |
| env: | |
| NIMIQ_LOKI_URL: ${{ secrets.LOKI_URL }} | |
| NIMIQ_LOKI_EXTRA_FIELDS: nimiq_ci_name=${{ github.workflow }}:nimiq_ci_run=${{ github.run_number }}:nimiq_test=${{ matrix.test }} | |
| - name: Redact Loki URL | |
| if: always() | |
| run: sed -i 's/url = "[^"]*"/url = "<redacted>"/g' temp-logs/*/conf/*/*.toml | |
| - name: Archive test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.test }}-logs | |
| path: | | |
| temp-logs/ | |
| - name: Retrieve failure reason | |
| if: always() | |
| run: | | |
| if [ -f temp-state/*/RESULT.TXT ]; then | |
| echo "FAIL_REASON=$(cat temp-state/*/RESULT.TXT)" >> $GITHUB_OUTPUT | |
| else | |
| echo "FAIL_REASON=other" >> $GITHUB_OUTPUT | |
| fi | |
| id: reason | |
| - name: Retrieve final timestamp and Run ID | |
| if: always() | |
| id: final_ts_run_id | |
| run: | | |
| echo "FINAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT | |
| echo "RUN_ID=$(cat temp-logs/*/conf/run_id.info)" >> $GITHUB_OUTPUT | |
| - name: Build Grafana dashboard link | |
| if: always() | |
| id: grafana_url | |
| run: | | |
| echo "GRAFANA_URL=http://localhost:3001/d/YbjdvlU7z/nimiq-test?orgId=1&var-env=ci&var-run_id=${{steps.final_ts_run_id.outputs.RUN_ID}}&from=${{steps.initial_ts.outputs.INITIAL_TS}}&to=${{steps.final_ts_run_id.outputs.FINAL_TS}}" >> $GITHUB_OUTPUT | |
| - name: Report potential deadlocks to slack | |
| if: always() && contains(steps.reason.outputs.FAIL_REASON, 'DEADLOCK') | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: ${{ job.status }} | |
| notification_title: 'Potential deadlock detected in ${{ matrix.test }}' | |
| footer: '<{run_url}|View Run>' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| # - name: Report long lock hold times | |
| # if: always() && contains(steps.reason.outputs.FAIL_REASON, 'LONG_LOCK_HOLD_TIME') | |
| # uses: ravsamhq/notify-slack-action@v2 | |
| # with: | |
| # status: ${{ job.status }} | |
| # notification_title: 'Long lock hold time in ${{ matrix.test }}' | |
| # footer: '<{run_url}|View Run>' | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| # - name: Report slow lock acquisitions | |
| # if: always() && contains(steps.reason.outputs.FAIL_REASON, 'SLOW_LOCK_ACQUISITION') | |
| # uses: ravsamhq/notify-slack-action@v2 | |
| # with: | |
| # status: ${{ job.status }} | |
| # notification_title: 'Slow lock acquisition detected in ${{ matrix.test }}' | |
| # footer: '<{run_url}|View Run>' | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| - name: Report Status to Slack | |
| if: always() && github.ref == 'refs/heads/albatross' | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: ${{ job.status }} | |
| notify_when: 'failure' | |
| notification_title: '${{ matrix.test }} failed because of ${{ steps.reason.outputs.FAIL_REASON }}' | |
| footer: '<{run_url}|View Run> | <${{steps.grafana_url.outputs.GRAFANA_URL}}|Grafana dashboard>' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} |