refactor!: black-box integration tests (#5124)

fix: lints, correct upload of `executor.wasm` fix: make `iroha_core` compile without `telemetry` feature ci: copy executor from script; debug; single test command ci: remove debug, enable full tests chore: remove rebase artifact ci: upload test network artifacts ci: add `ci` nextest profile fix: chores after rebase; print STDERR; handle early peer termination ci: use `NEXTEST_PROFILE` env var test: simplify parameter setting test chore: fix cargo suggestion, add `--path` refactor: replace `consensus_estimation` with `pipeline_time` test: adjust nextest profiles chore: update comment about kura test revert: redundant assert from #5140 ci: set retention days to 3 for test network runs ci: upload artifacts only on failure chore: use `expect` ci: refine artifacts upload policy --------- Signed-off-by: 0x009922 <[email protected]>
hyperledger-iroha · Oct 18, 2024 · bc79734 · bc79734
1 parent 237a8c6
commit bc79734
Show file tree

Hide file tree

Showing 78 changed files with 3,847 additions and 4,505 deletions.
diff --git a/.config/nextest.toml b/.config/nextest.toml
@@ -0,0 +1,8 @@
+[profile.default]
+retries = 2
+
+[profile.ci]
+fail-fast = false
+failure-output = "immediate-final"
+slow-timeout = { period = "30s", terminate-after = 2 }
+
diff --git a/.github/workflows/iroha2-dev-pr.yml b/.github/workflows/iroha2-dev-pr.yml
@@ -20,6 +20,8 @@ env:
   IROHA_CLI_DIR: "/__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/test"
   DOCKER_COMPOSE_PATH: defaults
   WASM_SAMPLES_TARGET_DIR: wasm_samples/target/prebuilt
+  TEST_NETWORK_TMP_DIR: /tmp
+  NEXTEST_PROFILE: ci
 
 jobs:
   consistency:
@@ -83,7 +85,7 @@ jobs:
           path: ${{ env.DOCKER_COMPOSE_PATH }}/executor.wasm
           retention-days: 1
 
-  unit_tests_with_coverage:
+  test_with_coverage:
     runs-on: [self-hosted, Linux, iroha2]
     container:
       image: hyperledger/iroha2-ci:nightly-2024-09-09
@@ -92,94 +94,46 @@ jobs:
       LLVM_PROFILE_FILE_NAME: "iroha-%p-%m.profraw"
     steps:
       - uses: actions/checkout@v4
+      - uses: taiki-e/install-action@nextest
+      - uses: taiki-e/install-action@cargo-llvm-cov
       - name: Download executor.wasm
         uses: actions/download-artifact@v4
         with:
           name: executor.wasm
           path: ${{ env.DOCKER_COMPOSE_PATH }}
-      - uses: taiki-e/install-action@nextest
-      - uses: taiki-e/install-action@cargo-llvm-cov
-      - name: Run unit tests (no default features)
+      - name: Download the rest of WASM samples
+        uses: actions/download-artifact@v4
+        with:
+          name: wasm_samples
+          path: ${{ env.WASM_SAMPLES_TARGET_DIR }}
+      - name: Install irohad
+        run: which irohad || cargo install --path crates/irohad --locked
+      - name: Test with no default features
+        id: test_no_features
         run: >
           mold --run cargo llvm-cov nextest
-          --no-fail-fast
-          --workspace --lib
           --no-default-features
-          --branch
-          --no-report
-      - name: Run unit tests (all features)
+          --branch --no-report
+      - name: Test with all features
+        id: test_all_features
         run: >
           mold --run cargo llvm-cov nextest
-          --no-fail-fast
-          --workspace --lib
           --all-features
-          --branch
-          --no-report
+          --branch --no-report
       - name: Generate lcov report
         run: cargo llvm-cov report --text --output-path coverage.txt
       - name: Upload lcov report
         uses: actions/upload-artifact@v4
         with:
           name: report-coverage
           path: coverage.txt
-
-  # include: iroha/tests/integration/
-  # exclude: iroha/tests/integration/extra_functional
-  integration:
-    runs-on: [self-hosted, Linux, iroha2]
-    container:
-      image: hyperledger/iroha2-ci:nightly-2024-09-09
-    needs: build_wasm_samples
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v4
-      - name: Download executor.wasm
-        uses: actions/download-artifact@v4
-        with:
-          name: executor.wasm
-          path: ${{ env.DOCKER_COMPOSE_PATH }}
-      - name: Download the rest of WASM samples
-        uses: actions/download-artifact@v4
-        with:
-          name: wasm_samples
-          path: ${{ env.WASM_SAMPLES_TARGET_DIR }}
-      - uses: taiki-e/install-action@nextest
-      - name: Run integration tests, with all features
-        run: >
-          mold --run cargo nextest run 
-          --all-features
-          --no-fail-fast 
-          --failure-output immediate-final 
-          -E 'package(iroha) and test(integration) and not test(extra_functional)'
-
-  # include: iroha/tests/integration/extra_functional
-  extra_functional:
-    runs-on: [self-hosted, Linux, iroha2]
-    container:
-      image: hyperledger/iroha2-ci:nightly-2024-09-09
-    needs: build_wasm_samples
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v4
-      - name: Download executor.wasm
-        uses: actions/download-artifact@v4
-        with:
-          name: executor.wasm
-          path: ${{ env.DOCKER_COMPOSE_PATH }}
-      - name: Download the rest of WASM samples
-        uses: actions/download-artifact@v4
+      - name: Upload test network artifacts
+        if: failure() && (steps.test_no_features.outcome == 'failure' || steps.test_all_features.outcome == 'failure')
+        uses: actions/upload-artifact@v4
         with:
-          name: wasm_samples
-          path: ${{ env.WASM_SAMPLES_TARGET_DIR }}
-      - uses: taiki-e/install-action@nextest
-      - name: Run integration tests, with all features
-        run: >
-          mold --run cargo nextest run 
-          --all-features
-          --no-fail-fast 
-          --failure-output final 
-          --test-threads 1
-          -E 'test(extra_functional)'
+          name: test_network_runs
+          path: ${{ env.TEST_NETWORK_TMP_DIR }}/irohad_test_network_*
+          retention-days: 3
 
   # Run the job to check that the docker containers are properly buildable
   pr-generator-build:

diff --git a/.github/workflows/iroha2-pr-ui.yml b/.github/workflows/iroha2-pr-ui.yml
@@ -17,6 +17,7 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
+  NEXTEST_PROFILE: ci
 
 jobs:
   tests:
@@ -31,4 +32,4 @@ jobs:
       - uses: actions/checkout@v4
       - uses: taiki-e/install-action@nextest
       - name: Run UI tests, with ${{ matrix.feature_flag }}
-        run: mold --run cargo nextest run --no-fail-fast  -E 'test(ui)' --${{ matrix.feature_flag }}
+        run: mold --run cargo nextest run -E 'test(ui)' --${{ matrix.feature_flag }}
diff --git a/.gitignore b/.gitignore
@@ -51,3 +51,4 @@ result
 /lcov.info
 test_docker
 **/*.wasm
+.iroha_test_network_run.json*