diff --git a/.github/workflows/cargo_llvm_cov.yml b/.github/workflows/cargo_llvm_cov.yml index c8a57156136..15000ad8e5b 100644 --- a/.github/workflows/cargo_llvm_cov.yml +++ b/.github/workflows/cargo_llvm_cov.yml @@ -1,185 +1,154 @@ -name: Code Coverage +name: Code Coverage (llvm-cov) on: schedule: - - cron: "0 9 * * *" # UTC timing is every day at 1am PST + - cron: "0 1 * * *" # every day at 1am (UTC) workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - # Disable incremental compilation. - # - # Incremental compilation is useful as part of an edit-build-test-edit cycle, - # as it lets the compiler avoid recompiling code that hasn't changed. However, - # on CI, we're not making small edits; we're almost always building the entire - # project from scratch. Thus, incremental compilation on CI actually - # introduces *additional* overhead to support making future builds - # faster...but no future builds will ever occur in any given CI environment. - # - # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow - # for details. - CARGO_INCREMENTAL: 0 - # Allow more retries for network requests in cargo (downloading crates) and - # rustup (installing toolchains). This should help to reduce flaky CI failures - # from transient network timeouts or other issues. - CARGO_NET_RETRY: 10 - RUSTUP_MAX_RETRIES: 10 - # Don't emit giant backtraces in the CI logs. - RUST_BACKTRACE: short - # RUSTFLAGS: -D warnings - RUSTDOCFLAGS: -D warnings + inputs: + branch-or-commit: + description: "Branch or commit to test code coverage." + type: string + required: false + default: "" jobs: - cargo-llvm-cov: + llvm-cov: name: Generate code coverage - runs-on: [self-hosted] - timeout-minutes: 120 + runs-on: self-hosted + timeout-minutes: 180 env: + # Don't emit giant backtraces in the CI logs. + RUST_BACKTRACE: short + # Rustup + RUSTUP_MAX_RETRIES: 10 + # Cargo CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + # Find a good balance between runtime performance and accurate code coverage analysis. + CARGO_PROFILE_TEST_OPT_LEVEL: 1 + CARGO_PROFILE_TEST_DEBUG: false + CARGO_PROFILE_TEST_DEBUG_ASSERTIONS: false + CARGO_PROFILE_TEST_OVERFLOW_CHECKS: false + CARGO_PROFILE_TEST_LTO: off + CARGO_PROFILE_TEST_CODEGEN_UNITS: 1 + # Allow more retries for network requests in cargo (downloading crates) and + # rustup (installing toolchains). This should help to reduce flaky CI failures + # from transient network timeouts or other issues. + CARGO_NET_RETRY: 10 steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 - - uses: bmwill/rust-cache@fb63fcd7a959767755b88b5af2f5cbf65fb8a127 # pin@v1 - - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@125e82eef6e60d2b0c20d4d1f4cf81db37f68bf3 # pin@cargo-llvm-cov - - - name: Install nextest - uses: taiki-e/install-action@7e58f89e24a544d88f7a74c6eed8a3df3fd4c658 # pin@nextest - - - name: Set Swap Space - uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c # pin@master + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: - swap-size-gb: 256 + ref: ${{ github.event.inputs.branch-or-commit || github.ref }} - - name: Install Rust - run: rustup update stable + - uses: bmwill/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c # v1.4.0 - - name: Run code coverage for nextest - run: IOTA_SKIP_SIMTESTS=1 cargo llvm-cov --ignore-run-fail --lcov --output-path lcov.info nextest + - name: Install nextest and cargo-llvm-cov + uses: taiki-e/install-action@375e0c7f08a66b8c2ba7e7eef31a6f91043a81b0 # v2.44.38 + with: + tool: nextest,cargo-llvm-cov - - name: Upload report to Codecov for nextest - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # pin@v4 + - name: Set Swap Space + uses: actionhippie/swap-space@73376950a0019f8e1f6a3d3d2673fe2aed74ae17 # v1.0.2 with: - files: lcov.info + size: 256G - - name: Run code coverage for simtest + - name: Install llvm-tools + run: | + TOOLCHAIN=$(rustup show active-toolchain | cut -d ' ' -f 1) + rustup component add llvm-tools --toolchain "$TOOLCHAIN" + LLVM_PROFDATA="$HOME/.rustup/toolchains/$TOOLCHAIN/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata" + + if [ -x "$LLVM_PROFDATA" ]; then + $LLVM_PROFDATA --version + echo "LLVM_PROFDATA=$LLVM_PROFDATA" >> $GITHUB_ENV + else + echo "🚨 Error 🚨: llvm-profdata not found at $LLVM_PROFDATA, or is not executable." + exit 1 + fi + + - name: Run code coverage (nextest) + run: | + set +e + + echo "Generating coverage data (.profraw files)." + IOTA_SKIP_SIMTESTS=1 cargo llvm-cov --ignore-run-fail --no-report nextest + + echo "Scanning for corrupted .profraw files. This might take a while." + find target/llvm-cov-target -name '*.profraw' | while read file; do + if ! "$LLVM_PROFDATA" show "$file" > /dev/null 2>&1; then + echo "$file: corruped -> removing" + rm "$file" + fi + done + + echo "Trying to merge .profraw files." + find target/llvm-cov-target -name '*.profraw' -print0 | xargs -0 $LLVM_PROFDATA merge \ + --failure-mode=warn \ + --sparse \ + --output target/nextest.profdata + + if [ -s "target/nextest.profdata" ]; then + echo "Successfully collected nextest coverage data." + else + echo "🚨 Error 🚨: Collecting nexttest coverage failed." + exit 1 + fi + + - name: Run code coverage (simtest) run: | git clean -fd - ./scripts/simtest/codecov.sh - - - name: Upload report to Codecov for simtest - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # pin@v4 - with: - files: lcov-simtest.info - notify: - name: Notify - needs: [cargo-llvm-cov] - runs-on: self-hosted - if: always() # always notify + set +e + echo "Generating coverage data (.profraw files)." + ./scripts/simtest/simtest-cov.sh + + echo "Scanning for corrupted .profraw files. This might take a while." + find target/llvm-cov-target -name '*.profraw' | while read file; do + if ! "$LLVM_PROFDATA" show "$file" > /dev/null 2>&1; then + echo "$file: corruped -> removing" + rm "$file" + fi + done + + echo "Trying to merge .profraw files." + find target/llvm-cov-target -name '*.profraw' -print0 | xargs -0 $LLVM_PROFDATA merge \ + --failure-mode=warn \ + --sparse \ + --output target/simtest.profdata + + if [ -s "target/simtest.profdata" ]; then + echo "Successfully collected simtest coverage data." + else + echo "🚨 Warning 🚨: Collecting simtest coverage failed." + fi + + - name: Create report (lcov.info) + run: | + cd target - steps: - - uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # pin@v3 + if [ -s "simtest.profdata" ]; then + echo "Merging nextest and simtest coverage data." + $LLVM_PROFDATA merge --failure-mode=warn --sparse nextest.profdata simtest.profdata --output merged.profdata + else + echo "Using only nextest coverage data." + mv nextest.profdata merged.profdata + fi - - name: Checkout iota repo develop branch - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v4 + [ -s "merged.profdata" ] || exit 1 - - name: Get iota commit - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - export iota_sha=$(git rev-parse HEAD) - echo "iota_sha=${iota_sha}" >> $GITHUB_ENV + echo "Creating report (lcov.info)." + LLVM_PROFILE_FILE="merged.profdata" cargo llvm-cov report \ + --lcov \ + --output-path lcov.info \ + --ignore-filename-regex 'external-crates/.*' - - name: Get a branch name for a iota commit - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - export iota_branch_name=$(gh api -H 'Accept: application/vnd.github+json' /repos/iotaledger/iota/commits/${{ env.iota_sha }}/branches-where-head --jq '.[].name' | head -n 1) - # if the commit is not the head of the branch, get it's base branch - [[ -z $iota_branch_name ]] && export iota_branch_name=$(gh api -H 'Accept: application/vnd.github+json' /repos/iotaledger/iota/commits/${{ env.iota_sha }}/pulls --jq '.[].base.ref' | head -n 1) - echo "iota_branch_name=${iota_branch_name}" >> $GITHUB_ENV - echo "iota_branch_name_url=$(echo ${iota_branch_name} | sed 's\/\%2F\g')" >> $GITHUB_ENV - - - name: Get link to logs - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh_job_link=$(gh api -X GET 'repos/iotaledger/iota/actions/runs/${{ github.run_id }}/jobs' --jq '.jobs.[0].html_url') - echo "gh_job_link=${gh_job_link}" >> $GITHUB_ENV + echo "Removing absolute path prefix: ${{ github.workspace }} from report." + sed --in-place "s#${{ github.workspace }}#.#g" lcov.info - - name: Get current oncall - run: | - export current_oncall=$(curl -s --request GET \ - --url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \ - --header 'Accept: application/json' \ - --header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \ - --header 'Content-Type: application/json' \ - | jq '.oncalls[].user.summary' | tr -d '"') - echo "current_oncall=$(echo ${current_oncall})" >> $GITHUB_ENV - - export oncall_name=$(curl -s --request GET \ - --url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \ - --header 'Accept: application/json' \ - --header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \ - --header 'Content-Type: application/json' \ - | jq '.oncalls[].escalation_policy.summary' | tr -d '"') - echo "oncall_name=$(echo ${oncall_name})" >> $GITHUB_ENV - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # pin@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Get slack id for the oncall - run: | - export slack_id=$(aws s3 cp s3://iota-employees-dir/employees.json - | jq --arg ONCALL "${{ env.current_oncall }}" '.[] | if .name == $ONCALL then .slack_id else empty end') - echo "slack_id=$(echo ${slack_id} | tr -d '"')" >> $GITHUB_ENV - - - name: Post to slack - uses: slackapi/slack-github-action@936158bbe252e9a6062e793ea4609642c966e302 # pin@v1.21.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - IOTA_SHA: ${{ env.iota_sha }} - IOTA_BRANCH_NAME: ${{ env.iota_branch_name }} - IOTA_BRANCH_NAME_URL: ${{ env.iota_branch_name_url }} - GH_JOB_LINK: ${{ env.gh_job_link }} - SLACK_ID: ${{ env.slack_id }} - ONCALL_NAME: ${{ env.oncall_name }} + - name: Upload coverage to Coveralls + uses: coverallsapp/github-action@cfd0633edbd2411b532b808ba7a8b5e04f76d2c8 # v2.3.4 with: - channel-id: "code-coverage" - payload: | - { - "text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`" - } - }, - { - "type": "divider" - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "IOTA commit: \nIOTA branch: `${{ env.IOTA_BRANCH_NAME }}`\nRun: <${{ env.GH_JOB_LINK }}|${{ github.run_id }}>" - } - }, - { - "type": "divider" - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "<@${{ env.SLACK_ID }}>, current `${{ env.ONCALL_NAME }}` oncall, please look over the code coverage for the `${{ env.IOTA_BRANCH_NAME }}` branch in IOTA repo." - } - } - ] - } + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: target/lcov.info + flag-name: nextest+simtest + fail-on-error: true diff --git a/scripts/simtest/codecov.sh b/scripts/simtest/simtest-cov.sh similarity index 80% rename from scripts/simtest/codecov.sh rename to scripts/simtest/simtest-cov.sh index d0b1c57e2bb..a8701b87c0c 100755 --- a/scripts/simtest/codecov.sh +++ b/scripts/simtest/simtest-cov.sh @@ -6,6 +6,7 @@ # verify that git repo is clean if [[ -n $(git status -s) ]]; then echo "Working directory is not clean. Please commit all changes before running this script." + echo $(git status -s) exit 1 fi @@ -15,7 +16,7 @@ git apply ./scripts/simtest/config-patch root_dir=$(git rev-parse --show-toplevel) export SIMTEST_STATIC_INIT_MOVE=$root_dir"/examples/move/basics" -MSIM_WATCHDOG_TIMEOUT_MS=60000 MSIM_TEST_SEED=1 cargo llvm-cov --ignore-run-fail --lcov --output-path lcov-simtest.info nextest --cargo-profile simulator +MSIM_WATCHDOG_TIMEOUT_MS=60000 MSIM_TEST_SEED=1 cargo llvm-cov --ignore-run-fail --no-report nextest # remove the patch -git checkout .cargo/config Cargo.toml Cargo.lock +git checkout .cargo/config.toml Cargo.toml Cargo.lock