From 8398d7a10c091443995d65cec1cc13ba49147b59 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 16 Nov 2023 18:51:02 +0100 Subject: [PATCH] Final review of CI changes --- .github/workflows/build-docker.yml | 18 +++----- .github/workflows/build-wasm.yml | 26 +++-------- .github/workflows/run-benchmarks.yml | 37 +--------------- .github/workflows/sanity-checks.yml | 33 +++----------- .../xperimental-codecov.yml.commented | 44 +++++++++++++++++++ 5 files changed, 64 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/xperimental-codecov.yml.commented diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 7e00931317..9bf12a38ae 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -1,12 +1,14 @@ name: Docker Build on: + # Keep in mind the Docker tagging on the "metadata" step if you add new triggers push: - branches: [main, 'release-v**'] + branches: [main] + release: + types: [published] pull_request: paths: - ".github/workflows/build-docker.yml" - "docker/centrifuge-chain/Dockerfile" - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}${{ github.ref == '/refs/heads/main' && github.sha || 'false' }} cancel-in-progress: true @@ -16,8 +18,6 @@ jobs: matrix: target: [ release, test ] runs-on: ubuntu-latest-8-cores - env: - WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} steps: - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 with: @@ -60,9 +60,8 @@ jobs: prefix=${{ matrix.target == 'test' && 'test-' || 'latest=auto' }} tags: | type=ref,event=branch - type=ref,event=tag - type=semver,pattern={{raw}} - type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }}pr- + type=semver,pattern={{raw}},suffix= + type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }} type=edge - name: Configure sccache @@ -74,16 +73,11 @@ jobs: - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 #v5 - # env: - # BUILDKIT_PROGRESS: plain - # DOCKER_BUILDKIT: 1 with: context: . file: ./docker/centrifuge-chain/Dockerfile build-args: | FEATURES=${{ matrix.target == 'test' && 'fast-runtime' || '' }} - # SCCACHE_GHA_ENABLED="true" - # RUSTC_WRAPPER=sccache push: ${{ github.ref == '/refs/heads/main' && true || false }} tags: ${{ steps.meta.outputs.tags }} # Cache options: diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index d60f9028e2..655b074db6 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -14,25 +14,16 @@ permissions: contents: read jobs: build-runtime-wasms: - # environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} - environment: production + environment: production # It needs the production GCP credentials to upload WASM # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min runs-on: ubuntu-latest-4-cores strategy: matrix: - # To test until I get one right - # target: [build-runtime, build-runtime-fast, build-runtime-testnet] - # package: [centrifuge-runtime, altair-runtime] target: [build-runtime, build-runtime-fast] package: [altair-runtime, centrifuge-runtime] - # rust_version: ["1.69.0"] name: ${{ matrix.target }}-${{ matrix.package }} - # Cannot make sccache work with the srtool container - # env: - # SCCACHE_GHA_ENABLED: "true" - # RUSTC_WRAPPER: "sccache" steps: # PREP - name: Check out code @@ -40,9 +31,6 @@ jobs: - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu - # Use default toolchain - # with: - # RUST_TOOLCHAIN: ${{ matrix.rust_version }} # sccache does not work inside the srtool container - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # 2.6.2 @@ -55,8 +43,7 @@ jobs: cache-directories: ./runtime # save-if: ${{ github.ref == 'refs/heads/master' }} # BUILD - - - id: buildopt + - id: buildopt run: | if ${{ matrix.target == 'build-runtime-fast'}} ; then echo "BUILD_OPTS=--features=fast-runtime" >> GITHUB_ENV @@ -68,13 +55,14 @@ jobs: docker run --rm --user root --platform=linux/amd64 -e PACKAGE=${{ matrix.package }} -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build paritytech/srtool:1.73.0-0.12.0 - /srtool/build + /srtool/build --app + # -a, --app + # Enable the "app" mode which is a mix of json output and outputting progress during the build. This flag is recommended for CI. the json output will be provided as a single line at the end in compact mode + # Alternative way of running SRTool that allows for "script-like" execution, + # extremely useful to debug: # - name: Run Docker SRTool # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 - # env: - # RUSTC_VERSION: ${{ matrix.rust_version }} - # TARPAULIN_VERSION: 0.19.1 # travis-ci codecov generator # with: # image: paritytech/srtool:${{ matrix.rust_version }} # options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e BUILD_OPTS=${{ env.BUILD_OPTS || ''}} -e PACKAGE=${{ matrix.package }} diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 1d1e437a5c..94b83fe61e 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -1,10 +1,7 @@ name: Run benchmarks # ToDo: add an arbitrary label to keep this cache separate from the builds on: - workflow_dispatch: - # Manually triggered by an authorized repo contributor. - push: - branches: main # To build the "benchmark-only" cache + workflow_dispatch: # Manually triggered by an authorized repo contributor. pull_request: paths: - '.github/workflows/run-benchmarks.yml' @@ -14,16 +11,12 @@ concurrency: jobs: benchmark: permissions: - pull-requests: write - contents: write + contents: read id-token: write runs-on: ubuntu-latest-8-cores strategy: matrix: runtimes: [centrifuge, altair] - env: - # RUST_TOOLCHAIN: "1.66.0" - RUSTC_WRAPPER: "sccache" steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 @@ -31,17 +24,11 @@ jobs: - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu with: - # RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} cache: enabled # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} - # - name: Checkout code - # uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 - # with: - # ref: 'main' - - name: Run Benchmark ${{ matrix.runtimes }} run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} @@ -52,23 +39,3 @@ jobs: with: name: ${{ matrix.runtimes }}-weights path: runtime/${{ matrix.runtimes }}/src/weights/ - -# Disabled a the org level -# fails with 'GitHub Actions is not permitted to create or approve pull requests.' - # - name: Create PR with new benchmarks - # uses: peter-evans/create-pull-request@v5 - # with: - # add-paths: runtime/${{ matrix.runtimes }}/src/weights/ - # base: main - # branch: benchmarks/${{ matrix.runtimes }}-${{ env.NOW }} - # delete-branch: true - # reviewers: ${{ github.actor }} - # draft: true - # labels: gha, benchmarks - # commit-message: "New benchmark weights generated from main by Github Actions Bot" - # title: New benchmark weights for ${{ matrix.runtimes }} - # body: | - # Automated PR generated by Benchmark PR Github Actions job - # Runtime: ${{ matrix.runtimes }} - # Commit that triggered: - # ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index c731405830..838ebf3458 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -2,9 +2,9 @@ on: # push: # branches: [main, 'release-v**'] pull_request: - paths: - - 'src/' - - '.github/workflows/sanity-checks.yml' + # paths: + # - 'src/' + # - '.github/workflows/sanity-checks.yml' name: Sanity checks concurrency: group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' @@ -27,38 +27,16 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - # RUST_TOOLCHAIN: "nightly-2023-02-07" cache: enabled # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} - # Tarpaulin requires cargo 1.70.0: - # - name: generate codecov reports - # run: | - # if ${{ contains(matrix.target, 'test' ) }}; then - # echo "---- GENERATE CODE COVERAGE ----" - # echo "# Install Tarpaulin" - # cargo install --locked cargo-tarpaulin - # # make Cargo.toml - # echo "Generate code coverage for ${{ matrix.target }}" - # cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml - # fi + - name: Runing cargo ${{ matrix.target }} run: ./ci/run-check.sh env: TARGET: ${{ matrix.target }} - RUSTC_WRAPPER: "sccache" - - # UPLOAD REPORTS (requires cargo 1.70.0) - # - name: Upload codecov report - # uses: codecov/codecov-action@v3 - # with: - # # token: ${{ secrets.CODECOV_TOKEN }} - # # files: ./coverage1.xml,./coverage2.xml # optional - # # flags: unittests # optional - # # name: codecov-umbrella # optional - # # fail_ci_if_error: true # optional (default = false) - # verbose: true # optional (default = false) + RUSTC_WRAPPER: "sccache" benchmark-check: name: bench-check-${{ matrix.runtime }} @@ -73,7 +51,6 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - # RUST_TOOLCHAIN: "nightly-2022-11-14" cache: enabled GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} diff --git a/.github/workflows/xperimental-codecov.yml.commented b/.github/workflows/xperimental-codecov.yml.commented new file mode 100644 index 0000000000..6600aff7c0 --- /dev/null +++ b/.github/workflows/xperimental-codecov.yml.commented @@ -0,0 +1,44 @@ +## THIS IS A WORK IN PROGRESS +on: + # push: + # branches: [main, 'release-v**'] + pull_request: +name: Sanity checks +concurrency: + group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true +permissions: + id-token: write + contents: read +jobs: + codecov: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + # Tarpaulin requires cargo 1.70.0: + - name: generate codecov reports + run: | + if ${{ contains(matrix.target, 'test' ) }}; then + echo "---- GENERATE CODE COVERAGE ----" + echo "# Install Tarpaulin" + cargo install --locked cargo-tarpaulin + # make Cargo.toml + echo "Generate code coverage for ${{ matrix.target }}" + cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml + fi + - name: Runing cargo ${{ matrix.target }} + run: ./ci/run-check.sh + env: + TARGET: ${{ matrix.target }} + RUSTC_WRAPPER: "sccache" + + # UPLOAD REPORTS (requires cargo 1.70.0) + - name: Upload codecov report + uses: codecov/codecov-action@v3 + with: + # token: ${{ secrets.CODECOV_TOKEN }} + # files: ./coverage1.xml,./coverage2.xml # optional + # flags: unittests # optional + # name: codecov-umbrella # optional + # fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false)