diff --git a/.dockerignore b/.dockerignore index f4ceea7856..93c3c5d6db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,5 @@ **/target/ +ci/ +.github/ +scripts/ +docker-compose* \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b525bfd297..b6d6611823 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,19 +1,19 @@ # Centrifudge Code Owners ## Changes to .github -.github/* @mikiquantum @mustermeiszer @branan @NunoAlexandre - +.github/* @mustermeiszer @branan @NunoAlexandre +.github/workflows @gpmayorga ## Changes to ci -ci/* @mikiquantum @mustermeiszer @branan @NunoAlexandre +ci/* @mustermeiszer @branan @NunoAlexandre ## Changes to the service of our chain. -node/* @mikiquantum @mustermeiszer @branan @NunoAlexandre +node/* @mustermeiszer @branan @NunoAlexandre ## Changes to chain-specs -node/res/* @mikiquantum @mustermeiszer @branan @NunoAlexandre @wischli +node/res/* @mustermeiszer @branan @NunoAlexandre @wischli ## Changes to toml files -*.toml @mikiquantum @mustermeiszer @branan @NunoAlexandre @lemunozm @wischli @cdamian @thea-leake +*.toml @mustermeiszer @branan @NunoAlexandre @lemunozm @wischli @cdamian @thea-leake ## Changes to specific pallets pallets/liquidity-pools/* @NunoAlexandre @cdamian @wischli @mustermeiszer @@ -41,9 +41,9 @@ libs/traits/src/changes.rs @lemunozm libs/traits/src/data.rs @lemunozm ## Changes to runtime -runtime/common/* @branan @mikiquantum @mustermeiszer @NunoAlexandre @offerijns @lemunozm -runtime/altair/* @branan @mikiquantum @mustermeiszer @NunoAlexandre @offerijns @wischli -runtime/centrifuge/* @branan @mikiquantum @mustermeiszer @NunoAlexandre @offerijns @wischli +runtime/common/* @branan @mustermeiszer @NunoAlexandre @offerijns @lemunozm +runtime/altair/* @branan @mustermeiszer @NunoAlexandre @offerijns @wischli +runtime/centrifuge/* @branan @mustermeiszer @NunoAlexandre @offerijns @wischli ## Changes to integration tests runtime/integration-tests/* @mustermeiszer @NunoAlexandre @wischli @cdamian diff --git a/.github/actions/prep-ubuntu/action.yml b/.github/actions/prep-ubuntu/action.yml new file mode 100644 index 0000000000..3069e3e0ed --- /dev/null +++ b/.github/actions/prep-ubuntu/action.yml @@ -0,0 +1,49 @@ +name: Prepare Ubuntu for Rust builds +description: cleanup and Rust Tools setup +inputs: + RUST_TOOLCHAIN: + description: toolchain version + default: "1.66" + cache: + description: cache type (enabled if set) + default: "disabled" + GWIP: + description: "Google Workload identity provider" + default: '' + GSA: + description: "Google Service Account" + default: '' +runs: + using: composite + steps: + - name: Prep build on Ubuntu + id: ubuntu_prep + shell: sh + run: | + echo "Pre cleanup" + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + echo "Post cleanup" + df -h + sudo apt-get install protobuf-compiler + + - name: Install latest nightly + uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 + with: + toolchain: ${{ inputs.RUST_TOOLCHAIN }} + default: true + + - name: SCcache setup + if: ${{ inputs.cache == 'enabled' }} + uses: ./.github/actions/sccache-gcloud + with: + GWIP: ${{ inputs.GWIP }} + GSA: ${{ inputs.GSA }} + + - name: setup Rust sccache wrapper + if: ${{ inputs.cache == 'enabled' }} + shell: sh + run: echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV + + diff --git a/.github/workflows/benchmark-check.yml b/.github/workflows/benchmark-check.yml deleted file mode 100644 index f661bcf5f7..0000000000 --- a/.github/workflows/benchmark-check.yml +++ /dev/null @@ -1,40 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Check Benchmarks -jobs: - tests: - name: ${{ matrix.target }} - strategy: - matrix: - os: [ubuntu-latest] - target: [benchmark-check] - runtime: [development, altair, centrifuge] - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-05-09" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: Run fast benchmarks - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - RUNTIME: ${{ matrix.runtime }} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b65bdbe0ed..c1057f5ee8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -6,11 +6,13 @@ jobs: benchmark: permissions: pull-requests: write + id-token: write name: benchmark runs-on: ubuntu-latest-4-cores if: ${{ startsWith(github.event.comment.body, '/benchmark ') && github.event.issue.pull_request && contains(github.event.comment.author_association, 'CONTRIBUTOR') }} env: - RUST_TOOLCHAIN: "nightly-2022-11-14" + RUST_TOOLCHAIN: "1.66.0" + RUSTC_WRAPPER: "sccache" steps: - uses: actions/github-script@v6 name: Parse Runtime From Comment @@ -34,15 +36,6 @@ jobs: body: `:clock1: Running Benchmarks for \`${{steps.parse-runtime.outputs.result}}\` :clock1:` }) return comment.data.id - - name: Prep build on Ubuntu - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - name: Get PR Hash id: get_pr_hash uses: actions/github-script@v6 @@ -60,21 +53,27 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ steps.get_pr_hash.outputs.result }} - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 + + - name: Prep build on Ubuntu + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ matrix.rust_version }} + + - name: SCcache setup + uses: ./.github/actions/sccache-gcloud with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} + - name: Run Benchmark ${{steps.parse-runtime.outputs.result}} - run: ./ci/script.sh - env: - TARGET: benchmark - RUNTIME: ${{steps.parse-runtime.outputs.result}} + run: ./scripts/runtime_benchmarks.sh ${{steps.parse-runtime.outputs.result}} + - name: Upload artifact uses: actions/upload-artifact@v3 with: name: ${{steps.parse-runtime.outputs.result}}-weights path: runtime/${{steps.parse-runtime.outputs.result}}/src/weights/ + - uses: actions/github-script@v6 name: Notify Benchmark finished with: diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000000..d1d20df52d --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,90 @@ +name: Docker Build +on: + push: + branches: [main, 'release-v**'] + pull_request: + paths: + - ".github/workflows/build-docker.yml" + - "Dockerfile" + workflow_dispatch: + +env: + RUST_TOOLCHAIN: "1.66" +jobs: + docker: + strategy: + matrix: + os: [ ubuntu-latest-4-cores ] + target: [ release, test ] + runs-on: ${{ matrix.os }} + env: + WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} + steps: + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Free space + # https://github.com/actions/runner-images/issues/2840#issuecomment-1284059930 + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - name: check available docker space + run: | + docker volume ls + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: DockerHub Login + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - run: echo "NOW=$(date -u +%Y%m%d)" >> $GITHUB_ENV + + - name: Setup docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: centrifugeio/centrifuge-chain + flavor: | + prefix=${{ matrix.target == 'test' && 'test-' || '' }} + suffix=${{ env.NOW }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=sha,format=short,prefix='git-' + + - name: Build and push centrifugeio/centrifuge-chain + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.ref == '/refs/heads/main' && true || false }} + tags: ${{ steps.meta.outputs.tags }} + # Cache options: + # https://docs.docker.com/build/ci/github-actions/cache/ + cache-from: type=registry,ref=centrifugeio/centrifuge-chain + # https://docs.docker.com/build/cache/backends/inline/ + cache-to: inline + + - name: Update DockerHub descriptions + if: contains(github.ref, 'refs/tags/release-v') + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + repository: centrifuge/centrifuge-chain + short-description: ${{ github.event.repository.description }} + enable-url-completion: true + + - if: failure() + name: Check available space after build failed + run: | + docker volume ls + df -h \ No newline at end of file diff --git a/.github/workflows/build-matrix.json b/.github/workflows/build-matrix.json deleted file mode 100644 index e8633688d3..0000000000 --- a/.github/workflows/build-matrix.json +++ /dev/null @@ -1,48 +0,0 @@ -[ - { - "os": "ubuntu-latest", - "target": "build-node", - "run_on_event": "any", - "rust_toolchain": "nightly-2022-11-14" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime", - "package": "altair-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime", - "package": "centrifuge-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime-fast", - "package": "altair-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime-fast", - "package": "centrifuge-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime-testnet", - "package": "altair-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - } -] diff --git a/.github/workflows/build-nix.yml b/.github/workflows/build-nix.yml deleted file mode 100644 index cbb02cf1e7..0000000000 --- a/.github/workflows/build-nix.yml +++ /dev/null @@ -1,25 +0,0 @@ -#### NOTE: This job is disabled until nix supports the latest cargo workspace features. -#### See: https://github.com/NixOS/nixpkgs/pull/217232 -# -#name: nix-build -#on: -# pull_request: -# push: -# branches: [main, 'release-v**'] -#jobs: -# build: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# with: -# # Nix Flakes doesn't work on shallow clones -# fetch-depth: 0 -# - uses: cachix/install-nix-action@v17 -# with: -# install_url: https://releases.nixos.org/nix/nix-2.10.3/install -# - uses: cachix/cachix-action@v10 -# with: -# name: centrifuge-chain -# authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" -# - name: Build centrifuge-chain -# run: nix build --no-allow-dirty -L \ No newline at end of file diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml new file mode 100644 index 0000000000..0003ae19a7 --- /dev/null +++ b/.github/workflows/build-wasm.yml @@ -0,0 +1,96 @@ +on: + push: + branches: [main, 'release-v**'] + pull_request: +name: Build&Publish WASM +concurrency: + group: 'build-wasm-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true +jobs: + build-runtime-wasms: + environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} + # Benchmark (with cache hit) + # ubuntu-latest 50 min + # ubuntu-latest-4-cores 34 min + runs-on: ubuntu-latest-8-cores + strategy: + matrix: + # To test until I get one right + # target: [build-runtime, build-runtime-fast, build-runtime-testnet] + # package: [centrifuge-runtime, altair-runtime] + target: [build-runtime, build-runtime-testnet, build-runtime-fast] + package: [altair-runtime, centrifuge-runtime] + rust_version: ["1.66.0"] + exclude: + - target: "build-runtime-testnet" + package: "centrifuge-runtime" + name: ${{ matrix.target }}-${{ matrix.package }} + # Cannot make sccache work with the srtool container + # env: + # SCCACHE_GHA_ENABLED: "true" + # RUSTC_WRAPPER: "sccache" + steps: + - name: Check out code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 + + - name: Prep build on Ubuntu + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ matrix.rust_version }} + + # sccache does not work inside the srtool container + - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # 2.6.2 + with: + prefix-key: build #-${{ matrix.package }} + shared-key: wasm + # save-if: ${{ github.ref == 'refs/heads/master' }} + + - name: Setup build options + id: buildopt + run: | + if ${{ matrix.target == 'build-runtime-testnet'}} ; then + echo "BUILD_OPTS="--features=testnet-runtime"" >> GITHUB_ENV + elif ${{ matrix.target == 'build-runtime-fast'}} ; then + echo "BUILD_OPTS="--features=fast-runtime"" >> GITHUB_ENV + fi + + - name: Run Docker SRTool + uses: addnab/docker-run-action@v3 + env: + RUSTC_VERSION: ${{ matrix.rust_version }} + BUILD_OPTS: ${{ env.BUILD_OPTS }} + with: + image: paritytech/srtool:${{ matrix.rust_version }} + options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e PACKAGE=${{ matrix.package }} + run: | + echo "BUILD_OPT=${BUILD_OPT}" + echo "PACKAGE=${PACKAGE}" + echo "---- List Cache Folder ----" + ls -la /cargo-home/ + du -sh /cargo-home/* + echo "---- Rust Versions ----" + rustc --version + rustup --version + cargo --version + echo "---- RUNNING BUILD ----" + /srtool/build + + - id: 'auth' + uses: 'google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033' # v1.1.1 + with: + workload_identity_provider: ${{ secrets.GWIP_ARTIFACTS }} + service_account: ${{ secrets.GSA_ARTIFACTS }} + + - name: Setup - gcloud / gsutil + uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 + + + - name: Publish artifacts to GCS + # if: ${{ github.ref == 'refs/heads/main' }} + shell: bash + run: | + target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) + runtime_name=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//" ) + gsutil cp \ + ./runtime/$runtime_name/target/srtool/release/wbuild/${{ matrix.package }}/${{ matrix.package }}.compact.compressed.wasm \ + gs://centrifuge-wasm-repo/$RUNTIME/$target/$runtime_name-$target-$(git rev-parse --short HEAD).wasm diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index a01782f0a5..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,86 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Build -jobs: - matrix_prep: - name: matrix-prep - runs-on: 'ubuntu-latest' - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - # Required as the JSON input file needs to be read - - uses: actions/checkout@v3 - - id: set-matrix - uses: JoshuaTheMiller/conditional-build-matrix@9ff11e2cad4aa166e1d1d834097121a969da8276 - with: - inputFile: '.github/workflows/build-matrix.json' - # we run build-node on every workflow but build-runtime only on pushes to branches - filter: '[?run_on_event==`any` || run_on_event==`${{ github.event_name }}`]' - build: - needs: matrix_prep - strategy: - matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} - name: ${{ matrix.target }} - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - if: startsWith(matrix.target, 'build-runtime') - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - with: # cache the runtime target instead of root target for build-runtime - working-directory: './runtime' - target-dir: './target/srtool' - - if: matrix.target == 'build-node' - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: ${{ matrix.target }} - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - RUST_TOOLCHAIN: ${{ matrix.rust_toolchain }} - PACKAGE: ${{matrix.package}} - - if: startsWith(matrix.target, 'build-runtime') - name: Setup - gcloud / gsutil - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.GCS_SA_KEY }} - project_id: ${{ secrets.GCS_PROJECT }} - export_default_credentials: true - - if: ${{ matrix.target == 'build-runtime' && matrix.package == 'altair-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/altair/target/srtool/release/wbuild/altair-runtime/altair_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/parachain/altair_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime-fast' && matrix.package == 'altair-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/altair/target/srtool/release/wbuild/altair-runtime/altair_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/test-parachain/altair_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime-testnet' && matrix.package == 'altair-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/altair/target/srtool/release/wbuild/altair-runtime/altair_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/parachain/algol-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime' && matrix.package == 'centrifuge-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/centrifuge/target/srtool/release/wbuild/centrifuge-runtime/centrifuge_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/parachain/centrifuge_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime-fast' && matrix.package == 'centrifuge-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/centrifuge/target/srtool/release/wbuild/centrifuge-runtime/centrifuge_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/test-parachain/centrifuge_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm \ No newline at end of file diff --git a/.github/workflows/docker-nix.yml b/.github/workflows/docker-nix.yml deleted file mode 100644 index 31e285cedb..0000000000 --- a/.github/workflows/docker-nix.yml +++ /dev/null @@ -1,31 +0,0 @@ -# name: docker -# on: -# push: -# branches: [main, 'release-v**'] -# jobs: -# docker: -# strategy: -# matrix: -# os: [ ubuntu-latest ] -# target: [".#dockerImage", ".#dockerImageFastRuntime"] -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# - uses: cachix/install-nix-action@v17 -# with: -# install_url: https://releases.nixos.org/nix/nix-2.10.3/install -# - name: Build Docker image -# run: | -# nix build --no-allow-dirty -L ${{ matrix.target }} -# docker load < result -# - name: Login to Docker Hub -# uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 -# with: -# username: ${{ secrets.DOCKER_HUB_USERNAME }} -# password: ${{ secrets.DOCKER_HUB_TOKEN }} -# - name: List images -# run: docker images -# - name: Push image to Docker Hub -# run: docker push --all-tags --quiet centrifugeio/centrifuge-chain diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 8c496ec10f..0000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: docker -on: - push: - branches: [main, 'release-v**'] - pull_request: - paths: - - ".github/workflows/docker.yml" - workflow_dispatch: - inputs: - docker_tag: - description: 'Docker Image tag' - required: true - default: '' -env: - RUST_TOOLCHAIN: "nightly-2022-11-14" -jobs: - docker: - strategy: - matrix: - os: [ ubuntu-latest-4-cores ] - target: [ release, test ] - runs-on: ${{ matrix.os }} - env: - WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Free space - # https://github.com/actions/runner-images/issues/2840#issuecomment-1284059930 - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: check available docker space - run: | - docker volume ls - df -h - - - name: Set docker tag - run: | - if [ "${WORKFLOW_TAG}" != "" ]; then - echo "DOCKER_TAG=${WORKFLOW_TAG}" >> $GITHUB_ENV - else - echo "DOCKER_TAG=$(echo $GITHUB_REF | cut -d'/' -f 3)" >> $GITHUB_ENV - fi - - if: matrix.target == 'release' - name: Build docker release - run: docker build --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -t centrifugeio/centrifuge-chain:${{ env.DOCKER_TAG }}-latest . - - if: matrix.target == 'test' - name: Build docker test - run: docker build --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} --build-arg OPTS="--features=fast-runtime" -t centrifugeio/centrifuge-chain:test-${{ env.DOCKER_TAG }}-latest . - - if: failure() - name: Check available space after build failed - run: | - docker volume ls - df -h - - name: Login to Docker Hub - uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - if: matrix.target == 'release' - name: Tag image release - run: docker tag centrifugeio/centrifuge-chain:${{ env.DOCKER_TAG }}-latest "centrifugeio/centrifuge-chain:${{ env.DOCKER_TAG }}-$(date -u +%Y%m%d%H%M%S)-$(git rev-parse --short HEAD)" - - if: matrix.target == 'test' - name: Tag image test - run: docker tag centrifugeio/centrifuge-chain:test-${{ env.DOCKER_TAG }}-latest "centrifugeio/centrifuge-chain:test-${{ env.DOCKER_TAG }}-$(date -u +%Y%m%d%H%M%S)-$(git rev-parse --short HEAD)" - - name: List images - run: docker images - - name: Push image to Docker Hub - run: docker push --all-tags --quiet centrifugeio/centrifuge-chain - if: contains(github.ref, 'refs/tags/release-v') || github.ref == 'refs/heads/main' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4c8539c6da..bcf1217ca2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,37 +7,29 @@ jobs: docs: permissions: contents: write - runs-on: ubuntu-latest + id-token: write + runs-on: ubuntu-latest-8-cores env: - RUST_TOOLCHAIN: "nightly-2022-11-14" + RUST_TOOLCHAIN: "1.66.0" RUSTDOCFLAGS: "-D warnings" + RUSTC_WRAPPER: "sccache" steps: - name: Checkout repository uses: actions/checkout@v3 - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 + uses: ./.github/actions/prep-ubuntu with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true + RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + cache: enabled + # Cache needs Google credentials: + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} - - name: Build Documentation + - name: Build documentation id: build_docs - uses: actions-rs/cargo@v1 - with: - command: doc - args: --all --no-deps + run: cargo doc --all --no-deps + - name: Build Documentation failed if: always() && steps.build_docs.outcome == 'failure' @@ -46,10 +38,11 @@ jobs: - name: Create Index file if: github.ref == 'refs/heads/main' - run: ./ci/create_index_for_rust_docs.sh + run: | + echo "" \ > ./target/doc/index.html - name: Deploy Docs - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # 3.9.3 if: github.ref == 'refs/heads/main' with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml deleted file mode 100644 index abc9c8fc43..0000000000 --- a/.github/workflows/lints.yml +++ /dev/null @@ -1,39 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Lints -jobs: - lints: - name: ${{ matrix.target }} - strategy: - matrix: - os: [ubuntu-latest] - target: [fmt, clippy] - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: Run lints - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml new file mode 100644 index 0000000000..8c7a3aad82 --- /dev/null +++ b/.github/workflows/sanity-checks.yml @@ -0,0 +1,69 @@ +on: + push: + branches: [main, 'release-v**'] + pull_request: + paths: + - 'src/' + - '.github/workflows/sanity-checks.yml' +name: Sanity checks +concurrency: + group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true +permissions: + id-token: write + contents: read +jobs: + test-n-lint: + name: ${{ matrix.target }} + runs-on: ubuntu-latest-4-cores + strategy: + matrix: + target: [test-general, test-integration, + lint-fmt, lint-clippy, cargo-build] # ,lint-taplo] + env: + RUST_TOOLCHAIN: "nightly-2022-11-14" + RUSTC_WRAPPER: "sccache" + steps: + - name: Check out code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 + + - name: Prep build + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + cache: enabled + # Cache needs Google credentials: + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} + + - name: Runing cargo ${{ matrix.target }} + run: ./ci/run-check.sh + env: + TARGET: ${{ matrix.target }} + + benchmark-check: + name: bench-${{ matrix.runtime }} + runs-on: ubuntu-latest #r-4-cores + strategy: + matrix: + runtime: [development, altair, centrifuge] + env: + RUST_TOOLCHAIN: "nightly-2022-11-14" + steps: + - name: Check out code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 + + - name: Prep build + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + setup_cache: true + cache: enabled + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} + + - name: Runing cargo ${{ matrix.target }} + run: ./ci/run-check.sh + env: + TARGET: benchmark-check + RUNTIME: ${{ matrix.runtime }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index d00c4333d6..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,39 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Tests -jobs: - tests: - name: ${{ matrix.target }} - strategy: - matrix: - os: [ubuntu-latest] - target: [tests, integration] - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: Run tests - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - diff --git a/Dockerfile b/Dockerfile index 0dcf4af01c..d78a10ec4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,58 +1,81 @@ -# Based from https://github.com/paritytech/substrate/blob/master/.maintain/Dockerfile +# Based on +# https://github.com/paritytech/polkadot-sdk/blob/master/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile +FROM docker.io/library/ubuntu:22.04 as builder -FROM phusion/baseimage:focal-1.0.0 as builder -LABEL maintainer="Centrifuge Team" -LABEL description="This is the build stage for the Centrifuge Chain client. Here the binary is created." + # Defaults + ENV RUST_BACKTRACE 1 + ENV DEBIAN_FRONTEND=noninteractive + ENV RUST_TOOLCHAIN=$RUST_TOOLCHAIN + ARG FEATURES="" + ARG RUST_TOOLCHAIN="1.66" + + RUN apt-get update && \ + # apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ + apt-get install -y \ + cmake \ + pkg-config \ + libssl-dev \ + git \ + clang \ + libclang-dev \ + protobuf-compiler \ + curl + + RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + ENV PATH="${PATH}:/root/.cargo/bin" + + # BUILD + COPY . centrifuge-chain + WORKDIR /centrifuge-chain + RUN echo $(ls -l /centrifuge-chain/) -ARG RUST_TOOLCHAIN=nightly -ENV DEBIAN_FRONTEND=noninteractive -ENV RUST_TOOLCHAIN=$RUST_TOOLCHAIN + RUN rustup default $RUST_TOOLCHAIN && \ + rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN && \ + cargo build "--release" --features=${FEATURES} -ARG PROFILE=release -ARG OPTS="" -WORKDIR /centrifuge-chain - -COPY . /centrifuge-chain - -RUN apt-get update && \ - apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ - apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev protobuf-compiler +# ===== SECOND STAGE ====== -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ - export PATH="$PATH:$HOME/.cargo/bin" && \ - rustup default $RUST_TOOLCHAIN && \ - rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN && \ - cargo build "--$PROFILE" $OPTS +FROM docker.io/library/ubuntu:22.04 -# ===== SECOND STAGE ====== + LABEL io.centrifuge.image.authors="guillermo@k-f.co" \ + io.centrifuge.image.vendor="Centrifuge" \ + io.centrifuge.image.title="centrifugeio/centrifuge-chain" \ + io.centrifuge.image.description="Centrifuge, the layer 1 of RWA. This is the official Centrifuge image with an injected binary." \ + io.centrifuge.image.source="https://github.com/centrifuge/centrifuge-chain/blob/main/Dockerfile" \ + # io.centrifuge.image.revision="${VCS_REF}" \ + io.centrifuge.image.created="${BUILD_DATE}" -FROM phusion/baseimage:focal-1.0.0 -LABEL maintainer="Centrifuge Team" -LABEL description="This is the 2nd stage: a very small image that contains the centrifuge-chain binary and will be used by users." -ARG PROFILE=release + COPY --from=builder /centrifuge-chain/target/release/centrifuge-chain /usr/local/bin -RUN mv /usr/share/ca* /tmp && \ - rm -rf /usr/share/* && \ - mv /tmp/ca-certificates /usr/share/ && \ - mkdir -p /root/.local/share/centrifuge-chain && \ - ln -s /root/.local/share/centrifuge-chain /data - # && \ - # useradd -m -u 1000 -U -s /bin/sh -d /centrifuge-chain centrifuge-chain # commented out since users do not seem to work with PVCs we currently use: https://stackoverflow.com/questions/46873796/allowing-access-to-a-persistentvolumeclaim-to-non-root-user/46907452 + RUN useradd -m -u 1000 -U -s /bin/sh -d /centrifuge centrifuge && \ + mkdir -p /data /centrifuge/.local/share && \ + chown -R centrifuge:centrifuge /data && \ + ln -s /data /centrifuge/.local/share/centrifuge -COPY --from=builder /centrifuge-chain/target/$PROFILE/centrifuge-chain /usr/local/bin + # checks + RUN ldd /usr/local/bin/centrifuge-chain && \ + /usr/local/bin/centrifuge-chain --version -# checks -RUN ldd /usr/local/bin/centrifuge-chain && \ - /usr/local/bin/centrifuge-chain --version + # Save sh and bash + RUN cp /usr/bin/sh /usr/bin/bash /usr/local/bin/ /root/ -# Shrinking -RUN rm -rf /usr/lib/python* && \ - rm -rf /usr/bin /usr/sbin /usr/share/man + # Unclutter + RUN mv /usr/share/ca* /tmp && \ + rm -rf /usr/share/* && \ + mv /tmp/ca-certificates /usr/share/ && \ + mkdir -p /root/.local/share/centrifuge-chain && \ + ln -s /root/.local/share/centrifuge-chain /data \ + # minimize the attack surface + rm -rf /usr/bin /usr/sbin && \ + rm -rf /usr/lib/python* && \ + # check if executable works in this container + ldd /usr/local/bin/centrifuge-chain && \ + /usr/local/bin/centrifuge-chain --version -# Add chain resources to image -COPY res /resources/ + # Add chain resources to image + COPY res /resources/ -# USER centrifuge-chain # see above +USER centrifuge EXPOSE 30333 9933 9944 VOLUME ["/data"] diff --git a/ci/run-check.sh b/ci/run-check.sh new file mode 100755 index 0000000000..5434dbb3a0 --- /dev/null +++ b/ci/run-check.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -eux + +# Enable warnings about unused extern crates +# export RUSTFLAGS=" -W unused-extern-crates" +rustc --version +rustup --version +cargo --version + +case $TARGET in + cargo-build) + cargo build --release "$@" + ;; + + test-general) + RUST_MIN_STACK=8388608 cargo test --workspace --release --features runtime-benchmarks,try-runtime --exclude runtime-integration-tests + ;; + + test-integration) + RUST_MIN_STACK=8388608 cargo test --release --package runtime-integration-tests --features fast-runtime + ;; + + lint-fmt) + cargo fmt -- --check + ;; + + lint-taplo) + taplo fmt --check + ;; + + lint-clippy) + cargo clippy --workspace -- -D warnings -A clippy::unnecessary-cast -A clippy::bool-to-int-with-if + ;; + benchmark-check) + ./scripts/check_benchmarks.sh $RUNTIME +esac diff --git a/ci/script.sh b/ci/script.sh deleted file mode 100755 index 7b576d984e..0000000000 --- a/ci/script.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-nightly-2023-02-07}" -SRTOOL_VERSION="${SRTOOL_VERSION:-1.66.1-0.9.25}" -PACKAGE="${PACKAGE:-centrifuge-runtime}" # Need to replicate job for all runtimes -RUNTIME="${RUNTIME:-centrifuge}" - -# Enable warnings about unused extern crates -export RUSTFLAGS=" -W unused-extern-crates" - -./scripts/init.sh install-toolchain - -rustc --version -rustup --version -cargo --version - -case $TARGET in - build-node) - cargo build --release "$@" - ;; - - build-runtime) - export RUSTC_VERSION=$RUST_TOOLCHAIN - docker run --rm -e PACKAGE=$PACKAGE -v $PWD:/build -v /tmp/cargo:/cargo-home paritytech/srtool:$SRTOOL_VERSION build - ;; - - build-runtime-fast) - export RUSTC_VERSION=$RUST_TOOLCHAIN - docker run --rm -e PACKAGE=$PACKAGE -e BUILD_OPTS="--features=fast-runtime" -v $PWD:/build -v /tmp/cargo:/cargo-home paritytech/srtool:$SRTOOL_VERSION build - ;; - - build-runtime-testnet) - export RUSTC_VERSION=$RUST_TOOLCHAIN - docker run --rm -e PACKAGE=$PACKAGE -e BUILD_OPTS="--features=testnet-runtime" -v $PWD:/build -v /tmp/cargo:/cargo-home paritytech/srtool:$SRTOOL_VERSION build - ;; - - tests) - RUST_MIN_STACK=8388608 cargo test --workspace --release --features runtime-benchmarks,try-runtime --exclude runtime-integration-tests - ;; - - integration) - RUST_MIN_STACK=8388608 cargo test --release --package runtime-integration-tests --features fast-runtime - ;; - - fmt) - cargo fmt -- --check - ;; - - taplo) - taplo fmt --check - ;; - - clippy) - cargo clippy --workspace -- -D warnings -A clippy::unnecessary-cast -A clippy::bool-to-int-with-if - ;; - - benchmark) - ./scripts/runtime_benchmarks.sh $RUNTIME - ;; - - benchmark-check) - ./scripts/check_benchmarks.sh $RUNTIME - -esac