diff --git a/.github/actions/librocksdb/action.yaml b/.github/actions/librocksdb/action.yaml new file mode 100644 index 0000000000..895185f43d --- /dev/null +++ b/.github/actions/librocksdb/action.yaml @@ -0,0 +1,55 @@ +--- +# This action builds and caches librocksdb. If we find that this solution consumes too much time, we can consider +# prebuilding librocksdb outside of the pipeline (eg. in the grovedb release process), publish as an artifact, and +# download it in the pipeline. +name: "librocksdb" +description: "Build and install librocksdb" +inputs: + version: + description: RocksDB version, eg. "8.10.2" + required: false + default: "8.10.2" + bucket: + description: S3 bucket to use for caching + required: false + default: multi-runner-cache-x1xibo9c + force: + description: Force rebuild + required: false + default: "false" + +runs: + using: composite + steps: + # Cache librocksdb using s3 bucket + - name: Restore cached librocksdb from S3 + id: librocksdb-cache + uses: strophy/actions-cache@opendal-update + with: + bucket: ${{ inputs.bucket }} + path: /opt/rocksdb + key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }} + + - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' || inputs.force == 'true' }} + shell: bash + name: Build librocksdb + run: | + set -ex + WORKDIR=/tmp/rocksdb-build + mkdir -p ${WORKDIR}/rocksdb + mkdir -p /opt/rocksdb/usr/local/lib/ + pushd ${WORKDIR}/rocksdb + + # building rocksdb + git clone https://github.com/facebook/rocksdb.git -b v${{ inputs.version }} --depth 1 . + make -j$(nproc) static_lib + make DESTDIR=/opt/rocksdb install-static + set +x + + echo Done. + echo Configuration: + echo + echo "ROCKSDB_STATIC='/opt/rocksdb/usr/local/lib/librocksdb.a'" + echo "ROCKSDB_LIB_DIR='/opt/rocksdb/usr/local/lib'" + + popd diff --git a/.github/workflows/cached.yml b/.github/workflows/cached.yml new file mode 100644 index 0000000000..12eed8dd3e --- /dev/null +++ b/.github/workflows/cached.yml @@ -0,0 +1,23 @@ +--- +name: Rebuild cached dependencies + +on: + workflow_dispatch: +jobs: + build-rust-deps: + name: Prebuild and cache some Rust dependencies + runs-on: ubuntu-24.04 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Precompile librocksdb + uses: ./.github/actions/librocksdb + with: + force: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 12f512b720..d8e24ef706 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,7 +7,7 @@ on: jobs: build: name: Deploy docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout main uses: actions/checkout@v4 diff --git a/.github/workflows/manage-runs.yml b/.github/workflows/manage-runs.yml index 2c07af5b4f..29bc43aa53 100644 --- a/.github/workflows/manage-runs.yml +++ b/.github/workflows/manage-runs.yml @@ -7,7 +7,7 @@ on: jobs: cancel-merged-or-closed-pr-runs: name: Cancel runs for merged or closed PRs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: octokit/request-action@v2.x id: get_active_workflows diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d475a3eef8..e75151c841 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -10,7 +10,7 @@ on: jobs: pr-title: name: PR title - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Validate conventional PR title uses: amannn/action-semantic-pull-request@v5 diff --git a/.github/workflows/release-docker-image.yml b/.github/workflows/release-docker-image.yml index 5c56ade272..54e4cb465a 100644 --- a/.github/workflows/release-docker-image.yml +++ b/.github/workflows/release-docker-image.yml @@ -89,7 +89,7 @@ jobs: publish-manifest: name: Publish image tags needs: build-image - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Download digests uses: actions/download-artifact@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dba5d592f5..4c43639d3a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -208,11 +208,11 @@ jobs: matrix: include: - package_type: tarballs - os: ubuntu-22.04 + os: ubuntu-24.04 - package_type: win - os: ubuntu-22.04 + os: ubuntu-24.04 - package_type: deb - os: ubuntu-22.04 + os: ubuntu-24.04 - package_type: macos os: macos-14 steps: diff --git a/.github/workflows/tests-codeql.yml b/.github/workflows/tests-codeql.yml index d00a66c8df..78a3f53aeb 100644 --- a/.github/workflows/tests-codeql.yml +++ b/.github/workflows/tests-codeql.yml @@ -4,7 +4,7 @@ on: jobs: codeql: name: Run Code QL - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: actions: read contents: read diff --git a/.github/workflows/tests-js-package.yml b/.github/workflows/tests-js-package.yml index ef508ec073..bdffc8cd41 100644 --- a/.github/workflows/tests-js-package.yml +++ b/.github/workflows/tests-js-package.yml @@ -17,7 +17,7 @@ on: jobs: lint: name: Linting - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write contents: read @@ -51,7 +51,7 @@ jobs: test: name: Tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write contents: read diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index e666491ebc..a05e68ee6c 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -12,7 +12,7 @@ on: lint-runner: description: Runner for linting. Must be JSON valid string. type: string - default: '"ubuntu-22.04"' + default: '"ubuntu-24.04"' check-each-feature: description: If true, try to build each individual feature for this crate type: boolean @@ -42,6 +42,9 @@ jobs: with: components: clippy + - name: Install librocksdb + uses: ./.github/actions/librocksdb + - uses: clechasseur/rs-clippy-check@v3 with: args: --package ${{ inputs.package }} --all-features --locked -- --no-deps @@ -50,10 +53,12 @@ jobs: SCCACHE_BUCKET: multi-runner-cache-x1xibo9c SCCACHE_REGION: ${{ secrets.AWS_REGION }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu + ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" + ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" formatting: name: Formatting - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Check out repo @@ -65,12 +70,19 @@ jobs: components: rustfmt cache: false + # This step doesn't need librocksdb, so we don't install it + - name: Check formatting + env: + RUSTC_WRAPPER: sccache + SCCACHE_BUCKET: multi-runner-cache-x1xibo9c + SCCACHE_REGION: ${{ secrets.AWS_REGION }} + SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu run: cargo fmt --check --package=${{ inputs.package }} unused_deps: name: Unused dependencies - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write contents: read @@ -89,6 +101,9 @@ jobs: - name: Setup Rust uses: ./.github/actions/rust + - name: Install librocksdb + uses: ./.github/actions/librocksdb + - name: Get crate ${{ inputs.package }} info id: crate_info uses: ./.github/actions/crate_info @@ -102,12 +117,14 @@ jobs: SCCACHE_BUCKET: multi-runner-cache-x1xibo9c SCCACHE_REGION: ${{ secrets.AWS_REGION }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu + ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" + ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" with: args: ${{ steps.crate_info.outputs.cargo_manifest_dir }} detect_structure_changes: name: Detect immutable structure changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # FIXME: as we use `gh pr view` below, this check can only # run on pull requests. We should find a way to run it # when manual triggers are used. @@ -184,6 +201,9 @@ jobs: - name: Setup Rust uses: ./.github/actions/rust + - name: Install librocksdb + uses: ./.github/actions/librocksdb + - name: Run tests run: cargo test --package=${{ inputs.package }} --all-features --locked env: @@ -191,6 +211,8 @@ jobs: SCCACHE_BUCKET: multi-runner-cache-x1xibo9c SCCACHE_REGION: ${{ secrets.AWS_REGION }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu + ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" + ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" check_each_feature: name: Check each feature @@ -199,7 +221,7 @@ jobs: if: ${{ inputs.check-each-feature }} steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 @@ -211,6 +233,9 @@ jobs: - name: Setup Rust uses: ./.github/actions/rust + - name: Install librocksdb + uses: ./.github/actions/librocksdb + - name: Get crate ${{ runner.arch }} info id: crate_info uses: ./.github/actions/crate_info @@ -223,6 +248,8 @@ jobs: SCCACHE_BUCKET: multi-runner-cache-x1xibo9c SCCACHE_REGION: ${{ secrets.AWS_REGION }} SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu + ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" + ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" run: | echo Verify all features disabled set -ex diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cca5f1c471..5f6cec2c08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,6 +2,11 @@ name: Tests on: workflow_dispatch: + inputs: + rebuild-deps: + description: "Rebuild cached Rust dependencies" + required: false + default: "false" pull_request: types: [opened, synchronize, reopened, ready_for_review] branches: @@ -16,13 +21,13 @@ jobs: changes: name: Determine changed packages if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: js-packages: ${{ steps.filter-js.outputs.changes }} rs-packages: ${{ steps.filter-rs.outputs.changes }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -36,6 +41,22 @@ jobs: with: filters: .github/package-filters/rs-packages.yml + build-rust-deps: + name: Prebuild and cache some Rust dependencies + runs-on: ubuntu-24.04 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Precompile librocksdb + uses: ./.github/actions/librocksdb + build-js: name: Build JS packages if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} @@ -69,6 +90,7 @@ jobs: name: Rust packages needs: - changes + - build-rust-deps secrets: inherit strategy: fail-fast: false @@ -77,17 +99,19 @@ jobs: uses: ./.github/workflows/tests-rs-package.yml with: package: ${{ matrix.rs-package }} - # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-22.04"' }} + # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-24.04"' }} # FIXME: Clippy fails on github hosted runners, most likely due to RAM usage. Using self-hosted runners for now. - lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' + # lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' + lint-runner: '["ubuntu-24.04"]' # Run drive tests on self-hosted 4x - test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' + # test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' + test-runner: '["ubuntu-24.04"]' check-each-feature: ${{ contains(fromJSON('["dash-sdk","rs-dapi-client","dapi-grpc","dpp","drive-abci"]'), matrix.rs-package) }} rs-crates-security: name: Rust crates security audit if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@v4 @@ -116,7 +140,7 @@ jobs: js-deps-versions: name: JS dependency versions check if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@v4 @@ -135,7 +159,7 @@ jobs: js-npm-security: name: JS NPM security audit if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@v4