Merge branch 'main' into liam-better-runtime-check-customisability

.github/workflows/rust-checks.yaml

@@ -10,32 +10,61 @@ concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
   cancel-in-progress: true
 
+env:
+  RUST_BACKTRACE: 1
+  # pin nightly to avoid constantly throwing out cache
+  TOOLCHAIN_LINT: nightly-2023-11-13
+
 jobs:
-  rust-checks:
-    runs-on: ubuntu-20.04
+  lint:
+    runs-on: ubuntu-latest
     steps:
-      - name: Checkout source code
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+      - name: Install Rust ${{ env.TOOLCHAIN_LINT }}
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ env.TOOLCHAIN_LINT }}
+          targets: wasm32-unknown-unknown
+          components: rustfmt, clippy
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: lint-v0
+      - name: cargo fmt
+        run: cargo +${{ env.TOOLCHAIN_LINT }} fmt --all -- --check
+      - name: Install deps for musl build
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler musl-tools clang build-essential curl llvm-dev libclang-dev linux-headers-generic libsnappy-dev liblz4-dev libzstd-dev libgflags-dev zlib1g-dev libbz2-dev
+          sudo ln -s /usr/bin/g++ /usr/bin/musl-g++
+      - name: cargo clippy
+        uses: actions-rs-plus/clippy-check@v2
+        with:
+          toolchain: ${{ env.TOOLCHAIN_LINT }}
+          args: --all-targets --all-features --locked --no-deps -- --deny warnings
 
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        toolchain: [stable]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust ${{ matrix.toolchain }}
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ matrix.toolchain }}
+          targets: wasm32-unknown-unknown
+      - uses: Swatinem/rust-cache@v2
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
           version: "3.6.1"
-
-      - name: Install clippy and fmt
-        run: rustup component add clippy rustfmt
-
-      - name: Add wasm32-unknown-unknown target
-        run: rustup target add wasm32-unknown-unknown
-
-      - name: Run Format Checks
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all
-
-      - name: Clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets -- --no-deps -D warnings
+      - name: build try-runtime-cli
+        # this is required for testing
+        # build --release or the execution time of the test is too long
+        run: cargo build --release -p try-runtime-cli
+      - name: cargo test
+        run: cargo test --release
+      - name: Check disk space
+        run: df . -h

Cargo.toml

@@ -7,7 +7,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.3.5"
+version = "0.4.0"
 authors = ["Parity Technologies <[email protected]>"]
 description = "Substrate's programmatic testing framework."
 edition = "2021"

core/src/commands/execute_block.rs

@@ -19,7 +19,6 @@ use std::{fmt::Debug, str::FromStr};
 
 use parity_scale_codec::Encode;
 use sc_executor::sp_wasm_interface::HostFunctions;
-use sp_rpc::{list::ListOrValue, number::NumberOrHex};
 use sp_runtime::{
     generic::SignedBlock,
     traits::{Block as BlockT, Header as HeaderT, NumberFor},
@@ -108,18 +107,37 @@ where
     // get the block number associated with this block.
     let block_ws_uri = command.block_ws_uri();
     let rpc = ws_client(&block_ws_uri).await?;
-    let next_hash = next_hash_of::<Block>(&rpc, ext.block_hash).await?;
 
-    log::info!(target: LOG_TARGET, "fetching next block: {:?} ", next_hash);
+    let live_state = match command.state {
+        State::Live(live_state) => live_state,
+        _ => {
+            unreachable!("execute block currently only supports Live state")
+        }
+    };
 
-    let block = ChainApi::<(), Block::Hash, Block::Header, SignedBlock<Block>>::block(
-        &rpc,
-        Some(next_hash),
-    )
-    .await
-    .map_err(rpc_err_handler)?
-    .expect("header exists, block should also exist; qed")
-    .block;
+    // The block we want to *execute* at is the block passed by the user
+    let execute_at = live_state.at::<Block>()?;
+
+    let prev_block_live_state = live_state.to_prev_block_live_state::<Block>().await?;
+
+    // Get state for the prev block
+    let ext = State::Live(prev_block_live_state)
+        .to_ext::<Block, HostFns>(
+            &shared,
+            &executor,
+            None,
+            TryRuntimeFeatureCheck::Check,
+            SpecVersionCheck::Skip,
+        )
+        .await?;
+
+    // Execute the desired block on top of it
+    let block =
+        ChainApi::<(), Block::Hash, Block::Header, SignedBlock<Block>>::block(&rpc, execute_at)
+            .await
+            .map_err(rpc_err_handler)?
+            .expect("header exists, block should also exist; qed")
+            .block;
 
     // A digest item gets added when the runtime is processing the block, so we need to pop
     // the last one to be consistent with what a gossiped block would contain.
@@ -140,6 +158,7 @@ where
 
     let _ = state_machine_call_with_proof::<Block, HostFns>(
         &ext,
+        &mut Default::default(),
         &executor,
         "TryRuntime_execute_block",
         &payload,
@@ -149,35 +168,3 @@ where
 
     Ok(())
 }
-
-pub(crate) async fn next_hash_of<Block: BlockT>(
-    rpc: &substrate_rpc_client::WsClient,
-    hash: Block::Hash,
-) -> sc_cli::Result<Block::Hash>
-where
-    Block: BlockT + serde::de::DeserializeOwned,
-    Block::Header: serde::de::DeserializeOwned,
-{
-    let number = ChainApi::<(), Block::Hash, Block::Header, ()>::header(rpc, Some(hash))
-        .await
-        .map_err(rpc_err_handler)
-        .and_then(|maybe_header| maybe_header.ok_or("header_not_found").map(|h| *h.number()))?;
-
-    let next = number + sp_runtime::traits::One::one();
-
-    let next_hash = match ChainApi::<(), Block::Hash, Block::Header, ()>::block_hash(
-        rpc,
-        Some(ListOrValue::Value(NumberOrHex::Number(
-            next.try_into()
-                .map_err(|_| "failed to convert number to block number")?,
-        ))),
-    )
-    .await
-    .map_err(rpc_err_handler)?
-    {
-        ListOrValue::Value(t) => t.expect("value passed in; value comes out; qed"),
-        _ => unreachable!(),
-    };
-
-    Ok(next_hash)
-}

core/src/commands/fast_forward.rs

@@ -246,6 +246,7 @@ where
         log::info!("Running migrations...");
         state_machine_call_with_proof::<Block, HostFns>(
             &ext,
+            &mut Default::default(),
             &executor,
             "TryRuntime_on_runtime_upgrade",
             command.try_state.encode().as_ref(),

core/src/commands/follow_chain.rs

@@ -157,8 +157,10 @@ where
             .as_mut()
             .expect("state_ext either existed or was just created");
 
+        let mut overlayed_changes = Default::default();
         let result = state_machine_call_with_proof::<Block, HostFns>(
             state_ext,
+            &mut overlayed_changes,
             &executor,
             "TryRuntime_execute_block",
             (
@@ -186,9 +188,7 @@ where
             continue;
         }
 
-        let (mut changes, _, _) = result.expect("checked to be Ok; qed");
-
-        let storage_changes = changes
+        let storage_changes = overlayed_changes
             .drain_storage_changes(
                 &state_ext.backend,
                 // Note that in case a block contains a runtime upgrade, state version could

core/src/commands/offchain_worker.rs

@@ -26,7 +26,7 @@ use crate::{
     build_executor,
     commands::execute_block::next_hash_of,
     full_extensions, parse, rpc_err_handler,
-    state::{LiveState, RuntimeChecks, State},
+    state::{LiveState, RuntimeChecks, SpecVersionCheck, State, TryRuntimeFeatureCheck},
     state_machine_call, SharedParams, LOG_TARGET,
 };
 
@@ -85,14 +85,20 @@ where
         .state
         .to_ext::<Block, HostFns>(&shared, &executor, None, runtime_checks)
         .await?;
+    let block_ws_uri = command.header_ws_uri();
+    let rpc = ws_client(&block_ws_uri).await?;
 
-    let header_ws_uri = command.header_ws_uri();
+    // The block we want to *execute* at is the block passed by the user
+    let execute_at = live_state.at::<Block>()?;
 
-    let rpc = ws_client(&header_ws_uri).await?;
-    let next_hash = next_hash_of::<Block>(&rpc, ext.block_hash).await?;
-    log::info!(target: LOG_TARGET, "fetching next header: {:?} ", next_hash);
+    let prev_block_live_state = live_state.to_prev_block_live_state::<Block>().await?;
 
-    let header = ChainApi::<(), Block::Hash, Block::Header, ()>::header(&rpc, Some(next_hash))
+    // Get state for the prev block
+    let ext = State::Live(prev_block_live_state)
+        .to_ext::<Block, HostFns>(&shared, &executor, None, runtime_checks)
+        .await?;
+
+    let header = ChainApi::<(), Block::Hash, Block::Header, ()>::header(&rpc, execute_at)
         .await
         .map_err(rpc_err_handler)
         .map(|maybe_header| maybe_header.ok_or("Header does not exist"))??;