feat(gtest): use real weights (#3808)

Co-authored-by: Arsenii Lyashenko <[email protected]>

.github/workflows/benchmarks.yml

@@ -44,6 +44,8 @@ jobs:
           # apply some patches for `pallets/gear/src/weights.rs`
           cp runtime/vara/src/weights/pallet_gear.rs pallets/gear/src/weights.rs
           sed -i -E 's/\w+::WeightInfo for SubstrateWeight/WeightInfo for SubstrateWeight/' pallets/gear/src/weights.rs
+          # generate code for lightweight scheduler that is used in gtest and other crates
+          ./scripts/weight-dump.sh
           # clear the target directory because our benchmarking machine is not ephemeral
           cargo clean
 

.gitignore

@@ -3,6 +3,7 @@ target/
 target-no-lazy/
 target-xwin/
 log/
+weight-dumps/
 .binpath
 .vscode
 .DS_Store

core-backend/src/tests.rs

@@ -606,8 +606,8 @@ fn test_syscalls_table() {
     };
     use gear_core::message::DispatchKind;
     use gear_wasm_instrument::{
+        gas_metering::CustomConstantCostRules,
         parity_wasm::{self, builder},
-        rules::CustomConstantCostRules,
         InstrumentationBuilder, SyscallName,
     };
 

core/src/code/mod.rs

@@ -21,9 +21,8 @@
 use crate::{ids::CodeId, message::DispatchKind, pages::WasmPage};
 use alloc::{collections::BTreeSet, vec::Vec};
 use gear_wasm_instrument::{
+    gas_metering::{CustomConstantCostRules, Rules},
     parity_wasm::{self, elements::Module},
-    rules::CustomConstantCostRules,
-    wasm_instrument::gas_metering::Rules,
     InstrumentationBuilder,
 };
 
@@ -376,7 +375,7 @@ impl CodeAndId {
 mod tests {
     use crate::code::{Code, CodeError, DataSectionError, ExportError};
     use alloc::vec::Vec;
-    use gear_wasm_instrument::rules::CustomConstantCostRules;
+    use gear_wasm_instrument::gas_metering::CustomConstantCostRules;
 
     fn wat2wasm(s: &str) -> Vec<u8> {
         wabt::Wat2Wasm::new()

core/src/program.rs

@@ -120,7 +120,7 @@ mod tests {
     use super::Program;
     use crate::{code::Code, ids::ProgramId};
     use alloc::vec::Vec;
-    use gear_wasm_instrument::rules::CustomConstantCostRules;
+    use gear_wasm_instrument::gas_metering::CustomConstantCostRules;
 
     fn parse_wat(source: &str) -> Vec<u8> {
         let module_bytes = wabt::Wat2Wasm::new()

gsdk/Cargo.toml

@@ -47,5 +47,8 @@ demo-messenger.workspace = true
 demo-new-meta.workspace = true
 demo-waiter = { workspace = true, features = ["std"] }
 
+[build-dependencies]
+gear-utils.workspace = true
+
 [features]
 testing = [ "rand" ]

gsdk/api-gen/Cargo.toml

@@ -10,6 +10,7 @@ sc-executor.workspace = true
 sc-executor-common.workspace = true
 sp-io.workspace = true
 gear-runtime-interface = { workspace = true, features = ["std"] }
+gear-utils.workspace = true
 color-eyre.workspace = true
 proc-macro2.workspace = true
 quote.workspace = true

gsdk/api-gen/src/main.rs

@@ -15,7 +15,9 @@
 //
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 use color_eyre::eyre::Result;
+use gear_utils::codegen::LICENSE;
 use heck::ToSnakeCase as _;
 use parity_scale_codec::Decode;
 use proc_macro2::{Ident, Span, TokenStream};
@@ -33,26 +35,6 @@ const RUNTIME_WASM: &str = "RUNTIME_WASM";
 const USAGE: &str = r#"
 Usage: RUNTIME_WASM=<path> generate-client-api
 "#;
-const LICENSE: &str = r#"
-// This file is part of Gear.
-//
-// Copyright (C) 2021-2024 Gear Technologies Inc.
-// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-#[allow(rustdoc::broken_intra_doc_links)] //subxt-codegen produces incorrect docs
-"#;
 
 fn main() -> Result<()> {
     color_eyre::install()?;

gsdk/build.rs

@@ -1,9 +1,5 @@
-use std::{
-    env, fs,
-    io::Write,
-    path::PathBuf,
-    process::{Command, Stdio},
-};
+use gear_utils::codegen::format_with_rustfmt;
+use std::{env, fs, path::PathBuf, process::Command};
 
 const GSDK_API_GEN: &str = "GSDK_API_GEN";
 const GSDK_API_GEN_PKG: &str = "gsdk-api-gen";
@@ -15,15 +11,15 @@ const ENV_RUNTIME_WASM: &str = "RUNTIME_WASM";
 
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
-    println!("cargo:rerun-if-env-changed={}", GSDK_API_GEN);
+    println!("cargo:rerun-if-env-changed={GSDK_API_GEN}");
 
     // This build script should only work when building gsdk
     // with GSDK_API_GEN=1
     if env::var(GSDK_API_GEN) != Ok("1".into()) {
         return;
     }
 
-    let generated = format!("{}/{}", env!("CARGO_MANIFEST_DIR"), GENERATED_API_PATH);
+    let generated = format!("{}/{GENERATED_API_PATH}", env!("CARGO_MANIFEST_DIR"));
     fs::write(generated, generate_api()).expect("Failed to write generated api");
 }
 
@@ -52,34 +48,12 @@ fn generate_api() -> Vec<u8> {
         .expect("Failed to generate client api.")
         .stdout;
 
-    format(&code).into_bytes()
-}
-
-// Format generated code with rustfmt.
-//
-// - remove the incompatible attributes.
-// - remove verbose whitespaces.
-fn format(stream: &[u8]) -> String {
-    let raw = String::from_utf8_lossy(stream).to_string();
-    let mut rustfmt = Command::new("rustfmt");
-    let mut code = rustfmt
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .spawn()
-        .expect("Spawn rustfmt failed");
-
-    code.stdin
-        .as_mut()
-        .expect("Get stdin of rustfmt failed")
-        .write_all(raw.as_bytes())
-        .expect("pipe generated code to rustfmt failed");
-
-    let out = code.wait_with_output().expect("Run rustfmt failed").stdout;
-    String::from_utf8_lossy(&out)
-        .to_string()
+    // Remove the incompatible attributes and verbose whitespaces.
+    format_with_rustfmt(&code)
         .replace(":: subxt", "::subxt")
         .replace(" : :: ", ": ::")
         .replace(" :: ", "::")
+        .into_bytes()
 }
 
 // Get the path of the compiled package.
@@ -90,7 +64,7 @@ fn get_path(
     pkg: &str,
     features: Vec<&'static str>,
 ) -> PathBuf {
-    let path = PathBuf::from(format!("{}/../target/{}/{}", root, profile, relative_path));
+    let path = PathBuf::from(format!("{root}/../target/{profile}/{relative_path}"));
 
     // If package has not been compiled, compile it.
     if !path.exists() {
@@ -111,9 +85,8 @@ fn get_path(
         // NOTE: not gonna compile the package here since it may block the
         // build process.
         panic!(
-            "package {} has not been compiled yet, please run \
+            "package {pkg} has not been compiled yet, please run \
              `cargo {}` first, or override environment `GEN_CLIENT_API` with `0` for disabling the api generation",
-            pkg,
             args.join(" ")
         );
     }

gsdk/src/metadata/generated.rs

@@ -15,7 +15,7 @@
 //
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
-#[allow(rustdoc::broken_intra_doc_links)] //subxt-codegen produces incorrect docs
+
 #[allow(dead_code, unused_imports, non_camel_case_types)]
 #[allow(clippy::all)]
 #[allow(rustdoc::broken_intra_doc_links)]

gtest/src/manager.rs

@@ -42,7 +42,7 @@ use gear_core::{
     reservation::{GasReservationMap, GasReserver},
 };
 use gear_core_errors::{ErrorReplyReason, SignalCode, SimpleExecutionError};
-use gear_wasm_instrument::rules::CustomConstantCostRules;
+use gear_wasm_instrument::gas_metering::Schedule;
 use rand::{rngs::StdRng, RngCore, SeedableRng};
 use std::{
     cell::{Ref, RefCell, RefMut},
@@ -1129,11 +1129,12 @@ impl JournalHandler for ExtManager {
         if let Some(code) = self.opt_binaries.get(&code_id).cloned() {
             for (init_message_id, candidate_id) in candidates {
                 if !self.actors.contains_key(&candidate_id) {
+                    let schedule = Schedule::default();
                     let code = Code::try_new(
                         code.clone(),
-                        1,
-                        |_| CustomConstantCostRules::default(),
-                        None,
+                        schedule.instruction_weights.version,
+                        |module| schedule.rules(module),
+                        schedule.limits.stack_height,
                     )
                     .expect("Program can't be constructed with provided code");
 

gtest/src/program.rs

@@ -31,7 +31,7 @@ use gear_core::{
 };
 use gear_core_errors::SignalCode;
 use gear_utils::{MemoryPageDump, ProgramMemoryDump};
-use gear_wasm_instrument::rules::CustomConstantCostRules;
+use gear_wasm_instrument::gas_metering::Schedule;
 use path_clean::PathClean;
 use std::{
     cell::RefCell,
@@ -489,8 +489,14 @@ impl<'a> Program<'a> {
         optimized: Vec<u8>,
         metadata: Option<Vec<u8>>,
     ) -> Self {
-        let code = Code::try_new(optimized, 1, |_| CustomConstantCostRules::default(), None)
-            .expect("Failed to create Program from code");
+        let schedule = Schedule::default();
+        let code = Code::try_new(
+            optimized,
+            schedule.instruction_weights.version,
+            |module| schedule.rules(module),
+            schedule.limits.stack_height,
+        )
+        .expect("Failed to create Program from code");
 
         let code_and_id: InstrumentedCodeAndId = CodeAndId::new(code).into();
         let (code, code_id) = code_and_id.into_parts();

pallets/gear/src/schedule.rs

@@ -36,7 +36,7 @@ use gear_core::{
 };
 use gear_wasm_instrument::{
     gas_metering::{MemoryGrowCost, Rules},
-    parity_wasm::elements,
+    parity_wasm::elements::{Instruction, Module, SignExtInstruction, Type},
 };
 use pallet_gear_proc_macro::{ScheduleDebug, WeightDebug};
 use scale_info::TypeInfo;
@@ -1125,15 +1125,15 @@ struct ScheduleRules<'a, T: Config> {
 }
 
 impl<T: Config> Schedule<T> {
-    pub fn rules(&self, module: &elements::Module) -> impl Rules + '_ {
+    pub fn rules(&self, module: &Module) -> impl Rules + '_ {
         ScheduleRules {
             schedule: self,
             params: module
                 .type_section()
                 .iter()
                 .flat_map(|section| section.types())
                 .map(|func| {
-                    let elements::Type::Function(func) = func;
+                    let Type::Function(func) = func;
                     func.params().len() as u32
                 })
                 .collect(),
@@ -1142,8 +1142,10 @@ impl<T: Config> Schedule<T> {
 }
 
 impl<'a, T: Config> Rules for ScheduleRules<'a, T> {
-    fn instruction_cost(&self, instruction: &elements::Instruction) -> Option<u32> {
-        use self::elements::{Instruction::*, SignExtInstruction::*};
+    fn instruction_cost(&self, instruction: &Instruction) -> Option<u32> {
+        use Instruction::*;
+        use SignExtInstruction::*;
+
         let w = &self.schedule.instruction_weights;
         let max_params = self.schedule.limits.parameters;
 
@@ -1267,10 +1269,14 @@ impl<'a, T: Config> Rules for ScheduleRules<'a, T> {
 mod test {
     use super::*;
     use crate::mock::Test;
-    use gear_wasm_instrument::{gas_metering::Rules, rules::CustomConstantCostRules};
+    use gear_wasm_instrument::{
+        gas_metering::{CustomConstantCostRules, Rules, Schedule as WasmInstrumentSchedule},
+        parity_wasm::elements,
+    };
 
-    fn all_measured_instructions() -> Vec<elements::Instruction> {
+    fn all_measured_instructions() -> Vec<Instruction> {
         use elements::{BlockType, BrTableData, Instruction::*};
+
         let default_table_data = BrTableData {
             table: Default::default(),
             default: 0,
@@ -1385,7 +1391,7 @@ mod test {
         ]
     }
 
-    fn default_wasm_module() -> elements::Module {
+    fn default_wasm_module() -> Module {
         let simple_wat = r#"
         (module
             (import "env" "memory" (memory 1))
@@ -1394,7 +1400,7 @@ mod test {
             (func $handle)
             (func $init)
         )"#;
-        elements::Module::from_bytes(
+        Module::from_bytes(
             wabt::Wat2Wasm::new()
                 .validate(false)
                 .convert(simple_wat)
@@ -1411,15 +1417,23 @@ mod test {
     #[test]
     fn instructions_backward_compatibility() {
         let schedule = Schedule::<Test>::default();
+        let wasm_instrument_schedule = WasmInstrumentSchedule::default();
 
         // used in `pallet-gear` to estimate the gas used by the program
         let schedule_rules = schedule.rules(&default_wasm_module());
 
-        // used in `gear-wasm-builder` to check program code at an early stage
+        // used to simulate real gas from `pallet-gear` in crates like gtest
+        let wasm_instrument_schedule_rules = wasm_instrument_schedule.rules(&default_wasm_module());
+
+        // used to simulate gas and reject unsupported instructions in unit tests
         let custom_cost_rules = CustomConstantCostRules::default();
 
         all_measured_instructions().iter().for_each(|i| {
             assert!(schedule_rules.instruction_cost(i).is_some());
+            assert_eq!(
+                schedule_rules.instruction_cost(i),
+                wasm_instrument_schedule_rules.instruction_cost(i)
+            );
             assert!(custom_cost_rules.instruction_cost(i).is_some());
         })
     }

pallets/gear/src/tests.rs

@@ -57,7 +57,7 @@ use gear_core_backend::error::{
     TrapExplanation, UnrecoverableExecutionError, UnrecoverableExtError, UnrecoverableWaitError,
 };
 use gear_core_errors::*;
-use gear_wasm_instrument::{rules::CustomConstantCostRules, STACK_END_EXPORT_NAME};
+use gear_wasm_instrument::{gas_metering::CustomConstantCostRules, STACK_END_EXPORT_NAME};
 use gstd::{collections::BTreeMap, errors::Error as GstdError};
 use pallet_gear_voucher::PrepaidCall;
 use sp_runtime::{