diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 9256c5ee13e..01a50f5f55a 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -139,7 +139,6 @@ jobs: -p "pallet-*" -p gear-lazy-pages -p gear-runtime-interface - --features=lazy-pages --release upload: diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index a4a3a9a2cb9..1337b6fc61c 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -43,9 +43,7 @@ jobs: run: curl -LsSf https://get.nexte.st/latest/mac | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - name: "Build: Node" - run: >- - cargo build - -p gear-cli --features=lazy-pages + run: cargo build -p gear-cli - name: "Test: Lazy pages" run: >- @@ -53,4 +51,3 @@ jobs: -p "pallet-*" -p gear-lazy-pages -p gear-runtime-interface - --features=lazy-pages diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7dc8cfc4c84..cea3b2e423c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,11 +77,8 @@ jobs: - name: "Check: Vara runtime imports" run: ./target/release/wasm-proc --check-runtime-imports target/release/wbuild/vara-runtime/vara_runtime.compact.wasm - - name: "Test: Gear pallet tests with lazy pages" - run: ./scripts/gear.sh test pallet --features lazy-pages --release --locked - - name: "Test: Gear workspace" - run: ./scripts/gear.sh test gear --exclude gclient --exclude gcli --exclude gsdk --features pallet-gear-debug/lazy-pages --release --locked + run: ./scripts/gear.sh test gear --exclude gclient --exclude gcli --exclude gsdk --release --locked - name: "Test: gsdk tests" run: ./scripts/gear.sh test gsdk --release @@ -252,7 +249,6 @@ jobs: -p "pallet-*" -p gear-lazy-pages -p gear-runtime-interface - --features=lazy-pages --release env: CARGO_BUILD_TARGET: x86_64-pc-windows-msvc diff --git a/.github/workflows/test-measurements.yaml b/.github/workflows/test-measurements.yaml index b9c3a16a657..fca0c365b9e 100644 --- a/.github/workflows/test-measurements.yaml +++ b/.github/workflows/test-measurements.yaml @@ -42,13 +42,13 @@ jobs: tar -xf /cache/check_cargo_registry_${{ github.ref_name }}.tar -C / - name: "Build: Gear" - run: ./scripts/gear.sh build gear --release --locked --features=dev,runtime-benchmarks,lazy-pages + run: ./scripts/gear.sh build gear --release --locked --features=dev,runtime-benchmarks - name: "Collect: Gear workspace tests" run: | mkdir -p ./target/analysis/tests/ mkdir -p ./target/analysis/output/ - for i in `seq 1 $COUNT`; do echo $i; ./scripts/gear.sh test gear --exclude gclient --exclude gcli --features pallet-gear-debug/lazy-pages --release -j1 > ./target/analysis/output/$i 2>&1 ; mv ./target/nextest/ci/junit.xml ./target/analysis/tests/$i; done + for i in `seq 1 $COUNT`; do echo $i; ./scripts/gear.sh test gear --exclude gclient --exclude gcli --release -j1 > ./target/analysis/output/$i 2>&1 ; mv ./target/nextest/ci/junit.xml ./target/analysis/tests/$i; done ./target/release/regression-analysis collect-data --data-folder-path ./target/analysis/tests/ --output-path ./target/pallet-tests.json - name: "Generate report: Gear workspace tests" diff --git a/Cargo.lock b/Cargo.lock index 108999619f0..41a2de7e4e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3654,6 +3654,7 @@ dependencies = [ "gear-core", "gear-core-errors", "gear-core-processor", + "gear-lazy-pages-common", "gmeta", "gsdk", "hex", @@ -3669,6 +3670,7 @@ dependencies = [ "schnorrkel", "serde", "serde_json", + "sp-io 7.0.0 (git+https://github.com/gear-tech/substrate.git?branch=gear-polkadot-v0.9.41-canary-no-sandbox)", "thiserror", "tokio", "which", @@ -3929,6 +3931,7 @@ dependencies = [ "log", "parity-scale-codec", "paste", + "proptest", "scale-info", "static_assertions", "wabt", @@ -3955,9 +3958,9 @@ dependencies = [ "gear-backend-common", "gear-core", "gear-core-errors", + "gear-lazy-pages-common", "gear-wasm-instrument", "log", - "proptest", "scale-info", "static_assertions", ] @@ -4421,6 +4424,7 @@ dependencies = [ "gear-backend-sandbox", "gear-core", "gear-core-processor", + "gear-lazy-pages-common", "gear-utils", "gear-wasm-instrument", "gsys", @@ -4755,6 +4759,7 @@ dependencies = [ "gear-core", "gear-core-errors", "gear-core-processor", + "gear-lazy-pages-common", "gear-utils", "gear-wasm-builder", "gear-wasm-instrument", @@ -4763,6 +4768,7 @@ dependencies = [ "parity-scale-codec", "path-clean", "rand 0.8.5", + "sp-io 7.0.0 (git+https://github.com/gear-tech/substrate.git?branch=gear-polkadot-v0.9.41-canary-no-sandbox)", ] [[package]] diff --git a/Makefile b/Makefile index 347a011742a..dbecebf5e2f 100644 --- a/Makefile +++ b/Makefile @@ -54,11 +54,11 @@ node-release: .PHONY: vara vara: - @ ./scripts/gear.sh build node --no-default-features --features=vara-native,lazy-pages + @ ./scripts/gear.sh build node --no-default-features --features=vara-native .PHONY: vara-release vara-release: - @ ./scripts/gear.sh build node --release --no-default-features --features=vara-native,lazy-pages + @ ./scripts/gear.sh build node --release --no-default-features --features=vara-native .PHONY: gear-replay gear-replay: @@ -197,13 +197,13 @@ test-doc: test-gear: #\ We use lazy-pages feature for pallet-gear-debug due to cargo building issue \ and fact that pallet-gear default is lazy-pages. - @ ./scripts/gear.sh test gear --exclude gclient --exclude gcli --exclude gsdk --features pallet-gear-debug/lazy-pages + @ ./scripts/gear.sh test gear --exclude gclient --exclude gcli --exclude gsdk .PHONY: test-gear-release test-gear-release: # \ We use lazy-pages feature for pallet-gear-debug due to cargo building issue \ and fact that pallet-gear default is lazy-pages. - @ ./scripts/gear.sh test gear --release --exclude gclient --exclude gcli --exclude gsdk --features pallet-gear-debug/lazy-pages + @ ./scripts/gear.sh test gear --release --exclude gclient --exclude gcli --exclude gsdk .PHONY: test-gsdk test-gsdk: node-release diff --git a/core-processor/Cargo.toml b/core-processor/Cargo.toml index 7230e40a335..cc07d9088ee 100644 --- a/core-processor/Cargo.toml +++ b/core-processor/Cargo.toml @@ -15,6 +15,7 @@ gear-core.workspace = true gear-core-errors = { workspace = true, features = ["codec"] } gear-backend-common.workspace = true gear-wasm-instrument.workspace = true +gear-lazy-pages-common.workspace = true scale-info = { workspace = true, features = ["derive"] } log.workspace = true @@ -23,10 +24,11 @@ static_assertions.workspace = true actor-system-error.workspace = true [dev-dependencies] -proptest.workspace = true env_logger.workspace = true enum-iterator.workspace = true [features] +default = ["std"] +std = ["gear-lazy-pages-common/std"] strict = [] -mock = [] \ No newline at end of file +mock = [] diff --git a/core-processor/src/common.rs b/core-processor/src/common.rs index 65d7df3f45f..16d0f842f07 100644 --- a/core-processor/src/common.rs +++ b/core-processor/src/common.rs @@ -545,8 +545,6 @@ pub struct WasmExecutionContext { pub gas_reserver: GasReserver, /// Program to be executed. pub program: Program, - /// Memory pages with initial data. - pub pages_initial_data: BTreeMap, /// Size of the memory block. pub memory_size: WasmPage, } diff --git a/core-processor/src/executor.rs b/core-processor/src/executor.rs index c98778ff592..c1fa182a76e 100644 --- a/core-processor/src/executor.rs +++ b/core-processor/src/executor.rs @@ -26,7 +26,7 @@ use crate::{ }; use actor_system_error::actor_system_error; use alloc::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeSet, format, string::{String, ToString}, vec::Vec, @@ -41,12 +41,12 @@ use gear_core::{ env::Externalities, gas::{CountersOwner, GasAllowanceCounter, GasCounter, ValueCounter}, ids::ProgramId, - memory::{AllocationsContext, Memory, MemoryError, PageBuf}, + memory::{AllocationsContext, Memory}, message::{ ContextSettings, DispatchKind, IncomingDispatch, IncomingMessage, MessageContext, WasmEntryPoint, }, - pages::{GearPage, PageU32Size, WasmPage}, + pages::{PageU32Size, WasmPage}, program::Program, reservation::GasReserver, }; @@ -67,9 +67,6 @@ pub enum ActorPrepareMemoryError { /// Stack end page, which value is specified in WASM code, cannot be bigger than static memory size. #[display(fmt = "Stack end page {_0:?} is bigger then WASM static memory size {_1:?}")] StackEndPageBiggerWasmMemSize(WasmPage, WasmPage), - /// It's not allowed to set initial data for stack memory pages, if they are specified in WASM code. - #[display(fmt = "Set initial data for stack pages is restricted")] - StackPagesHaveInitialData, /// Stack is not aligned to WASM page size #[display(fmt = "Stack end addr {_0:#x} must be aligned to WASM page size")] StackIsNotAligned(u32), @@ -80,24 +77,10 @@ pub enum SystemPrepareMemoryError { /// Mem size less then static pages num #[display(fmt = "Mem size less then static pages num")] InsufficientMemorySize, - /// Page with data is not allocated for program - #[display(fmt = "{_0:?} is not allocated for program")] - PageIsNotAllocated(GearPage), - /// Cannot read initial memory data from wasm memory. - #[display(fmt = "Cannot read data for {_0:?}: {_1}")] - InitialMemoryReadFailed(GearPage, MemoryError), - /// Cannot write initial data to wasm memory. - #[display(fmt = "Cannot write initial data for {_0:?}: {_1}")] - InitialDataWriteFailed(GearPage, MemoryError), - /// Initial pages data must be empty in lazy pages mode - #[display(fmt = "Initial pages data must be empty when execute with lazy pages")] - InitialPagesContainsDataInLazyPagesMode, } /// Make checks that everything with memory goes well. -fn check_memory<'a>( - allocations: &BTreeSet, - pages_with_data: impl Iterator, +fn check_memory( static_pages: WasmPage, memory_size: WasmPage, ) -> Result<(), SystemPrepareMemoryError> { @@ -110,31 +93,13 @@ fn check_memory<'a>( return Err(SystemPrepareMemoryError::InsufficientMemorySize); } - // Checks that all pages with data are in allocations set. - for page in pages_with_data { - let wasm_page = page.to_page(); - if wasm_page >= static_pages && !allocations.contains(&wasm_page) { - return Err(SystemPrepareMemoryError::PageIsNotAllocated(*page)); - } - } - Ok(()) } -fn lazy_pages_check_initial_data( - initial_pages_data: &BTreeMap, -) -> Result<(), SystemPrepareMemoryError> { - initial_pages_data - .is_empty() - .then_some(()) - .ok_or(SystemPrepareMemoryError::InitialPagesContainsDataInLazyPagesMode) -} - /// Writes initial pages data to memory and prepare memory for execution. fn prepare_memory( mem: &mut EnvMem, program_id: ProgramId, - pages_data: &mut BTreeMap, static_pages: WasmPage, stack_end: Option, globals_config: GlobalsAccessConfig, @@ -158,94 +123,17 @@ fn prepare_memory( None }; - // Set initial data for pages - for (page, data) in pages_data.iter_mut() { - mem.write(page.offset(), data) - .map_err(|err| SystemPrepareMemoryError::InitialDataWriteFailed(*page, err))?; - } - - if ProcessorExt::LAZY_PAGES_ENABLED { - lazy_pages_check_initial_data(pages_data)?; - - ProcessorExt::lazy_pages_init_for_program( - mem, - program_id, - stack_end, - globals_config, - lazy_pages_weights, - ); - } else { - // If we executes without lazy pages, then we have to save all initial data for static pages, - // in order to be able to identify pages, which has been changed during execution. - // Skip stack page if they are specified. - let begin = stack_end.unwrap_or_default(); - - if pages_data.keys().any(|&p| p < begin.to_page()) { - return Err(ActorPrepareMemoryError::StackPagesHaveInitialData.into()); - } + ProcessorExt::lazy_pages_init_for_program( + mem, + program_id, + stack_end, + globals_config, + lazy_pages_weights, + ); - let non_stack_pages = begin.iter_end(static_pages).unwrap_or_else(|err| { - unreachable!( - "We have already checked that `stack_end` is <= `static_pages`, but get: {}", - err - ) - }); - for page in non_stack_pages.flat_map(|p| p.to_pages_iter()) { - if pages_data.contains_key(&page) { - // This page already has initial data - continue; - } - let mut data = PageBuf::new_zeroed(); - mem.read(page.offset(), &mut data) - .map_err(|err| SystemPrepareMemoryError::InitialMemoryReadFailed(page, err))?; - pages_data.insert(page, data); - } - } Ok(()) } -/// Returns pages and their new data, which must be updated or uploaded to storage. -fn get_pages_to_be_updated( - old_pages_data: BTreeMap, - new_pages_data: BTreeMap, - static_pages: WasmPage, -) -> BTreeMap { - if ProcessorExt::LAZY_PAGES_ENABLED { - // In lazy pages mode we update some page data in storage, - // when it has been write accessed, so no need to compare old and new page data. - new_pages_data.keys().for_each(|page| { - log::trace!("{:?} has been write accessed, update it in storage", page) - }); - return new_pages_data; - } - - let mut page_update = BTreeMap::new(); - let mut old_pages_data = old_pages_data; - let static_gear_pages = static_pages.to_page(); - for (page, new_data) in new_pages_data { - let initial_data = if let Some(initial_data) = old_pages_data.remove(&page) { - initial_data - } else { - // If it's static page without initial data, - // then it's stack page and we skip this page update. - if page < static_gear_pages { - continue; - } - - // If page has no data in `pages_initial_data` then data is zeros. - // Because it's default data for wasm pages which is not static, - // and for all static pages we save data in `pages_initial_data` in E::new. - PageBuf::new_zeroed() - }; - - if new_data != initial_data { - page_update.insert(page, new_data); - log::trace!("{page:?} has been changed - will be updated in storage"); - } - } - page_update -} - /// Execute wasm with dispatch and return dispatch result. pub fn execute_wasm( balance: u128, @@ -264,7 +152,6 @@ where gas_allowance_counter, gas_reserver, program, - mut pages_initial_data, memory_size, } = context; @@ -277,13 +164,7 @@ where let static_pages = program.static_pages(); let allocations = program.allocations(); - check_memory( - allocations, - pages_initial_data.keys(), - static_pages, - memory_size, - ) - .map_err(SystemExecutionError::PrepareMemory)?; + check_memory(static_pages, memory_size).map_err(SystemExecutionError::PrepareMemory)?; // Creating allocations context. let allocations_context = @@ -353,7 +234,6 @@ where prepare_memory::( memory, program_id, - &mut pages_initial_data, static_pages, stack_end, globals_config, @@ -377,12 +257,10 @@ where }; // released pages initial data will be added to `pages_initial_data` after execution. - if E::Ext::LAZY_PAGES_ENABLED { - E::Ext::lazy_pages_post_execution_actions(&mut memory); + E::Ext::lazy_pages_post_execution_actions(&mut memory); - if !E::Ext::lazy_pages_status().is_normal() { - termination = ext.current_counter_type().into() - } + if !E::Ext::lazy_pages_status().is_normal() { + termination = ext.current_counter_type().into() } (termination, memory, ext) @@ -416,11 +294,6 @@ where .into_ext_info(&memory) .map_err(SystemExecutionError::IntoExtInfo)?; - if E::Ext::LAZY_PAGES_ENABLED { - lazy_pages_check_initial_data(&pages_initial_data) - .map_err(SystemExecutionError::PrepareMemory)?; - } - // Parsing outcome. let kind = match termination { ActorTerminationReason::Exit(value_dest) => DispatchResultKind::Exit(value_dest), @@ -437,8 +310,9 @@ where ActorTerminationReason::GasAllowanceExceeded => DispatchResultKind::GasAllowanceExceed, }; - let page_update = - get_pages_to_be_updated::(pages_initial_data, info.pages_data, static_pages); + // With lazy-pages we update some page data in storage, + // when it has been write accessed, so no need to compare old and new page data. + let page_update = info.pages_data; // Getting new programs that are scheduled to be initialized (respected messages are in `generated_dispatches` collection) let program_candidates = info.program_candidates_data; @@ -467,7 +341,6 @@ where pub fn execute_for_reply( function: EP, instrumented_code: InstrumentedCode, - pages_initial_data: Option>, allocations: Option>, program_id: Option, payload: Vec, @@ -481,8 +354,6 @@ where EP: WasmEntryPoint, { let program = Program::new(program_id.unwrap_or_default(), instrumented_code); - let mut pages_initial_data: BTreeMap = - pages_initial_data.unwrap_or_default(); let static_pages = program.static_pages(); let allocations = allocations.unwrap_or_else(|| program.allocations().clone()); @@ -558,7 +429,6 @@ where prepare_memory::( memory, program.id(), - &mut pages_initial_data, static_pages, stack_end, globals_config, @@ -623,18 +493,13 @@ where #[cfg(test)] mod tests { use super::*; - use alloc::vec::Vec; use gear_backend_common::lazy_pages::Status; - use gear_core::{ - memory::PageBufInner, - pages::{PageNumber, WasmPage}, - }; + use gear_core::pages::WasmPage; struct TestExt; struct LazyTestExt; impl ProcessorExternalities for TestExt { - const LAZY_PAGES_ENABLED: bool = false; fn new(_context: ProcessorContext) -> Self { Self } @@ -655,8 +520,6 @@ mod tests { } impl ProcessorExternalities for LazyTestExt { - const LAZY_PAGES_ENABLED: bool = true; - fn new(_context: ProcessorContext) -> Self { Self } @@ -676,118 +539,14 @@ mod tests { } } - fn prepare_pages_and_allocs() -> (Vec, BTreeSet) { - let data = [0u16, 1, 2, 8, 18, 25, 27, 28, 93, 146, 240, 518]; - let pages = data.map(Into::into); - (pages.to_vec(), pages.map(|p| p.to_page()).into()) - } - - fn prepare_pages() -> BTreeMap { - let mut pages = BTreeMap::new(); - for i in 0..=255 { - let buffer = PageBufInner::filled_with(i); - pages.insert((i as u16).into(), PageBuf::from_inner(buffer)); - } - pages - } - #[test] fn check_memory_insufficient() { - let res = check_memory(&[].into(), [].iter(), 8.into(), 4.into()); + let res = check_memory(8.into(), 4.into()); assert_eq!(res, Err(SystemPrepareMemoryError::InsufficientMemorySize)); } - #[test] - fn check_memory_not_allocated() { - let (pages, mut allocs) = prepare_pages_and_allocs(); - let last = *allocs.iter().last().unwrap(); - allocs.remove(&last); - let res = check_memory(&allocs, pages.iter(), 2.into(), 4.into()); - assert_eq!( - res, - Err(SystemPrepareMemoryError::PageIsNotAllocated( - *pages.last().unwrap() - )) - ); - } - #[test] fn check_memory_ok() { - let (pages, allocs) = prepare_pages_and_allocs(); - check_memory(&allocs, pages.iter(), 4.into(), 8.into()).unwrap(); - } - - #[test] - fn lazy_pages_to_update() { - let new_pages = prepare_pages(); - let res = - get_pages_to_be_updated::(Default::default(), new_pages.clone(), 0.into()); - // All touched pages are to be updated in lazy mode - assert_eq!(res, new_pages); - } - - #[test] - fn no_pages_to_update() { - let old_pages = prepare_pages(); - let mut new_pages = old_pages.clone(); - let static_pages = 4; - let res = - get_pages_to_be_updated::(old_pages, new_pages.clone(), static_pages.into()); - assert_eq!(res, Default::default()); - - // Change static pages - for i in 0..static_pages { - let buffer = PageBufInner::filled_with(42); - new_pages.insert(i.into(), PageBuf::from_inner(buffer)); - } - // Do not include non-static pages - let new_pages = new_pages - .into_iter() - .take(WasmPage::from(static_pages).to_page::().raw() as _) - .collect(); - let res = - get_pages_to_be_updated::(Default::default(), new_pages, static_pages.into()); - assert_eq!(res, Default::default()); - } - - #[test] - fn pages_to_update() { - let old_pages = prepare_pages(); - let mut new_pages = old_pages.clone(); - - let page_with_zero_data = WasmPage::from(30).to_page(); - let changes: BTreeMap = [ - ( - WasmPage::from(1).to_page(), - PageBuf::from_inner(PageBufInner::filled_with(42u8)), - ), - ( - WasmPage::from(5).to_page(), - PageBuf::from_inner(PageBufInner::filled_with(84u8)), - ), - (page_with_zero_data, PageBuf::new_zeroed()), - ] - .into_iter() - .collect(); - new_pages.extend(changes.clone().into_iter()); - - // Change pages - let static_pages = 4.into(); - let res = get_pages_to_be_updated::(old_pages, new_pages.clone(), static_pages); - assert_eq!(res, changes); - - // There was no any old page - let res = - get_pages_to_be_updated::(Default::default(), new_pages.clone(), static_pages); - - // The result is all pages except the static ones - for page in static_pages.to_page::().iter_from_zero() { - new_pages.remove(&page); - } - - // Remove page with zero data, because it must not be updated. - new_pages.remove(&page_with_zero_data); - - assert_eq!(res, new_pages); + check_memory(4.into(), 8.into()).unwrap(); } } diff --git a/core-processor/src/ext.rs b/core-processor/src/ext.rs index aa872cd3d9b..7d8098dd92e 100644 --- a/core-processor/src/ext.rs +++ b/core-processor/src/ext.rs @@ -39,20 +39,20 @@ use gear_core::{ }, ids::{CodeId, MessageId, ProgramId, ReservationId}, memory::{ - AllocError, AllocationsContext, GrowHandler, Memory, MemoryError, MemoryInterval, - NoopGrowHandler, PageBuf, + AllocError, AllocationsContext, GrowHandler, Memory, MemoryError, MemoryInterval, PageBuf, }, message::{ ContextOutcomeDrain, GasLimit, HandlePacket, InitPacket, MessageContext, Packet, ReplyPacket, }, - pages::{GearPage, PageU32Size, WasmPage}, + pages::{PageU32Size, WasmPage}, reservation::GasReserver, }; use gear_core_errors::{ ExecutionError as FallibleExecutionError, ExtError as FallibleExtErrorCore, MessageError, ProgramRentError, ReplyCode, ReservationError, SignalCode, }; +use gear_lazy_pages_common as lazy_pages; use gear_wasm_instrument::syscalls::SysCallName; /// Processor context. @@ -155,9 +155,6 @@ impl ProcessorContext { /// Trait to which ext must have to work in processor wasm executor. /// Currently used only for lazy-pages support. pub trait ProcessorExternalities { - /// Whether this extension works with lazy pages. - const LAZY_PAGES_ENABLED: bool; - /// Create new fn new(context: ProcessorContext) -> Self; @@ -286,6 +283,39 @@ impl BackendAllocSyscallError for AllocExtError { } } +struct LazyGrowHandler { + old_mem_addr: Option, + old_mem_size: WasmPage, +} + +impl GrowHandler for LazyGrowHandler { + fn before_grow_action(mem: &mut impl Memory) -> Self { + // New pages allocation may change wasm memory buffer location. + // So we remove protections from lazy-pages + // and then in `after_grow_action` we set protection back for new wasm memory buffer. + let old_mem_addr = mem.get_buffer_host_addr(); + lazy_pages::remove_lazy_pages_prot(mem); + Self { + old_mem_addr, + old_mem_size: mem.size(), + } + } + + fn after_grow_action(self, mem: &mut impl Memory) { + // Add new allocations to lazy pages. + // Protect all lazy pages including new allocations. + let new_mem_addr = mem.get_buffer_host_addr().unwrap_or_else(|| { + unreachable!("Memory size cannot be zero after grow is applied for memory") + }); + lazy_pages::update_lazy_pages_and_protect_again( + mem, + self.old_mem_addr, + self.old_mem_size, + new_mem_addr, + ); + } +} + /// Structure providing externalities for running host functions. pub struct Ext { /// Processor context. @@ -300,8 +330,6 @@ pub struct Ext { /// Empty implementation for non-substrate (and non-lazy-pages) using impl ProcessorExternalities for Ext { - const LAZY_PAGES_ENABLED: bool = false; - fn new(context: ProcessorContext) -> Self { let current_counter = if context.gas_counter.left() <= context.gas_allowance_counter.left() { @@ -318,36 +346,87 @@ impl ProcessorExternalities for Ext { } fn lazy_pages_init_for_program( - _mem: &mut impl Memory, - _prog_id: ProgramId, - _stack_end: Option, - _globals_config: GlobalsAccessConfig, - _lazy_pages_weights: LazyPagesWeights, + mem: &mut impl Memory, + prog_id: ProgramId, + stack_end: Option, + globals_config: GlobalsAccessConfig, + lazy_pages_weights: LazyPagesWeights, ) { - unreachable!("Must not be called: lazy-pages is unsupported by this ext") + lazy_pages::init_for_program(mem, prog_id, stack_end, globals_config, lazy_pages_weights); } - fn lazy_pages_post_execution_actions(_mem: &mut impl Memory) { - unreachable!("Must not be called: lazy-pages is unsupported by this ext") + fn lazy_pages_post_execution_actions(mem: &mut impl Memory) { + lazy_pages::remove_lazy_pages_prot(mem); } fn lazy_pages_status() -> Status { - unreachable!("Must not be called: lazy-pages is unsupported by this ext") + lazy_pages::get_status() } } impl BackendExternalities for Ext { fn into_ext_info(self, memory: &impl Memory) -> Result { - let pages_for_data = - |static_pages: WasmPage, allocations: &BTreeSet| -> Vec { - static_pages - .iter_from_zero() - .chain(allocations.iter().copied()) - .flat_map(|p| p.to_pages_iter()) - .collect() - }; + let ProcessorContext { + allocations_context, + message_context, + gas_counter, + gas_reserver, + system_reservation, + program_candidates_data, + program_rents, + .. + } = self.context; + + let (static_pages, initial_allocations, allocations) = allocations_context.into_parts(); + + // Accessed pages are all pages, that had been released and are in allocations set or static. + let mut accessed_pages = lazy_pages::get_write_accessed_pages(); + accessed_pages.retain(|p| { + let wasm_page = p.to_page(); + wasm_page < static_pages || allocations.contains(&wasm_page) + }); + log::trace!("accessed pages numbers = {:?}", accessed_pages); + + let mut pages_data = BTreeMap::new(); + for page in accessed_pages { + let mut buf = PageBuf::new_zeroed(); + memory.read(page.offset(), &mut buf)?; + pages_data.insert(page, buf); + } + + let (outcome, mut context_store) = message_context.drain(); + let ContextOutcomeDrain { + outgoing_dispatches: generated_dispatches, + awakening, + reply_deposits, + } = outcome.drain(); + + let system_reservation_context = SystemReservationContext { + current_reservation: system_reservation, + previous_reservation: context_store.system_reservation(), + }; - self.into_ext_info_inner(memory, pages_for_data) + context_store.set_reservation_nonce(&gas_reserver); + if let Some(reservation) = system_reservation { + context_store.add_system_reservation(reservation); + } + + let info = ExtInfo { + gas_amount: gas_counter.to_amount(), + gas_reserver, + system_reservation_context, + allocations: (allocations != initial_allocations) + .then_some(allocations) + .unwrap_or_default(), + pages_data, + generated_dispatches, + awakening, + reply_deposits, + context_store, + program_candidates_data, + program_rents, + }; + Ok(info) } fn gas_amount(&self) -> GasAmount { @@ -355,11 +434,11 @@ impl BackendExternalities for Ext { } fn pre_process_memory_accesses( - _reads: &[MemoryInterval], - _writes: &[MemoryInterval], - _gas_counter: &mut u64, + reads: &[MemoryInterval], + writes: &[MemoryInterval], + gas_counter: &mut u64, ) -> Result<(), ProcessAccessError> { - Ok(()) + lazy_pages::pre_process_memory_accesses(reads, writes, gas_counter) } } @@ -591,7 +670,18 @@ impl Externalities for Ext { pages_num: u32, mem: &mut impl Memory, ) -> Result { - self.alloc_inner::(pages_num, mem) + let pages = WasmPage::new(pages_num).map_err(|_| AllocError::ProgramAllocOutOfBounds)?; + + self.context + .allocations_context + .alloc::(pages, mem, |pages| { + Ext::charge_gas_if_enough( + &mut self.context.gas_counter, + &mut self.context.gas_allowance_counter, + self.context.page_costs.mem_grow.calc(pages), + ) + }) + .map_err(Into::into) } fn free(&mut self, page: WasmPage) -> Result<(), Self::AllocError> { @@ -1031,89 +1121,6 @@ impl Externalities for Ext { } } -impl Ext { - /// Inner alloc realization. - pub fn alloc_inner( - &mut self, - pages_num: u32, - mem: &mut impl Memory, - ) -> Result { - let pages = WasmPage::new(pages_num).map_err(|_| AllocError::ProgramAllocOutOfBounds)?; - - self.context - .allocations_context - .alloc::(pages, mem, |pages| { - Ext::charge_gas_if_enough( - &mut self.context.gas_counter, - &mut self.context.gas_allowance_counter, - self.context.page_costs.mem_grow.calc(pages), - ) - }) - .map_err(Into::into) - } - - /// Into ext info inner impl. - /// `pages_for_data` returns vector of pages which data will be stored in info. - pub fn into_ext_info_inner( - self, - memory: &impl Memory, - pages_for_data: impl FnOnce(WasmPage, &BTreeSet) -> Vec, - ) -> Result { - let ProcessorContext { - allocations_context, - message_context, - gas_counter, - gas_reserver, - system_reservation, - program_candidates_data, - program_rents, - .. - } = self.context; - - let (static_pages, initial_allocations, allocations) = allocations_context.into_parts(); - let mut pages_data = BTreeMap::new(); - for page in pages_for_data(static_pages, &allocations) { - let mut buf = PageBuf::new_zeroed(); - memory.read(page.offset(), &mut buf)?; - pages_data.insert(page, buf); - } - - let (outcome, mut context_store) = message_context.drain(); - let ContextOutcomeDrain { - outgoing_dispatches: generated_dispatches, - awakening, - reply_deposits, - } = outcome.drain(); - - let system_reservation_context = SystemReservationContext { - current_reservation: system_reservation, - previous_reservation: context_store.system_reservation(), - }; - - context_store.set_reservation_nonce(&gas_reserver); - if let Some(reservation) = system_reservation { - context_store.add_system_reservation(reservation); - } - - let info = ExtInfo { - gas_amount: gas_counter.to_amount(), - gas_reserver, - system_reservation_context, - allocations: (allocations != initial_allocations) - .then_some(allocations) - .unwrap_or_default(), - pages_data, - generated_dispatches, - awakening, - reply_deposits, - context_store, - program_candidates_data, - program_rents, - }; - Ok(info) - } -} - #[cfg(test)] mod tests { use super::*; @@ -1636,133 +1643,4 @@ mod tests { assert_eq!(dispatch.message().payload_bytes(), &[3, 4, 5]); } - - mod property_tests { - use super::*; - use gear_core::{ - memory::HostPointer, - pages::{PageError, PageNumber}, - }; - use proptest::{ - arbitrary::any, - collection::size_range, - prop_oneof, proptest, - strategy::{Just, Strategy}, - test_runner::Config as ProptestConfig, - }; - - struct TestMemory(WasmPage); - - impl Memory for TestMemory { - type GrowError = PageError; - - fn grow(&mut self, pages: WasmPage) -> Result<(), Self::GrowError> { - self.0 = self.0.add(pages)?; - Ok(()) - } - - fn size(&self) -> WasmPage { - self.0 - } - - fn write(&mut self, _offset: u32, _buffer: &[u8]) -> Result<(), MemoryError> { - unimplemented!() - } - - fn read(&self, _offset: u32, _buffer: &mut [u8]) -> Result<(), MemoryError> { - unimplemented!() - } - - unsafe fn get_buffer_host_addr_unsafe(&mut self) -> HostPointer { - unimplemented!() - } - } - - #[derive(Debug, Clone)] - enum Action { - Alloc { pages: WasmPage }, - Free { page: WasmPage }, - } - - fn actions() -> impl Strategy> { - let action = wasm_page_number().prop_flat_map(|page| { - prop_oneof![ - Just(Action::Alloc { pages: page }), - Just(Action::Free { page }) - ] - }); - proptest::collection::vec(action, 0..1024) - } - - fn allocations() -> impl Strategy> { - proptest::collection::btree_set(wasm_page_number(), size_range(0..1024)) - } - - fn wasm_page_number() -> impl Strategy { - any::().prop_map(WasmPage::from) - } - - fn proptest_config() -> ProptestConfig { - ProptestConfig { - cases: 1024, - ..Default::default() - } - } - - #[track_caller] - fn assert_alloc_error(err: ::AllocError) { - match err { - AllocExtError::Alloc( - AllocError::IncorrectAllocationData(_) | AllocError::ProgramAllocOutOfBounds, - ) => {} - err => Err(err).unwrap(), - } - } - - #[track_caller] - fn assert_free_error(err: ::AllocError) { - match err { - AllocExtError::Alloc(AllocError::InvalidFree(_)) => {} - err => Err(err).unwrap(), - } - } - - proptest! { - #![proptest_config(proptest_config())] - #[test] - fn alloc( - static_pages in wasm_page_number(), - allocations in allocations(), - max_pages in wasm_page_number(), - mem_size in wasm_page_number(), - actions in actions(), - ) { - let _ = env_logger::try_init(); - - let ctx = AllocationsContext::new(allocations, static_pages, max_pages); - let ctx = ProcessorContextBuilder::new() - .with_gas(GasCounter::new(u64::MAX)) - .with_allowance(GasAllowanceCounter::new(u64::MAX)) - .with_allocation_context(ctx) - .build(); - let mut ext = Ext::new(ctx); - let mut mem = TestMemory(mem_size); - - for action in actions { - match action { - Action::Alloc { pages } => { - if let Err(err) = ext.alloc(pages.raw(), &mut mem) { - assert_alloc_error(err); - } - } - Action::Free { page } => { - if let Err(err) = ext.free(page) { - assert_free_error(err); - } - }, - } - } - } - } - } } diff --git a/core-processor/src/processing.rs b/core-processor/src/processing.rs index 69c36f22cb4..094dc7117be 100644 --- a/core-processor/src/processing.rs +++ b/core-processor/src/processing.rs @@ -27,16 +27,14 @@ use crate::{ ext::ProcessorExternalities, precharge::SuccessfulDispatchResultKind, }; -use alloc::{collections::BTreeMap, string::ToString, vec::Vec}; +use alloc::{string::ToString, vec::Vec}; use gear_backend_common::{ BackendExternalities, BackendSyscallError, Environment, SystemReservationContext, }; use gear_core::{ env::Externalities, ids::{MessageId, ProgramId}, - memory::PageBuf, message::{ContextSettings, DispatchKind, IncomingDispatch, ReplyMessage, StoredDispatch}, - pages::GearPage, reservation::GasReservationState, }; use gear_core_errors::{ErrorReplyReason, SignalCode}; @@ -46,7 +44,6 @@ pub fn process( block_config: &BlockConfig, execution_context: ProcessExecutionContext, random_data: (Vec, u32), - memory_pages: BTreeMap, ) -> Result, SystemExecutionError> where E: Environment, @@ -97,7 +94,6 @@ where gas_allowance_counter: execution_context.gas_allowance_counter, gas_reserver: execution_context.gas_reserver, program: execution_context.program, - pages_initial_data: memory_pages, memory_size: execution_context.memory_size, }; diff --git a/core/Cargo.toml b/core/Cargo.toml index 2b5cb2e556e..666e400433f 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -29,6 +29,7 @@ byteorder.workspace = true [dev-dependencies] wabt.workspace = true env_logger.workspace = true +proptest.workspace = true [features] strict = [] diff --git a/core/src/memory.rs b/core/src/memory.rs index 3d676e09fee..6b429b78dd7 100644 --- a/core/src/memory.rs +++ b/core/src/memory.rs @@ -497,4 +497,122 @@ mod tests { test(1238, 3498); test(0, 64444); } + + mod property_tests { + use super::*; + use crate::{memory::HostPointer, pages::PageError}; + use proptest::{ + arbitrary::any, + collection::size_range, + prop_oneof, proptest, + strategy::{Just, Strategy}, + test_runner::Config as ProptestConfig, + }; + + struct TestMemory(WasmPage); + + impl Memory for TestMemory { + type GrowError = PageError; + + fn grow(&mut self, pages: WasmPage) -> Result<(), Self::GrowError> { + self.0 = self.0.add(pages)?; + Ok(()) + } + + fn size(&self) -> WasmPage { + self.0 + } + + fn write(&mut self, _offset: u32, _buffer: &[u8]) -> Result<(), MemoryError> { + unimplemented!() + } + + fn read(&self, _offset: u32, _buffer: &mut [u8]) -> Result<(), MemoryError> { + unimplemented!() + } + + unsafe fn get_buffer_host_addr_unsafe(&mut self) -> HostPointer { + unimplemented!() + } + } + + #[derive(Debug, Clone)] + enum Action { + Alloc { pages: WasmPage }, + Free { page: WasmPage }, + } + + fn actions() -> impl Strategy> { + let action = wasm_page_number().prop_flat_map(|page| { + prop_oneof![ + Just(Action::Alloc { pages: page }), + Just(Action::Free { page }) + ] + }); + proptest::collection::vec(action, 0..1024) + } + + fn allocations() -> impl Strategy> { + proptest::collection::btree_set(wasm_page_number(), size_range(0..1024)) + } + + fn wasm_page_number() -> impl Strategy { + any::().prop_map(WasmPage::from) + } + + fn proptest_config() -> ProptestConfig { + ProptestConfig { + cases: 1024, + ..Default::default() + } + } + + #[track_caller] + fn assert_alloc_error(err: AllocError) { + match err { + AllocError::IncorrectAllocationData(_) | AllocError::ProgramAllocOutOfBounds => {} + err => Err(err).unwrap(), + } + } + + #[track_caller] + fn assert_free_error(err: AllocError) { + match err { + AllocError::InvalidFree(_) => {} + err => Err(err).unwrap(), + } + } + + proptest! { + #![proptest_config(proptest_config())] + #[test] + fn alloc( + static_pages in wasm_page_number(), + allocations in allocations(), + max_pages in wasm_page_number(), + mem_size in wasm_page_number(), + actions in actions(), + ) { + let _ = env_logger::try_init(); + + let mut ctx = AllocationsContext::new(allocations, static_pages, max_pages); + let mut mem = TestMemory(mem_size); + + for action in actions { + match action { + Action::Alloc { pages } => { + if let Err(err) = ctx.alloc::(pages, &mut mem, |_| Ok(())) { + assert_alloc_error(err); + } + } + Action::Free { page } => { + if let Err(err) = ctx.free(page) { + assert_free_error(err); + } + } + } + } + } + } + } } diff --git a/docker/Vara-Dockerfile b/docker/Vara-Dockerfile index 0744a6a0cb2..4abe70f29fc 100644 --- a/docker/Vara-Dockerfile +++ b/docker/Vara-Dockerfile @@ -33,7 +33,7 @@ RUN rustc --version RUN rustup update nightly && rustup target add wasm32-unknown-unknown --toolchain nightly # Build -RUN cargo build -p gear-cli --no-default-features --features=vara-native,lazy-pages --profile $PROFILE +RUN cargo build -p gear-cli --no-default-features --features=vara-native --profile $PROFILE # ===== SECOND STAGE ====== diff --git a/examples/new-meta/tests/read_state.rs b/examples/new-meta/tests/read_state.rs index d285ac014f8..7ee30cc09bb 100644 --- a/examples/new-meta/tests/read_state.rs +++ b/examples/new-meta/tests/read_state.rs @@ -106,7 +106,7 @@ fn read_state_with_wasm_func_returns_transformed_state() { const FUNC_NAME: &str = "first_wallet"; assert!(META_EXPORTS_V1.contains(&FUNC_NAME)); - let actual_state = program + let actual_state: Option = program .read_state_using_wasm(FUNC_NAME, META_WASM_V1.to_vec(), state_args!()) .expect("Unable to read program state"); @@ -126,7 +126,7 @@ fn read_state_with_parameterized_wasm_func_returns_transformed_state() { name: "OtherName".into(), }; - let actual_state = program + let actual_state: Option = program .read_state_using_wasm( FUNC_NAME, META_WASM_V2.to_vec(), @@ -151,7 +151,7 @@ fn read_state_with_two_args_wasm_func_returns_transformed_state() { let name = "OtherName".to_string(); let surname = "OtherSurname".to_string(); - let actual_state = program + let actual_state: Option = program .read_state_using_wasm( FUNC_NAME, META_WASM_V2.to_vec(), diff --git a/gcli/Cargo.toml b/gcli/Cargo.toml index f676cbac66d..9d9821b9bf3 100644 --- a/gcli/Cargo.toml +++ b/gcli/Cargo.toml @@ -40,10 +40,12 @@ clap = { workspace = true, features = ["derive"] } thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } whoami.workspace = true -core-processor.workspace = true +core-processor = { workspace = true, features = [ "std" ] } gear-backend-sandbox = { workspace = true, features = [ "std" ] } +gear-lazy-pages-common = { workspace = true, features = [ "std" ] } reqwest = { workspace = true, default-features = false, features = [ "json", "rustls-tls" ] } etc.workspace = true +sp-io = { workspace = true, features = [ "std" ] } [dev-dependencies] rand.workspace = true diff --git a/gcli/src/meta/mod.rs b/gcli/src/meta/mod.rs index fe8b3772c83..059d0fac8bc 100644 --- a/gcli/src/meta/mod.rs +++ b/gcli/src/meta/mod.rs @@ -73,6 +73,8 @@ pub enum Meta { } impl Meta { + const PAGE_STORAGE_PREFIX: [u8; 32] = *b"gcligcligcligcligcligcligcligcli"; + fn format_metadata(meta: &MetadataRepr, fmt: &mut fmt::Formatter) -> fmt::Result { let registry = PortableRegistry::decode(&mut meta.registry.as_ref()).map_err(|_| fmt::Error)?; @@ -107,20 +109,25 @@ impl Meta { /// Execute meta method. fn execute(wasm: InstrumentedCode, method: &str) -> Result> { - core_processor::informational::execute_for_reply::< - gear_backend_sandbox::SandboxEnvironment, - String, - >( - method.into(), - wasm, - None, - None, - None, - Default::default(), - u64::MAX, - BlockInfo::default(), - ) - .map_err(Error::WasmExecution) + assert!(gear_lazy_pages_common::try_to_enable_lazy_pages( + Self::PAGE_STORAGE_PREFIX + )); + + sp_io::TestExternalities::default().execute_with(|| { + core_processor::informational::execute_for_reply::< + gear_backend_sandbox::SandboxEnvironment, + String, + >( + method.into(), + wasm, + None, + None, + Default::default(), + u64::MAX, + BlockInfo::default(), + ) + .map_err(Error::WasmExecution) + }) } /// Decode metawasm from wasm binary. diff --git a/gtest/Cargo.toml b/gtest/Cargo.toml index 55886642194..fc1ab748a54 100644 --- a/gtest/Cargo.toml +++ b/gtest/Cargo.toml @@ -10,10 +10,13 @@ gear-core.workspace = true gear-core-errors.workspace = true gear-backend-common.workspace = true gear-backend-sandbox = { workspace = true, features = ["std"] } -core-processor.workspace = true +core-processor = { workspace = true, features = ["std"] } +gear-lazy-pages-common = { workspace = true, features = ["std"] } gear-wasm-builder.workspace = true gear-utils.workspace = true +sp-io = { workspace = true, features = ["std"] } + anyhow.workspace = true codec = { workspace = true, features = ["derive"] } hex.workspace = true diff --git a/gtest/src/manager.rs b/gtest/src/manager.rs index 23a5d0cb964..54a41e9bd16 100644 --- a/gtest/src/manager.rs +++ b/gtest/src/manager.rs @@ -19,7 +19,7 @@ use crate::{ log::{CoreLog, RunResult}, program::{Gas, WasmProgram}, - Result, TestError, DISPATCH_HOLD_COST, EPOCH_DURATION_IN_BLOCKS, EXISTENTIAL_DEPOSIT, + Result, System, TestError, DISPATCH_HOLD_COST, EPOCH_DURATION_IN_BLOCKS, EXISTENTIAL_DEPOSIT, INITIAL_RANDOM_SEED, MAILBOX_THRESHOLD, MAX_RESERVATIONS, MODULE_INSTANTIATION_BYTE_COST, MODULE_INSTRUMENTATION_BYTE_COST, MODULE_INSTRUMENTATION_COST, READ_COST, READ_PER_BYTE_COST, RENT_COST, RESERVATION_COST, RESERVE_FOR, WAITLIST_COST, WRITE_COST, WRITE_PER_BYTE_COST, @@ -45,9 +45,12 @@ use gear_core::{ use gear_core_errors::{ErrorReplyReason, SignalCode, SimpleExecutionError}; use gear_wasm_instrument::wasm_instrument::gas_metering::ConstantCostRules; use rand::{rngs::StdRng, RngCore, SeedableRng}; +use sp_io::TestExternalities; use std::{ + cell::RefCell, collections::{BTreeMap, BTreeSet, HashMap, VecDeque}, convert::TryInto, + rc::Rc, time::{SystemTime, UNIX_EPOCH}, }; @@ -117,13 +120,7 @@ impl TestActor { } // Gets a new executable actor derived from the inner program. - fn get_executable_actor_data( - &self, - ) -> Option<( - ExecutableActorData, - CoreProgram, - BTreeMap, - )> { + fn get_executable_actor_data(&self) -> Option<(ExecutableActorData, CoreProgram)> { let (program, pages_data, code_id, gas_reservation_map) = match self { TestActor::Initialized(Program::Genuine { program, @@ -161,7 +158,6 @@ impl TestActor { gas_reservation_map, }, program, - pages_data, )) } } @@ -219,6 +215,8 @@ pub(crate) struct ExtManager { pub(crate) gas_limits: BTreeMap, pub(crate) delayed_dispatches: HashMap>, + pub(crate) externalities: Rc>, + // Last run info pub(crate) origin: ProgramId, pub(crate) msg_id: MessageId, @@ -325,6 +323,29 @@ impl ExtManager { } } + pub(crate) fn with_externalities(&mut self, f: F) -> R + where + F: FnOnce(&mut Self) -> R, + { + let externalities = self.externalities.clone(); + let mut externalities = externalities.borrow_mut(); + externalities.execute_with(|| f(self)) + } + + fn update_storage_pages(program_id: ProgramId, memory_pages: &BTreeMap) { + // write pages into storage so lazy-pages can access them + for (page, buf) in memory_pages { + let page_no: u32 = (*page).into(); + let prefix = [ + System::PAGE_STORAGE_PREFIX.as_slice(), + program_id.into_bytes().as_slice(), + page_no.to_le_bytes().as_slice(), + ] + .concat(); + sp_io::storage::set(&prefix, buf); + } + } + #[track_caller] fn validate_dispatch(&mut self, dispatch: &Dispatch) { if 0 < dispatch.value() && dispatch.value() < crate::EXISTENTIAL_DEPOSIT { @@ -362,7 +383,7 @@ impl ExtManager { pub(crate) fn validate_and_run_dispatch(&mut self, dispatch: Dispatch) -> RunResult { self.validate_dispatch(&dispatch); - self.run_dispatch(dispatch) + self.with_externalities(|this| this.run_dispatch(dispatch)) } #[track_caller] @@ -409,14 +430,8 @@ impl ExtManager { if actor.is_dormant() { self.process_dormant(balance, dispatch); - } else if let Some((data, program, memory_pages)) = actor.get_executable_actor_data() { - self.process_normal( - balance, - data, - program.code().clone(), - memory_pages, - dispatch, - ); + } else if let Some((data, program)) = actor.get_executable_actor_data() { + self.process_normal(balance, data, program.code().clone(), dispatch); } else if let Some(mock) = actor.take_mock() { self.process_mock(mock, dispatch); } else { @@ -447,11 +462,10 @@ impl ExtManager { .get_mut(program_id) .ok_or_else(|| TestError::ActorNotFound(*program_id))?; - if let Some((_, program, memory_pages)) = actor.get_executable_actor_data() { + if let Some((_, program)) = actor.get_executable_actor_data() { core_processor::informational::execute_for_reply::, _>( String::from("state"), program.code().clone(), - Some(memory_pages), Some(program.allocations().clone()), Some(*program_id), Default::default(), @@ -494,7 +508,6 @@ impl ExtManager { mapping_code, None, None, - None, mapping_code_payload, u64::MAX, self.block_info, @@ -592,9 +605,11 @@ impl ExtManager { }; match program { - Program::Genuine { pages_data, .. } => *pages_data = memory_pages, + Program::Genuine { pages_data, .. } => *pages_data = memory_pages.clone(), Program::Mock(_) => panic!("Can't read memory of mock program"), } + + self.with_externalities(|_this| Self::update_storage_pages(*program_id, &memory_pages)) } #[track_caller] @@ -775,14 +790,13 @@ impl ExtManager { balance: u128, data: ExecutableActorData, code: InstrumentedCode, - memory_pages: BTreeMap, dispatch: StoredDispatch, ) { - self.process_dispatch(balance, Some((data, code)), memory_pages, dispatch); + self.process_dispatch(balance, Some((data, code)), dispatch); } fn process_dormant(&mut self, balance: u128, dispatch: StoredDispatch) { - self.process_dispatch(balance, None, Default::default(), dispatch); + self.process_dispatch(balance, None, dispatch); } #[track_caller] @@ -790,7 +804,6 @@ impl ExtManager { &mut self, balance: u128, data: Option<(ExecutableActorData, InstrumentedCode)>, - memory_pages: BTreeMap, dispatch: StoredDispatch, ) { let dest = dispatch.destination(); @@ -868,7 +881,6 @@ impl ExtManager { &block_config, (context, code, balance).into(), self.random_data.clone(), - memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic violated: {}", e)); @@ -1001,6 +1013,8 @@ impl JournalHandler for ExtManager { .expect("Can't find existing program"); if let Some(actor_pages_data) = actor.get_pages_data_mut() { + Self::update_storage_pages(program_id, &pages_data); + actor_pages_data.append(&mut pages_data); } else { unreachable!("No pages data found for program") diff --git a/gtest/src/program.rs b/gtest/src/program.rs index cc0c4e5b426..dca23cfb66a 100644 --- a/gtest/src/program.rs +++ b/gtest/src/program.rs @@ -447,7 +447,9 @@ impl<'a> Program<'a> { /// Reads the program’s state as a byte vector. pub fn read_state_bytes(&self) -> Result> { - self.manager.borrow_mut().read_state_bytes(&self.id) + self.manager + .borrow_mut() + .with_externalities(|this| this.read_state_bytes(&self.id)) } /// Reads the program’s transformed state as a byte vector. The transformed @@ -502,9 +504,9 @@ impl<'a> Program<'a> { wasm: Vec, args: Option>, ) -> Result> { - self.manager - .borrow_mut() - .read_state_bytes_using_wasm(&self.id, fn_name, wasm, args) + self.manager.borrow_mut().with_externalities(|this| { + this.read_state_bytes_using_wasm(&self.id, fn_name, wasm, args) + }) } /// Reads and decodes the program's state . diff --git a/gtest/src/system.rs b/gtest/src/system.rs index 81462312b43..b4e3c3a70b1 100644 --- a/gtest/src/system.rs +++ b/gtest/src/system.rs @@ -37,8 +37,13 @@ impl Default for System { } impl System { + pub(crate) const PAGE_STORAGE_PREFIX: [u8; 32] = *b"gtestgtestgtestgtestgtestgtest00"; + /// Create a new system. pub fn new() -> Self { + assert!(gear_lazy_pages_common::try_to_enable_lazy_pages( + Self::PAGE_STORAGE_PREFIX + )); Default::default() } diff --git a/node/cli/Cargo.toml b/node/cli/Cargo.toml index 02fe1c8c9d2..5a52ca5e042 100644 --- a/node/cli/Cargo.toml +++ b/node/cli/Cargo.toml @@ -63,7 +63,7 @@ gcli = { workspace = true, optional = true } substrate-build-script-utils.workspace = true [features] -default = ["gear-native", "vara-native", "lazy-pages"] +default = ["gear-native", "vara-native"] gear-native = [ "service/gear-native", "gear-runtime", @@ -73,11 +73,6 @@ vara-native = [ "vara-runtime", "pallet-gear-staking-rewards", ] -lazy-pages = [ - "service/lazy-pages", - "vara-runtime?/lazy-pages", - "gear-runtime?/lazy-pages", -] runtime-benchmarks = [ "service/runtime-benchmarks", "frame-benchmarking", diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index 79ff6da306f..9ad6098d13d 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -109,10 +109,6 @@ vara-native = [ "pallet-im-online", "sp-authority-discovery" ] -lazy-pages = [ - "gear-runtime?/lazy-pages", - "vara-runtime?/lazy-pages", -] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", "frame-benchmarking-cli/runtime-benchmarks", diff --git a/pallets/gear-debug/Cargo.toml b/pallets/gear-debug/Cargo.toml index af5b894aecf..f139f859414 100644 --- a/pallets/gear-debug/Cargo.toml +++ b/pallets/gear-debug/Cargo.toml @@ -75,5 +75,4 @@ runtime-benchmarks = [ "frame-system/runtime-benchmarks", "frame-support/runtime-benchmarks", ] -lazy-pages = ["pallet-gear/lazy-pages"] try-runtime = ["frame-support/try-runtime"] diff --git a/pallets/gear-debug/src/tests/mod.rs b/pallets/gear-debug/src/tests/mod.rs index 2008ae84895..683853ff36c 100644 --- a/pallets/gear-debug/src/tests/mod.rs +++ b/pallets/gear-debug/src/tests/mod.rs @@ -25,13 +25,11 @@ use common::{ ActiveProgram, CodeStorage, Origin as _, PausedProgramStorage, ProgramStorage, }; use frame_support::{assert_err, assert_ok}; -#[cfg(feature = "lazy-pages")] -use gear_core::pages::GearPage; use gear_core::{ ids::{CodeId, MessageId, ProgramId}, memory::PageBuf, message::{DispatchKind, StoredDispatch, StoredMessage, UserMessage}, - pages::{PageNumber, PageU32Size, WasmPage}, + pages::{GearPage, PageNumber, PageU32Size, WasmPage}, }; use gear_wasm_instrument::STACK_END_EXPORT_NAME; use pallet_gear::{ @@ -393,7 +391,6 @@ fn get_last_snapshot() -> DebugData { } } -#[cfg(feature = "lazy-pages")] #[test] fn check_not_allocated_pages() { // Currently we has no mechanism to restrict not allocated pages access during wasm execution @@ -595,7 +592,6 @@ fn check_not_allocated_pages() { }) } -#[cfg(feature = "lazy-pages")] #[test] fn check_changed_pages_in_storage() { // This test checks that only pages, which has been write accessed, @@ -903,12 +899,6 @@ fn check_gear_stack_end() { persistent_pages.insert(gear_page2, page_data.clone()); persistent_pages.insert(gear_page3, page_data); - #[cfg(feature = "lazy-pages")] - log::debug!("LAZY-PAGES IS ON"); - - #[cfg(not(feature = "lazy-pages"))] - log::debug!("LAZY-PAGES IS OFF"); - System::assert_last_event( crate::Event::DebugDataSnapshot(DebugData { dispatch_queue: vec![], diff --git a/pallets/gear-scheduler/Cargo.toml b/pallets/gear-scheduler/Cargo.toml index b8d9a29ccb6..d85e5827e3b 100644 --- a/pallets/gear-scheduler/Cargo.toml +++ b/pallets/gear-scheduler/Cargo.toml @@ -31,7 +31,7 @@ sp-std.workspace = true sp-io.workspace = true [dev-dependencies] -core-processor.workspace = true +core-processor = { workspace = true, features = ["std"] } pallet-gear-bank = { workspace = true, features = ["std"] } pallet-gear = { workspace = true, features = ["std"] } pallet-gear-messenger = { workspace = true, features = ["std"] } diff --git a/pallets/gear/Cargo.toml b/pallets/gear/Cargo.toml index 8dc26661bee..73ecbc023b6 100644 --- a/pallets/gear/Cargo.toml +++ b/pallets/gear/Cargo.toml @@ -26,7 +26,7 @@ static_assertions.workspace = true # Internal deps common.workspace = true gear-runtime-interface = { workspace = true } -gear-lazy-pages-common = { workspace = true, optional = true } +gear-lazy-pages-common.workspace = true core-processor.workspace = true gear-core.workspace = true gear-core-errors.workspace = true @@ -131,7 +131,9 @@ std = [ "frame-system/std", "gear-wasm-instrument/std", "scopeguard/use_std", + "core-processor/std", "gear-backend-sandbox/std", + "gear-lazy-pages-common/std", "scale-info/std", "sp-io/std", "sp-std/std", @@ -149,7 +151,6 @@ std = [ "pallet-gear-proc-macro/full", "primitive-types/std", "serde/std", - "gear-lazy-pages-common?/std", "sp-consensus-babe/std", "test-syscalls?/std", "demo-read-big-state?/std", @@ -175,5 +176,4 @@ runtime-benchmarks = [ ] runtime-benchmarks-checkers = [] try-runtime = ["frame-support/try-runtime"] -lazy-pages = ["gear-lazy-pages-common"] fuzz = ["pallet-gear-gas/fuzz", "common/fuzz"] diff --git a/pallets/gear/src/benchmarking/mod.rs b/pallets/gear/src/benchmarking/mod.rs index 089b6ba2799..591f93069c5 100644 --- a/pallets/gear/src/benchmarking/mod.rs +++ b/pallets/gear/src/benchmarking/mod.rs @@ -236,7 +236,6 @@ where &exec.block_config, exec.context, exec.random_data, - exec.memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic invalidated: {}", e)) } @@ -350,7 +349,6 @@ pub struct Exec { block_config: BlockConfig, context: ProcessExecutionContext, random_data: (Vec, u32), - memory_pages: BTreeMap, } benchmarks! { @@ -369,12 +367,10 @@ benchmarks! { check_all { syscalls_integrity::main_test::(); tests::check_stack_overflow::(); - #[cfg(feature = "lazy-pages")] - { - tests::lazy_pages::lazy_pages_charging::(); - tests::lazy_pages::lazy_pages_charging_special::(); - tests::lazy_pages::lazy_pages_gas_exceed::(); - } + + tests::lazy_pages::lazy_pages_charging::(); + tests::lazy_pages::lazy_pages_charging_special::(); + tests::lazy_pages::lazy_pages_gas_exceed::(); } : {} #[extra] @@ -384,12 +380,9 @@ benchmarks! { #[extra] check_lazy_pages_all { - #[cfg(feature = "lazy-pages")] - { - tests::lazy_pages::lazy_pages_charging::(); - tests::lazy_pages::lazy_pages_charging_special::(); - tests::lazy_pages::lazy_pages_gas_exceed::(); - } + tests::lazy_pages::lazy_pages_charging::(); + tests::lazy_pages::lazy_pages_charging_special::(); + tests::lazy_pages::lazy_pages_gas_exceed::(); } : {} #[extra] @@ -399,19 +392,16 @@ benchmarks! { #[extra] check_lazy_pages_charging { - #[cfg(feature = "lazy-pages")] tests::lazy_pages::lazy_pages_charging::(); }: {} #[extra] check_lazy_pages_charging_special { - #[cfg(feature = "lazy-pages")] tests::lazy_pages::lazy_pages_charging_special::(); }: {} #[extra] check_lazy_pages_gas_exceed { - #[cfg(feature = "lazy-pages")] tests::lazy_pages::lazy_pages_gas_exceed::(); }: {} diff --git a/pallets/gear/src/benchmarking/tests/lazy_pages.rs b/pallets/gear/src/benchmarking/tests/lazy_pages.rs index 10e3513d270..94a92d7e6df 100644 --- a/pallets/gear/src/benchmarking/tests/lazy_pages.rs +++ b/pallets/gear/src/benchmarking/tests/lazy_pages.rs @@ -307,7 +307,6 @@ where &exec.block_config, exec.context, exec.random_data, - exec.memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic invalidated: {}", e)); @@ -384,7 +383,6 @@ where &exec.block_config, exec.context, exec.random_data, - exec.memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic invalidated: {}", e)); @@ -544,7 +542,6 @@ where &exec.block_config, exec.context, exec.random_data, - exec.memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic invalidated: {}", e)); @@ -588,7 +585,6 @@ where &exec.block_config, exec.context, exec.random_data, - exec.memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic invalidated: {}", e)); @@ -630,7 +626,6 @@ where &exec.block_config, exec.context, exec.random_data, - exec.memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic invalidated: {}", e)); diff --git a/pallets/gear/src/benchmarking/tests/mod.rs b/pallets/gear/src/benchmarking/tests/mod.rs index d163bcf2baf..e07981baff9 100644 --- a/pallets/gear/src/benchmarking/tests/mod.rs +++ b/pallets/gear/src/benchmarking/tests/mod.rs @@ -23,7 +23,6 @@ use super::*; -#[cfg(feature = "lazy-pages")] pub mod lazy_pages; pub mod syscalls_integrity; mod utils; @@ -76,7 +75,6 @@ where &exec.block_config, exec.context, exec.random_data, - exec.memory_pages, ) .unwrap() .into_iter() diff --git a/pallets/gear/src/benchmarking/utils.rs b/pallets/gear/src/benchmarking/utils.rs index 60992c73675..d04a116c50a 100644 --- a/pallets/gear/src/benchmarking/utils.rs +++ b/pallets/gear/src/benchmarking/utils.rs @@ -21,10 +21,10 @@ use super::Exec; use crate::{ manager::{CodeInfo, ExtManager, HandleKind}, - Config, CostsPerBlockOf, CurrencyOf, DbWeightOf, MailboxOf, Pallet as Gear, QueueOf, - RentCostPerBlockOf, + Config, CostsPerBlockOf, CurrencyOf, DbWeightOf, MailboxOf, Pallet as Gear, ProgramStorageOf, + QueueOf, RentCostPerBlockOf, }; -use common::{scheduler::SchedulingCostsPerBlock, storage::*, CodeStorage, Origin}; +use common::{scheduler::SchedulingCostsPerBlock, storage::*, CodeStorage, Origin, ProgramStorage}; use core_processor::{ configs::{BlockConfig, BlockInfo}, ContextChargedForCode, ContextChargedForInstrumentation, @@ -40,11 +40,6 @@ use sp_core::H256; use sp_runtime::traits::UniqueSaturatedInto; use sp_std::{convert::TryInto, prelude::*}; -#[cfg(feature = "lazy-pages")] -use crate::ProgramStorageOf; -#[cfg(feature = "lazy-pages")] -use common::ProgramStorage; - const DEFAULT_BLOCK_NUMBER: u32 = 0; const DEFAULT_INTERVAL: u32 = 1_000; @@ -119,7 +114,6 @@ where T: Config, T::AccountId: Origin, { - #[cfg(feature = "lazy-pages")] assert!(gear_lazy_pages_common::try_to_enable_lazy_pages( ProgramStorageOf::::pages_final_prefix() )); @@ -295,6 +289,5 @@ where block_config, context: (context, code, balance).into(), random_data: (vec![0u8; 32], 0), - memory_pages: Default::default(), }) } diff --git a/pallets/gear/src/ext.rs b/pallets/gear/src/ext.rs deleted file mode 100644 index a278c23b06e..00000000000 --- a/pallets/gear/src/ext.rs +++ /dev/null @@ -1,367 +0,0 @@ -// This file is part of Gear. - -// Copyright (C) 2022-2023 Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use alloc::{collections::BTreeSet, vec::Vec}; -use core_processor::{ - AllocExtError, Ext, FallibleExtError, ProcessorContext, ProcessorExternalities, - UnrecoverableExtError, -}; -use gear_backend_common::{ - lazy_pages::{GlobalsAccessConfig, LazyPagesWeights, Status}, - memory::ProcessAccessError, - BackendExternalities, ExtInfo, -}; -use gear_core::{ - costs::RuntimeCosts, - env::{Externalities, PayloadSliceLock, UnlockPayloadBound}, - gas::{ChargeError, CounterType, CountersOwner, GasAmount, GasLeft}, - ids::{MessageId, ProgramId, ReservationId}, - memory::{GrowHandler, Memory, MemoryError, MemoryInterval}, - message::{HandlePacket, InitPacket, ReplyPacket}, - pages::{GearPage, PageU32Size, WasmPage}, -}; -use gear_core_errors::{ReplyCode, SignalCode}; -use gear_lazy_pages_common as lazy_pages; -use gear_wasm_instrument::syscalls::SysCallName; - -/// Ext with lazy pages support. -pub struct LazyPagesExt { - inner: Ext, -} - -impl BackendExternalities for LazyPagesExt { - fn into_ext_info(self, memory: &impl Memory) -> Result { - let pages_for_data = - |static_pages: WasmPage, allocations: &BTreeSet| -> Vec { - // Accessed pages are all pages, that had been released and are in allocations set or static. - let mut accessed_pages = lazy_pages::get_write_accessed_pages(); - accessed_pages.retain(|p| { - let wasm_page = p.to_page(); - wasm_page < static_pages || allocations.contains(&wasm_page) - }); - log::trace!("accessed pages numbers = {:?}", accessed_pages); - accessed_pages - }; - self.inner.into_ext_info_inner(memory, pages_for_data) - } - - fn gas_amount(&self) -> GasAmount { - self.inner.context.gas_counter.to_amount() - } - - fn pre_process_memory_accesses( - reads: &[MemoryInterval], - writes: &[MemoryInterval], - gas_counter: &mut u64, - ) -> Result<(), ProcessAccessError> { - lazy_pages::pre_process_memory_accesses(reads, writes, gas_counter) - } -} - -impl ProcessorExternalities for LazyPagesExt { - const LAZY_PAGES_ENABLED: bool = true; - - fn new(context: ProcessorContext) -> Self { - Self { - inner: Ext::new(context), - } - } - - fn lazy_pages_init_for_program( - mem: &mut impl Memory, - prog_id: ProgramId, - stack_end: Option, - globals_config: GlobalsAccessConfig, - lazy_pages_weights: LazyPagesWeights, - ) { - lazy_pages::init_for_program(mem, prog_id, stack_end, globals_config, lazy_pages_weights); - } - - fn lazy_pages_post_execution_actions(mem: &mut impl Memory) { - lazy_pages::remove_lazy_pages_prot(mem); - } - - fn lazy_pages_status() -> Status { - lazy_pages::get_status() - } -} - -struct LazyGrowHandler { - old_mem_addr: Option, - old_mem_size: WasmPage, -} - -impl GrowHandler for LazyGrowHandler { - fn before_grow_action(mem: &mut impl Memory) -> Self { - // New pages allocation may change wasm memory buffer location. - // So we remove protections from lazy-pages - // and then in `after_grow_action` we set protection back for new wasm memory buffer. - let old_mem_addr = mem.get_buffer_host_addr(); - lazy_pages::remove_lazy_pages_prot(mem); - Self { - old_mem_addr, - old_mem_size: mem.size(), - } - } - - fn after_grow_action(self, mem: &mut impl Memory) { - // Add new allocations to lazy pages. - // Protect all lazy pages including new allocations. - let new_mem_addr = mem.get_buffer_host_addr().unwrap_or_else(|| { - unreachable!("Memory size cannot be zero after grow is applied for memory") - }); - lazy_pages::update_lazy_pages_and_protect_again( - mem, - self.old_mem_addr, - self.old_mem_size, - new_mem_addr, - ); - } -} - -impl CountersOwner for LazyPagesExt { - fn charge_gas_runtime(&mut self, cost: RuntimeCosts) -> Result<(), ChargeError> { - self.inner.charge_gas_runtime(cost) - } - - fn charge_gas_runtime_if_enough(&mut self, cost: RuntimeCosts) -> Result<(), ChargeError> { - self.inner.charge_gas_runtime_if_enough(cost) - } - - fn charge_gas_if_enough(&mut self, amount: u64) -> Result<(), ChargeError> { - self.inner.charge_gas_if_enough(amount) - } - - fn gas_left(&self) -> GasLeft { - self.inner.gas_left() - } - - fn current_counter_type(&self) -> CounterType { - self.inner.current_counter_type() - } - - fn decrease_current_counter_to(&mut self, amount: u64) { - self.inner.decrease_current_counter_to(amount) - } - - fn define_current_counter(&mut self) -> u64 { - self.inner.define_current_counter() - } -} - -impl Externalities for LazyPagesExt { - type UnrecoverableError = UnrecoverableExtError; - type FallibleError = FallibleExtError; - type AllocError = AllocExtError; - - fn alloc( - &mut self, - pages_num: u32, - mem: &mut impl Memory, - ) -> Result { - self.inner.alloc_inner::(pages_num, mem) - } - - fn free(&mut self, page: WasmPage) -> Result<(), Self::AllocError> { - self.inner.free(page) - } - - fn block_height(&self) -> Result { - self.inner.block_height() - } - - fn block_timestamp(&self) -> Result { - self.inner.block_timestamp() - } - - fn send_init(&mut self) -> Result { - self.inner.send_init() - } - - fn send_push(&mut self, handle: u32, buffer: &[u8]) -> Result<(), Self::FallibleError> { - self.inner.send_push(handle, buffer) - } - - fn send_push_input( - &mut self, - handle: u32, - offset: u32, - len: u32, - ) -> Result<(), Self::FallibleError> { - self.inner.send_push_input(handle, offset, len) - } - - fn reply_push(&mut self, buffer: &[u8]) -> Result<(), Self::FallibleError> { - self.inner.reply_push(buffer) - } - - fn send_commit( - &mut self, - handle: u32, - msg: HandlePacket, - delay: u32, - ) -> Result { - self.inner.send_commit(handle, msg, delay) - } - - fn reservation_send_commit( - &mut self, - id: ReservationId, - handle: u32, - msg: HandlePacket, - delay: u32, - ) -> Result { - self.inner.reservation_send_commit(id, handle, msg, delay) - } - - fn reply_commit(&mut self, msg: ReplyPacket) -> Result { - self.inner.reply_commit(msg) - } - - fn reservation_reply_commit( - &mut self, - id: ReservationId, - msg: ReplyPacket, - ) -> Result { - self.inner.reservation_reply_commit(id, msg) - } - - fn reply_to(&self) -> Result { - self.inner.reply_to() - } - - fn signal_from(&self) -> Result { - self.inner.signal_from() - } - - fn reply_push_input(&mut self, offset: u32, len: u32) -> Result<(), Self::FallibleError> { - self.inner.reply_push_input(offset, len) - } - - fn source(&self) -> Result { - self.inner.source() - } - - fn reply_code(&self) -> Result { - self.inner.reply_code() - } - - fn signal_code(&self) -> Result { - self.inner.signal_code() - } - - fn message_id(&self) -> Result { - self.inner.message_id() - } - - fn pay_program_rent( - &mut self, - program_id: ProgramId, - rent: u128, - ) -> Result<(u128, u32), Self::FallibleError> { - self.inner.pay_program_rent(program_id, rent) - } - - fn program_id(&self) -> Result { - self.inner.program_id() - } - - fn debug(&self, data: &str) -> Result<(), Self::UnrecoverableError> { - self.inner.debug(data) - } - - fn size(&self) -> Result { - self.inner.size() - } - - fn random(&self) -> Result<(&[u8], u32), Self::UnrecoverableError> { - self.inner.random() - } - - fn reserve_gas( - &mut self, - amount: u64, - blocks: u32, - ) -> Result { - self.inner.reserve_gas(amount, blocks) - } - - fn unreserve_gas(&mut self, id: ReservationId) -> Result { - self.inner.unreserve_gas(id) - } - - fn system_reserve_gas(&mut self, amount: u64) -> Result<(), Self::FallibleError> { - self.inner.system_reserve_gas(amount) - } - - fn gas_available(&self) -> Result { - self.inner.gas_available() - } - - fn value(&self) -> Result { - self.inner.value() - } - - fn wait(&mut self) -> Result<(), Self::UnrecoverableError> { - self.inner.wait() - } - - fn wait_for(&mut self, duration: u32) -> Result<(), Self::UnrecoverableError> { - self.inner.wait_for(duration) - } - - fn wait_up_to(&mut self, duration: u32) -> Result { - self.inner.wait_up_to(duration) - } - - fn wake(&mut self, waker_id: MessageId, delay: u32) -> Result<(), Self::FallibleError> { - self.inner.wake(waker_id, delay) - } - - fn value_available(&self) -> Result { - self.inner.value_available() - } - - fn create_program( - &mut self, - packet: InitPacket, - delay: u32, - ) -> Result<(MessageId, ProgramId), Self::FallibleError> { - self.inner.create_program(packet, delay) - } - - fn reply_deposit( - &mut self, - message_id: MessageId, - amount: u64, - ) -> Result<(), Self::FallibleError> { - self.inner.reply_deposit(message_id, amount) - } - - fn forbidden_funcs(&self) -> &BTreeSet { - &self.inner.context.forbidden_funcs - } - - fn lock_payload(&mut self, at: u32, len: u32) -> Result { - self.inner.lock_payload(at, len) - } - - fn unlock_payload(&mut self, payload_holder: &mut PayloadSliceLock) -> UnlockPayloadBound { - self.inner.unlock_payload(payload_holder) - } -} diff --git a/pallets/gear/src/lib.rs b/pallets/gear/src/lib.rs index e236c02de33..325d7251c68 100644 --- a/pallets/gear/src/lib.rs +++ b/pallets/gear/src/lib.rs @@ -24,9 +24,6 @@ extern crate alloc; #[cfg(feature = "runtime-benchmarks")] mod benchmarking; -#[cfg(feature = "lazy-pages")] -mod ext; - mod internal; mod queue; mod runtime_api; @@ -58,6 +55,7 @@ use core::marker::PhantomData; use core_processor::{ common::{DispatchOutcome as CoreDispatchOutcome, ExecutableActorData, JournalNote}, configs::{BlockConfig, BlockInfo}, + Ext, }; use frame_support::{ dispatch::{DispatchError, DispatchResultWithPostInfo, PostDispatchInfo}, @@ -74,6 +72,7 @@ use gear_core::{ message::*, pages::{GearPage, WasmPage}, }; +use gear_lazy_pages_common as lazy_pages; use manager::{CodeInfo, QueuePostProcessingData}; use primitive_types::H256; use sp_runtime::{ @@ -86,15 +85,6 @@ use sp_std::{ prelude::*, }; -#[cfg(feature = "lazy-pages")] -use gear_lazy_pages_common as lazy_pages; - -#[cfg(feature = "lazy-pages")] -use ext::LazyPagesExt as Ext; - -#[cfg(not(feature = "lazy-pages"))] -use core_processor::Ext; - type ExecutionEnvironment = gear_backend_sandbox::SandboxEnvironment; pub(crate) type AccountIdOf = ::AccountId; @@ -991,48 +981,13 @@ pub mod pallet { } } - pub(crate) fn enable_lazy_pages() -> bool { - #[cfg(feature = "lazy-pages")] - { - let prefix = ProgramStorageOf::::pages_final_prefix(); - if !lazy_pages::try_to_enable_lazy_pages(prefix) { - unreachable!("By some reasons we cannot run lazy-pages on this machine"); - } - true - } - - #[cfg(not(feature = "lazy-pages"))] - { - false + pub(crate) fn enable_lazy_pages() { + let prefix = ProgramStorageOf::::pages_final_prefix(); + if !lazy_pages::try_to_enable_lazy_pages(prefix) { + unreachable!("By some reasons we cannot run lazy-pages on this machine"); } } - pub(crate) fn get_and_track_memory_pages( - manager: &mut ExtManager, - program_id: ProgramId, - pages_with_data: &BTreeSet, - lazy_pages_enabled: bool, - ) -> Option> { - let pages = if lazy_pages_enabled { - Default::default() - } else { - match ProgramStorageOf::::get_program_data_for_pages( - program_id, - pages_with_data.iter(), - ) { - Ok(data) => data, - Err(err) => { - log::error!("Cannot get data for program pages: {err:?}"); - return None; - } - } - }; - - manager.insert_program_id_loaded_pages(program_id); - - Some(pages) - } - pub(crate) fn block_config() -> BlockConfig { let block_info = BlockInfo { height: Self::block_number().unique_saturated_into(), diff --git a/pallets/gear/src/manager/journal.rs b/pallets/gear/src/manager/journal.rs index b60e65d147f..f5c4deed77d 100644 --- a/pallets/gear/src/manager/journal.rs +++ b/pallets/gear/src/manager/journal.rs @@ -333,6 +333,8 @@ where ProgramStorageOf::::update_active_program(program_id, |p| { for (page, data) in pages_data { + log::trace!("{:?} has been write accessed, update it in storage", page); + ProgramStorageOf::::set_program_page_data(program_id, page, data); p.pages_with_data.insert(page); } diff --git a/pallets/gear/src/queue.rs b/pallets/gear/src/queue.rs index c4381b91543..7bc4280f9a3 100644 --- a/pallets/gear/src/queue.rs +++ b/pallets/gear/src/queue.rs @@ -21,7 +21,6 @@ use core_processor::{common::PrechargedDispatch, ContextChargedForInstrumentatio pub(crate) struct QueueStep<'a, T: Config, F> { pub block_config: &'a BlockConfig, - pub lazy_pages_enabled: bool, pub ext_manager: &'a mut ExtManager, pub gas_limit: GasBalanceOf, pub dispatch: StoredDispatch, @@ -31,7 +30,6 @@ pub(crate) struct QueueStep<'a, T: Config, F> { #[derive(Debug)] pub(crate) enum QueueStepError { - NoMemoryPages, ActorData(PrechargedDispatch), } @@ -46,7 +44,6 @@ where pub(crate) fn execute(self) -> Result, QueueStepError> { let Self { block_config, - lazy_pages_enabled, ext_manager, gas_limit, dispatch, @@ -136,13 +133,7 @@ where }; // Load program memory pages. - let memory_pages = Pallet::::get_and_track_memory_pages( - ext_manager, - program_id, - &context.actor_data().pages_with_data, - lazy_pages_enabled, - ) - .ok_or(QueueStepError::NoMemoryPages)?; + ext_manager.insert_program_id_loaded_pages(program_id); let (random, bn) = T::Randomness::random(dispatch_id.as_ref()); @@ -150,7 +141,6 @@ where block_config, (context, code, balance).into(), (random.encode(), bn.unique_saturated_into()), - memory_pages, ) .unwrap_or_else(|e| unreachable!("core-processor logic invalidated: {}", e)); @@ -169,14 +159,14 @@ where { /// Message Queue processing. pub(crate) fn process_queue(mut ext_manager: ExtManager) { + Self::enable_lazy_pages(); + let block_config = Self::block_config(); if T::DebugInfo::is_remap_id_enabled() { T::DebugInfo::remap_id(); } - let lazy_pages_enabled = Self::enable_lazy_pages(); - while QueueProcessingOf::::allowed() { let dispatch = match QueueOf::::dequeue() .unwrap_or_else(|e| unreachable!("Message queue corrupted! {:?}", e)) @@ -226,7 +216,6 @@ where let step = QueueStep { block_config: &block_config, - lazy_pages_enabled, ext_manager: &mut ext_manager, gas_limit, dispatch, @@ -257,7 +246,6 @@ where MessageWaitedSystemReason::ProgramIsNotInitialized.into_reason(), ); } - Err(QueueStepError::NoMemoryPages) => continue, } } diff --git a/pallets/gear/src/runtime_api.rs b/pallets/gear/src/runtime_api.rs index a37fea57381..d3dc1d94a9a 100644 --- a/pallets/gear/src/runtime_api.rs +++ b/pallets/gear/src/runtime_api.rs @@ -30,7 +30,6 @@ pub(crate) const RUNTIME_API_BLOCK_LIMITS_COUNT: u64 = 6; pub(crate) struct CodeWithMemoryData { pub instrumented_code: InstrumentedCode, pub allocations: BTreeSet, - pub program_pages: Option>, } impl Pallet @@ -46,6 +45,8 @@ where allow_other_panics: bool, allow_skip_zero_replies: bool, ) -> Result> { + Self::enable_lazy_pages(); + let account = ::from_origin(source); let balance = CurrencyOf::::free_balance(&account); @@ -102,8 +103,6 @@ where let mut block_config = Self::block_config(); block_config.forbidden_funcs = [SysCallName::GasAvailable].into(); - let lazy_pages_enabled = Self::enable_lazy_pages(); - let mut min_limit = 0; let mut reserved = 0; let mut burned = 0; @@ -144,7 +143,6 @@ where let step = QueueStep { block_config: &block_config, - lazy_pages_enabled, ext_manager: &mut ext_manager, gas_limit, dispatch: queued_dispatch, @@ -250,24 +248,11 @@ where let instrumented_code = T::CodeStorage::get_code(code_id) .ok_or_else(|| String::from("Failed to get code for given program id"))?; - #[cfg(not(feature = "lazy-pages"))] - let program_pages = Some( - ProgramStorageOf::::get_program_data_for_pages( - program_id, - program.pages_with_data.iter(), - ) - .map_err(|e| format!("Get program pages data error: {e:?}"))?, - ); - - #[cfg(feature = "lazy-pages")] - let program_pages = None; - let allocations = program.allocations; Ok(CodeWithMemoryData { instrumented_code, allocations, - program_pages, }) } @@ -278,13 +263,7 @@ where wasm: Vec, argument: Option>, ) -> Result, String> { - #[cfg(feature = "lazy-pages")] - { - let prefix = ProgramStorageOf::::pages_final_prefix(); - if !lazy_pages::try_to_enable_lazy_pages(prefix) { - unreachable!("By some reasons we cannot run lazy-pages on this machine"); - } - } + Self::enable_lazy_pages(); let schedule = T::Schedule::get(); @@ -322,7 +301,6 @@ where instrumented_code, None, None, - None, payload, BlockGasLimitOf::::get() * RUNTIME_API_BLOCK_LIMITS_COUNT, block_info, @@ -333,20 +311,13 @@ where program_id: ProgramId, payload: Vec, ) -> Result, String> { - #[cfg(feature = "lazy-pages")] - { - let prefix = ProgramStorageOf::::pages_final_prefix(); - if !lazy_pages::try_to_enable_lazy_pages(prefix) { - unreachable!("By some reasons we cannot run lazy-pages on this machine"); - } - } + Self::enable_lazy_pages(); log::debug!("Reading state of {program_id:?}"); let CodeWithMemoryData { instrumented_code, allocations, - program_pages, } = Self::code_with_memory(program_id)?; let block_info = BlockInfo { @@ -357,7 +328,6 @@ where core_processor::informational::execute_for_reply::, String>( String::from("state"), instrumented_code, - program_pages, Some(allocations), Some(program_id), payload, @@ -367,20 +337,13 @@ where } pub(crate) fn read_metahash_impl(program_id: ProgramId) -> Result { - #[cfg(feature = "lazy-pages")] - { - let prefix = ProgramStorageOf::::pages_final_prefix(); - if !lazy_pages::try_to_enable_lazy_pages(prefix) { - unreachable!("By some reasons we cannot run lazy-pages on this machine"); - } - } + Self::enable_lazy_pages(); log::debug!("Reading metahash of {program_id:?}"); let CodeWithMemoryData { instrumented_code, allocations, - program_pages, } = Self::code_with_memory(program_id)?; let block_info = BlockInfo { @@ -391,7 +354,6 @@ where core_processor::informational::execute_for_reply::, String>( String::from("metahash"), instrumented_code, - program_pages, Some(allocations), Some(program_id), Default::default(), diff --git a/pallets/gear/src/tests.rs b/pallets/gear/src/tests.rs index ae861d0d365..c5f771636ad 100644 --- a/pallets/gear/src/tests.rs +++ b/pallets/gear/src/tests.rs @@ -2989,7 +2989,6 @@ fn restrict_start_section() { }); } -#[cfg(feature = "lazy-pages")] #[test] fn memory_access_cases() { // This test access different pages in wasm linear memory. @@ -3231,7 +3230,6 @@ fn memory_access_cases() { }); } -#[cfg(feature = "lazy-pages")] #[test] fn lazy_pages() { use gear_core::pages::{GearPage, PageU32Size}; @@ -4170,16 +4168,12 @@ fn claim_value_works() { // In `calculate_gas_info` program start to work with page data in storage, // so need to take in account gas, which spent for data loading. - let charged_for_page_load = if cfg!(feature = "lazy-pages") { - gas_price( - ::Schedule::get() - .memory_weights - .load_page_data - .ref_time(), - ) - } else { - 0 - }; + let charged_for_page_load = gas_price( + ::Schedule::get() + .memory_weights + .load_page_data + .ref_time(), + ); // Gas left returns to sender from consuming of value tree while claiming. let expected_sender_balance = diff --git a/pallets/payment/Cargo.toml b/pallets/payment/Cargo.toml index 93c809367f0..84d09e5244c 100644 --- a/pallets/payment/Cargo.toml +++ b/pallets/payment/Cargo.toml @@ -75,5 +75,4 @@ runtime-benchmarks = [ "frame-support/runtime-benchmarks", "parity-wasm", ] -lazy-pages = ["pallet-gear/lazy-pages"] try-runtime = ["frame-support/try-runtime"] diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index b673f851b3d..a35574095b4 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -53,6 +53,7 @@ std = [ "pallet-gear-bank/std", "pallet-gear-messenger/std", "runtime-primitives/std", + "gear-core-processor/std", "sp-runtime/std", "sp-std/std", "validator-set/std", diff --git a/runtime/gear/Cargo.toml b/runtime/gear/Cargo.toml index fdd6f2deb3a..03ef3c92499 100644 --- a/runtime/gear/Cargo.toml +++ b/runtime/gear/Cargo.toml @@ -177,8 +177,3 @@ try-runtime = [ "runtime-common/try-runtime", ] dev = ["pallet-gear-debug", "pallet-gear-program/dev"] -lazy-pages = [ - "pallet-gear/lazy-pages", - "pallet-gear-payment/lazy-pages", - "pallet-gear-debug?/lazy-pages", -] diff --git a/runtime/vara/Cargo.toml b/runtime/vara/Cargo.toml index f179c2bd346..6f79232dfc9 100644 --- a/runtime/vara/Cargo.toml +++ b/runtime/vara/Cargo.toml @@ -260,8 +260,3 @@ dev = [ "pallet-gear-program/dev", "pallet-sudo", ] -lazy-pages = [ - "pallet-gear/lazy-pages", - "pallet-gear-payment/lazy-pages", - "pallet-gear-debug?/lazy-pages", -] diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh index 2ddaff402ab..e78dbe6c55c 100755 --- a/scripts/ci_build.sh +++ b/scripts/ci_build.sh @@ -24,13 +24,8 @@ echo "Check: Gear runtime imports" echo "Check: Vara runtime imports" ./target/release/wasm-proc --check-runtime-imports target/release/wbuild/vara-runtime/vara_runtime.compact.wasm -echo "Test: Gear pallet tests with lazy pages" - cargo test -p pallet-gear --features=lazy-pages --release --locked - cargo test -p pallet-gear-debug --features=lazy-pages --release --locked - cargo test -p pallet-gear-payment --features=lazy-pages --release --locked - echo "Test: Gear workspace" - ./scripts/gear.sh test gear --exclude gclient --exclude gcli --features pallet-gear-debug/lazy-pages --release --locked + ./scripts/gear.sh test gear --exclude gclient --exclude gcli --release --locked echo "Test: `gcli`" ./scripts/gear.sh test gcli --release --locked --retries 3 diff --git a/utils/runtime-fuzzer/Cargo.toml b/utils/runtime-fuzzer/Cargo.toml index 0c145193e63..6c0f0cd4de7 100644 --- a/utils/runtime-fuzzer/Cargo.toml +++ b/utils/runtime-fuzzer/Cargo.toml @@ -22,7 +22,7 @@ runtime-primitives.workspace = true gear-common.workspace = true gear-core.workspace = true gear-utils.workspace = true -gear-runtime = { workspace = true, features = ["std", "fuzz", "lazy-pages"] } +gear-runtime = { workspace = true, features = ["std", "fuzz"] } pallet-gear.workspace = true pallet-gear-bank.workspace = true diff --git a/utils/wasm-gen/Cargo.toml b/utils/wasm-gen/Cargo.toml index 9b9c6464ea9..ea4e5c01fbf 100644 --- a/utils/wasm-gen/Cargo.toml +++ b/utils/wasm-gen/Cargo.toml @@ -26,4 +26,5 @@ proptest.workspace = true gear-backend-sandbox = { workspace = true, features = ["std"] } gear-backend-common = { workspace = true, features = ["mock"] } -gear-core-processor = { workspace = true, features = ["mock"] } +gear-core-processor = { workspace = true, features = ["std", "mock"] } +gear-lazy-pages-common = { workspace = true, features = ["std"] } diff --git a/utils/wasm-gen/src/tests.rs b/utils/wasm-gen/src/tests.rs index 1f45db0b6fd..658e838ea2c 100644 --- a/utils/wasm-gen/src/tests.rs +++ b/utils/wasm-gen/src/tests.rs @@ -21,6 +21,7 @@ use arbitrary::Unstructured; use gear_backend_common::{TerminationReason, TrapExplanation}; use gear_core::{ code::Code, + ids::{CodeId, ProgramId}, memory::Memory, message::{IncomingMessage, ReplyPacket}, pages::WASM_PAGE_SIZE, @@ -240,6 +241,12 @@ fn execute_wasm_with_syscall_injected( const INITIAL_PAGES: u16 = 1; const INJECTED_SYSCALLS: u32 = 8; + const PROGRAM_STORAGE_PREFIX: [u8; 32] = *b"execute_wasm_with_syscall_inject"; + + assert!(gear_lazy_pages_common::try_to_enable_lazy_pages( + PROGRAM_STORAGE_PREFIX + )); + // We create Unstructured from zeroes here as we just need any let buf = vec![0; UNSTRUCTURED_SIZE]; let mut unstructured = Unstructured::new(&buf); @@ -289,10 +296,14 @@ fn execute_wasm_with_syscall_injected( // Imitate that reply was already sent. let _ = message_context.reply_commit(ReplyPacket::auto(), None); + let code_id = CodeId::generate(code.original_code()); + let program_id = ProgramId::generate(code_id, b""); + let processor_context = ProcessorContext { message_context, max_pages: INITIAL_PAGES.into(), rent_cost: 10, + program_id, ..ProcessorContext::new_mock() }; @@ -307,7 +318,15 @@ fn execute_wasm_with_syscall_injected( .expect("Failed to create environment"); let report = env - .execute(|mem, _, _| -> Result<(), u32> { + .execute(|mem, _stack_end, globals_config| -> Result<(), u32> { + gear_core_processor::Ext::lazy_pages_init_for_program( + mem, + program_id, + Some(mem.size()), + globals_config, + Default::default(), + ); + if let Some(mem_write) = initial_memory_write { return mem .write(mem_write.offset, &mem_write.content)