Move all DA layer information into Kernel. Prevent user-space access …

…to Kernel State (#1251)

* Add slot height to context

* Deduplicate state/accessory values using trait

* Deduplicate state/accessory maps with trait

* Deduplicate state/accessory vecs with trait

* Move vec tests into trait

* Minor refactor: reorganize layout

* Revert "Add slot height to context"

This reverts commit 1e8326d.

* Fix tests

* fmt

* clippy

* update lockfiles

* fix test

* lint again

* fix macro test

* fix test; lint again

* fix docs. Lint again

* even more docs and linting

* clippy again

* Fix benches

* Implement versioned kernel values

* Add slot height to context

* protect kernel working set

* lint

* Fix no_std

* Add working_set tests

* fix fuzz target

* Rename KernelStateValue to VersionedStateValue

* Remove dead NonInstantiable type

* Reclaim name KernelValue. Change old to VersionedStateValue

* Fix doc comment

* It all compiles

* Add true and visible heights to chain-state

* WIP

* All unrelated tests pass

* KernelModuleInfo macros

* Fix compilation

* Finish initializing kernel

* Update lockfile

* Remove dead code

* fmt

* Clippy

* Fix no-std and docs

* Fix benches

* Fix feature flag in rng-da

* fix features in attester-incentives

Cargo.toml

@@ -30,6 +30,7 @@ members = [
     "module-system/sov-modules-rollup-blueprint",
     "module-system/sov-modules-macros",
     "module-system/sov-modules-core",
+    "module-system/sov-soft-confirmations-kernel",
     "module-system/sov-state",
     "module-system/sov-modules-api",
     "module-system/module-schemas",

examples/demo-rollup/benches/node/rollup_bench.rs

@@ -1,5 +1,5 @@
 use std::env;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -10,16 +10,16 @@ use demo_stf::runtime::Runtime;
 use sov_db::ledger_db::{LedgerDB, SlotCommit};
 use sov_mock_da::{MockBlock, MockBlockHeader};
 use sov_modules_api::default_context::DefaultContext;
-use sov_modules_stf_blueprint::kernels::basic::BasicKernel;
-use sov_modules_stf_blueprint::StfBlueprint;
+use sov_modules_stf_blueprint::kernels::basic::{BasicKernel, BasicKernelGenesisConfig};
+use sov_modules_stf_blueprint::{GenesisParams, StfBlueprint};
 use sov_prover_storage_manager::new_orphan_storage;
 use sov_risc0_adapter::host::Risc0Verifier;
 use sov_rng_da_service::{RngDaService, RngDaSpec};
 use sov_rollup_interface::da::Time;
 use sov_rollup_interface::services::da::DaService;
 use sov_rollup_interface::stf::StateTransitionFunction;
 use sov_state::DefaultStorageSpec;
-use sov_stf_runner::{from_toml_path, RollupConfig};
+use sov_stf_runner::{from_toml_path, read_json_file, RollupConfig};
 use tempfile::TempDir;
 
 fn rollup_bench(_bench: &mut Criterion) {
@@ -55,13 +55,22 @@ fn rollup_bench(_bench: &mut Criterion) {
         RngDaSpec,
         Risc0Verifier,
         Runtime<DefaultContext, RngDaSpec>,
-        BasicKernel<DefaultContext>,
+        BasicKernel<DefaultContext, _>,
     >::new();
 
-    let demo_genesis_config = get_genesis_config(&GenesisPaths::from_dir(
-        "../test-data/genesis/integration-tests",
-    ))
-    .unwrap();
+    let demo_genesis_config = {
+        let integ_test_conf_dir: &Path = "../../test-data/genesis/integration-tests".as_ref();
+        let rt_params =
+            get_genesis_config::<DefaultContext, _>(&GenesisPaths::from_dir(integ_test_conf_dir))
+                .unwrap();
+
+        let chain_state = read_json_file(integ_test_conf_dir.join("chain_state.json")).unwrap();
+        let kernel_params = BasicKernelGenesisConfig { chain_state };
+        GenesisParams {
+            runtime: rt_params,
+            kernel: kernel_params,
+        }
+    };
 
     let (mut current_root, storage) = stf.init_chain(storage, demo_genesis_config);
 

examples/demo-rollup/benches/node/rollup_coarse_measure.rs

@@ -2,7 +2,7 @@
 extern crate prettytable;
 
 use std::env;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -15,8 +15,8 @@ use prometheus::{Histogram, HistogramOpts, Registry};
 use sov_db::ledger_db::{LedgerDB, SlotCommit};
 use sov_mock_da::{MockBlock, MockBlockHeader, MockDaSpec};
 use sov_modules_api::default_context::DefaultContext;
-use sov_modules_stf_blueprint::kernels::basic::BasicKernel;
-use sov_modules_stf_blueprint::{StfBlueprint, TxEffect};
+use sov_modules_stf_blueprint::kernels::basic::{BasicKernel, BasicKernelGenesisConfig};
+use sov_modules_stf_blueprint::{GenesisParams, StfBlueprint, TxEffect};
 use sov_prover_storage_manager::ProverStorageManager;
 use sov_risc0_adapter::host::Risc0Verifier;
 use sov_rng_da_service::{RngDaService, RngDaSpec};
@@ -25,7 +25,7 @@ use sov_rollup_interface::services::da::{DaService, SlotData};
 use sov_rollup_interface::stf::StateTransitionFunction;
 use sov_rollup_interface::storage::HierarchicalStorageManager;
 use sov_state::DefaultStorageSpec;
-use sov_stf_runner::{from_toml_path, RollupConfig};
+use sov_stf_runner::{from_toml_path, read_json_file, RollupConfig};
 use tempfile::TempDir;
 
 fn print_times(
@@ -121,13 +121,22 @@ async fn main() -> Result<(), anyhow::Error> {
         RngDaSpec,
         Risc0Verifier,
         Runtime<DefaultContext, RngDaSpec>,
-        BasicKernel<DefaultContext>,
+        BasicKernel<DefaultContext, _>,
     >::new();
 
-    let demo_genesis_config = get_genesis_config(&GenesisPaths::from_dir(
-        "../test-data/genesis/integration-tests",
-    ))
-    .unwrap();
+    let demo_genesis_config = {
+        let integ_test_conf_dir: &Path = "../../test-data/genesis/integration-tests".as_ref();
+        let rt_params =
+            get_genesis_config::<DefaultContext, _>(&GenesisPaths::from_dir(integ_test_conf_dir))
+                .unwrap();
+
+        let chain_state = read_json_file(integ_test_conf_dir.join("chain_state.json")).unwrap();
+        let kernel_params = BasicKernelGenesisConfig { chain_state };
+        GenesisParams {
+            runtime: rt_params,
+            kernel: kernel_params,
+        }
+    };
 
     let (mut current_root, storage) = stf.init_chain(storage, demo_genesis_config);
 

examples/demo-rollup/benches/prover/prover_bench.rs

@@ -21,8 +21,8 @@ use regex::Regex;
 use risc0::MOCK_DA_ELF;
 use sov_modules_api::default_context::DefaultContext;
 use sov_modules_api::SlotData;
-use sov_modules_stf_blueprint::kernels::basic::BasicKernel;
-use sov_modules_stf_blueprint::StfBlueprint;
+use sov_modules_stf_blueprint::kernels::basic::{BasicKernel, BasicKernelGenesisConfig};
+use sov_modules_stf_blueprint::{GenesisParams, StfBlueprint};
 use sov_prover_storage_manager::ProverStorageManager;
 use sov_risc0_adapter::host::Risc0Host;
 #[cfg(feature = "bench")]
@@ -33,7 +33,7 @@ use sov_rollup_interface::stf::StateTransitionFunction;
 use sov_rollup_interface::storage::HierarchicalStorageManager;
 use sov_rollup_interface::zk::ZkvmHost;
 use sov_state::DefaultStorageSpec;
-use sov_stf_runner::{from_toml_path, RollupConfig};
+use sov_stf_runner::{from_toml_path, read_json_file, RollupConfig};
 use tempfile::TempDir;
 
 use crate::datagen::get_bench_blocks;
@@ -168,13 +168,23 @@ async fn main() -> Result<(), anyhow::Error> {
         MockDaSpec,
         Risc0Host,
         Runtime<DefaultContext, MockDaSpec>,
-        BasicKernel<DefaultContext>,
+        BasicKernel<DefaultContext, _>,
     >::new();
 
-    let genesis_config = get_genesis_config(&GenesisPaths::from_dir(
-        "../test-data/genesis/integration-tests",
-    ))
-    .unwrap();
+    let genesis_config = {
+        let integ_test_conf_dir: &Path = "../../test-data/genesis/integration-tests".as_ref();
+        let rt_params =
+            get_genesis_config::<DefaultContext, _>(&GenesisPaths::from_dir(integ_test_conf_dir))
+                .unwrap();
+
+        let chain_state = read_json_file(integ_test_conf_dir.join("chain_state.json")).unwrap();
+        let kernel_params = BasicKernelGenesisConfig { chain_state };
+        GenesisParams {
+            runtime: rt_params,
+            kernel: kernel_params,
+        }
+    };
+
     println!("Starting from empty storage, initialization chain");
     let genesis_block = MockBlock::default();
     let (mut prev_state_root, storage) = stf.init_chain(

examples/demo-rollup/provers/risc0/guest-celestia/src/bin/rollup.rs

@@ -20,7 +20,7 @@ risc0_zkvm::guest::entry!(main);
 pub fn main() {
     let guest = Risc0Guest::new();
     let storage = ZkStorage::new();
-    let stf: StfBlueprint<ZkDefaultContext, _, _, Runtime<_, _>, BasicKernel<_>> =
+    let stf: StfBlueprint<ZkDefaultContext, _, _, Runtime<_, _>, BasicKernel<_, _>> =
         StfBlueprint::new();
 
     let stf_verifier = StfVerifier::new(

examples/demo-rollup/provers/risc0/guest-mock/src/bin/mock_da.rs

@@ -36,7 +36,7 @@ pub fn main() {
     #[cfg(feature = "bench")]
     let start_cycles = env::get_cycle_count();
 
-    let stf: StfBlueprint<ZkDefaultContext, _, _, Runtime<_, _>, BasicKernel<_>> =
+    let stf: StfBlueprint<ZkDefaultContext, _, _, Runtime<_, _>, BasicKernel<_, _>> =
         StfBlueprint::new();
 
     let stf_verifier = StfVerifier::new(stf, MockDaVerifier {});

examples/demo-rollup/src/celestia_rollup.rs

@@ -34,8 +34,8 @@ impl RollupBlueprint for CelestiaDemoRollup {
 
     type NativeRuntime = Runtime<Self::NativeContext, Self::DaSpec>;
 
-    type NativeKernel = BasicKernel<Self::NativeContext>;
-    type ZkKernel = BasicKernel<Self::ZkContext>;
+    type NativeKernel = BasicKernel<Self::NativeContext, Self::DaSpec>;
+    type ZkKernel = BasicKernel<Self::ZkContext, Self::DaSpec>;
 
     type ProverService = ParallelProverService<
         <<Self::NativeContext as Spec>::Storage as Storage>::Root,