Made more clear what code is native (#563)

* Made more clear what code is native

* Added missing std cfg flag

* Hope now this works

node/src/command.rs

@@ -115,7 +115,7 @@ pub fn run() -> Result<()> {
 							);
 						}
 						cmd.run::<Block, (
-							frame_system::header_builder::hosted_header_builder::HostFunctions,
+							frame_system::native::hosted_header_builder::hosted_header_builder::HostFunctions,
 							avail_base::mem_tmp_storage::hosted_mem_tmp_storage::HostFunctions,
 							da_runtime::kate::native::hosted_kate::HostFunctions,
 						)>(config)

node/src/service.rs

@@ -52,7 +52,7 @@ pub struct ExecutorDispatch;
 impl sc_executor::NativeExecutionDispatch for ExecutorDispatch {
 	type ExtendHostFunctions = (
 		frame_benchmarking::benchmarking::HostFunctions,
-		frame_system::header_builder::hosted_header_builder::HostFunctions,
+		frame_system::native::hosted_header_builder::hosted_header_builder::HostFunctions,
 		avail_base::mem_tmp_storage::hosted_mem_tmp_storage::HostFunctions,
 		da_runtime::kate::native::hosted_kate::HostFunctions,
 	);

pallets/dactr/src/benchmarking.rs

@@ -12,7 +12,9 @@ use frame_benchmarking::{
 	impl_benchmark_test_suite, v1::BenchmarkError, v2::*, whitelisted_caller,
 };
 use frame_support::traits::Get;
-use frame_system::{header_builder::hosted_header_builder, limits::BlockLength, RawOrigin};
+use frame_system::{
+	limits::BlockLength, native::hosted_header_builder::hosted_header_builder, RawOrigin,
+};
 use scale_info::{StaticTypeInfo, TypeInfo};
 use sp_core::H256;
 use sp_runtime::{

pallets/dactr/src/extensions/extensions_mock.rs

@@ -3,7 +3,7 @@
 use frame_support::weights::WeightToFee;
 use frame_support::{derive_impl, weights::IdentityFee};
 use frame_system::{
-	header_builder::da::HeaderExtensionBuilder, mocking::MockUncheckedExtrinsic,
+	mocking::MockUncheckedExtrinsic, native::hosted_header_builder::da::HeaderExtensionBuilder,
 	test_utils::TestRandomness,
 };
 use pallet_transaction_payment::CurrencyAdapter;

pallets/dactr/src/mock.rs

@@ -6,7 +6,7 @@ use frame_support::{
 	weights::{ConstantMultiplier, IdentityFee},
 };
 use frame_system::{
-	header_builder::da::HeaderExtensionBuilder, mocking::MockUncheckedExtrinsic,
+	mocking::MockUncheckedExtrinsic, native::hosted_header_builder::da::HeaderExtensionBuilder,
 	test_utils::TestRandomness,
 };
 use pallet_transaction_payment::CurrencyAdapter;

pallets/mandate/src/mock.rs

@@ -31,7 +31,8 @@ parameter_types! {
 impl frame_system::Config for Test {
 	type Block = Block;
 	type BlockHashCount = BlockHashCount;
-	type HeaderExtensionBuilder = frame_system::header_builder::da::HeaderExtensionBuilder<Test>;
+	type HeaderExtensionBuilder =
+		frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder<Test>;
 	type OnSetCode = ();
 	type PalletInfo = PalletInfo;
 	type Randomness = frame_system::test_utils::TestRandomness<Test>;

pallets/mocked_runtime/src/lib.rs

@@ -139,7 +139,8 @@ impl frame_system::Config for Runtime {
 	type DbWeight = ();
 	type Hash = sp_core::H256;
 	type Hashing = BlakeTwo256;
-	type HeaderExtensionBuilder = frame_system::header_builder::da::HeaderExtensionBuilder<Runtime>;
+	type HeaderExtensionBuilder =
+		frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder<Runtime>;
 	type Nonce = u64;
 	type Lookup = IdentityLookup<u64>;
 	type MaxConsumers = ConstU32<16>;

pallets/system/benches/bench.rs

@@ -21,9 +21,9 @@ use frame_support::{
 	weights::Weight,
 };
 use frame_system::{
-	header_builder::da::HeaderExtensionBuilder,
 	limits::{BlockLength, BlockWeights},
 	mocking::{MockDaBlock, MockUncheckedExtrinsic},
+	native::hosted_header_builder::da::HeaderExtensionBuilder,
 	test_utils::TestRandomness,
 };
 use sp_core::H256;

pallets/system/benchmarking/src/lib.rs

@@ -23,7 +23,9 @@
 use codec::Encode;
 use frame_benchmarking::{impl_benchmark_test_suite, v2::*};
 use frame_support::{dispatch::DispatchClass, storage, traits::Get};
-use frame_system::{header_builder::HeaderExtensionBuilder, Call, Pallet as System, RawOrigin};
+use frame_system::{
+	native::hosted_header_builder::HeaderExtensionBuilder, Call, Pallet as System, RawOrigin,
+};
 use sp_core::storage::well_known_keys;
 use sp_runtime::traits::Hash;
 use sp_std::{prelude::*, vec};

pallets/system/benchmarking/src/mock.rs

@@ -22,7 +22,7 @@
 use codec::Encode;
 use frame_support::{derive_impl, traits::ConstU32};
 use frame_system::{
-	header_builder::da::HeaderExtensionBuilder, mocking::MockUncheckedExtrinsic,
+	mocking::MockUncheckedExtrinsic, native::hosted_header_builder::da::HeaderExtensionBuilder,
 	test_utils::TestRandomness,
 };
 use sp_runtime::BuildStorage;

pallets/system/src/lib.rs

@@ -146,8 +146,8 @@ use sp_std::{fmt::Debug, marker::PhantomData, prelude::*};
 use sp_version::RuntimeVersion;
 use sp_weights::{RuntimeDbWeight, Weight};
 
-pub mod header_builder;
-pub use header_builder::HeaderExtensionBuilder;
+pub mod native;
+pub use native::hosted_header_builder::HeaderExtensionBuilder;
 
 pub mod limits;
 #[cfg(any(feature = "std", test))]
@@ -527,7 +527,7 @@ pub mod pallet {
 
 		/// Header builder
 		#[pallet::no_default]
-		type HeaderExtensionBuilder: header_builder::HeaderExtensionBuilder;
+		type HeaderExtensionBuilder: native::hosted_header_builder::HeaderExtensionBuilder;
 
 		/// Source of random seeds.
 		#[pallet::no_default]
@@ -1862,7 +1862,7 @@ impl<T: Config> Pallet<T> {
 
 		let block_length = Self::block_length();
 
-		let extension = header_builder::da::HeaderExtensionBuilder::<T>::build(
+		let extension = native::hosted_header_builder::da::HeaderExtensionBuilder::<T>::build(
 			header_extension_builder_data.to_app_extrinsics(),
 			header_extension_builder_data.data_root(),
 			block_length,

pallets/system/src/mock.rs

@@ -112,8 +112,8 @@ impl Config for Test {
 	type OnSetCode = ();
 	type MaxConsumers = ConstU32<16>;
 	type AccountData = u32;
-
-	type HeaderExtensionBuilder = frame_system::header_builder::da::HeaderExtensionBuilder<Test>;
+	type HeaderExtensionBuilder =
+		frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder<Test>;
 	type Randomness = TestRandomness<Test>;
 	type Header = Header;
 	type HeaderExtensionDataFilter = ();

pallets/system/src/mocking.rs

@@ -30,7 +30,7 @@ use sp_runtime::{
 	transaction_validity::TransactionValidityError,
 };
 
-use crate::{header_builder::da::BlockNumber, Config};
+use crate::{native::hosted_header_builder::da::BlockNumber, Config};
 
 #[derive(Clone, Copy, Default, Debug, Encode, Decode, PartialEq, Eq, TypeInfo)]
 pub struct DefaultGetAppId {}

pallets/system/src/native/build_extension_v1.rs

@@ -0,0 +1,128 @@
+// !!!!
+// If the logic is changed in this file it will break Turing. Do not change it.
+// If the logic is changed in avail-core it will break Turing as well. Do no change it.
+// !!!!
+#![cfg(feature = "std")]
+
+use super::hosted_header_builder::MIN_WIDTH;
+use crate::limits::BlockLength;
+use avail_base::metrics::avail::{
+	HeaderExtensionBuilderMetrics as Metrics, MetricObserver, ObserveKind,
+};
+use avail_core::{
+	app_extrinsic::AppExtrinsic,
+	header::{extension as he, HeaderExtension},
+	kate_commitment as kc, HeaderVersion,
+};
+use kate::{
+	couscous::multiproof_params,
+	gridgen::{AsBytes, EvaluationGrid},
+	pmp::m1_blst::M1NoPrecomp,
+	Seed,
+};
+use sp_core::H256;
+use sp_runtime::SaturatedConversion;
+use std::{sync::OnceLock, vec::Vec};
+
+static PMP: OnceLock<M1NoPrecomp> = OnceLock::new();
+
+fn build_grid(
+	submitted: Vec<AppExtrinsic>,
+	block_length: BlockLength,
+	seed: Seed,
+) -> Result<EvaluationGrid, String> {
+	let _metric_observer = MetricObserver::new(ObserveKind::HEGrid);
+
+	let grid = EvaluationGrid::from_extrinsics(
+		submitted,
+		MIN_WIDTH,
+		block_length.cols.0.saturated_into(), // even if we run on a u16 target this is fine
+		block_length.rows.0.saturated_into(),
+		seed,
+	)
+	.map_err(|e| format!("Grid construction failed: {e:?}"))?;
+
+	Ok(grid)
+}
+
+fn build_commitment(grid: &EvaluationGrid) -> Result<Vec<u8>, String> {
+	let _metric_observer = MetricObserver::new(ObserveKind::HECommitment);
+
+	// couscous has pp for degree upto 1024
+	let pmp = PMP.get_or_init(multiproof_params);
+
+	let poly_grid = grid
+		.make_polynomial_grid()
+		.map_err(|e| format!("Make polynomial grid failed: {e:?}"))?;
+
+	let extended_grid = poly_grid
+		.extended_commitments(pmp, 2)
+		.map_err(|e| format!("Grid extension failed: {e:?}"))?;
+
+	let mut commitment = Vec::new();
+	for c in extended_grid.iter() {
+		match c.to_bytes() {
+			Ok(bytes) => commitment.extend(bytes),
+			Err(e) => return Err(format!("Commitment serialization failed: {:?}", e)),
+		}
+	}
+
+	Ok(commitment)
+}
+
+pub fn build_extension(
+	submitted: Vec<AppExtrinsic>,
+	data_root: H256,
+	block_length: BlockLength,
+	_block_number: u32,
+	seed: Seed,
+	version: HeaderVersion,
+) -> HeaderExtension {
+	let _metric_observer = MetricObserver::new(ObserveKind::HETotalExecutionTime);
+
+	// Build the grid
+	let maybe_grid = build_grid(submitted, block_length, seed);
+
+	// We get the grid or return an empty header in case of an error
+	let grid = match maybe_grid {
+		Ok(res) => res,
+		Err(message) => {
+			log::error!("NODE_CRITICAL_ERROR_001 - A critical error has occured: {message:?}.");
+			log::error!("NODE_CRITICAL_ERROR_001 - If you see this, please warn Avail team and raise an issue.");
+			return HeaderExtension::get_faulty_header(data_root, version);
+		},
+	};
+
+	let maybe_commitment = build_commitment(&grid);
+
+	// We get the commitment or return an empty header in case of an error
+	let commitment = match maybe_commitment {
+		Ok(res) => res,
+		Err(message) => {
+			log::error!("NODE_CRITICAL_ERROR_002 - A critical error has occured: {message:?}.");
+			log::error!("NODE_CRITICAL_ERROR_002 - If you see this, please warn Avail team and raise an issue.");
+			return HeaderExtension::get_faulty_header(data_root, version);
+		},
+	};
+
+	// Note that this uses the original dims, _not the extended ones_
+	let rows = grid.dims().rows().get();
+	let cols = grid.dims().cols().get();
+
+	// Grid Metrics
+	Metrics::observe_grid_rows(rows as f64);
+	Metrics::observe_grid_cols(cols as f64);
+
+	let app_lookup = grid.lookup().clone();
+
+	match version {
+		HeaderVersion::V3 => {
+			let commitment = kc::v3::KateCommitment::new(rows, cols, data_root, commitment);
+			he::v3::HeaderExtension {
+				app_lookup,
+				commitment,
+			}
+			.into()
+		},
+	}
+}

pallets/system/src/header_builder/version_select.rs → pallets/system/src/native/build_extension_v2.rs

@@ -1,6 +1,11 @@
-use super::MIN_WIDTH;
-use crate::limits::BlockLength;
+// !!!!
+// If the logic is changed in this file it will break Turing/Mainnet. Do not change it.
+// If the logic is changed in avail-core it will break Turing/Mainnet as well. Do no change it.
+// !!!!
+#![cfg(feature = "std")]
 
+use super::hosted_header_builder::MIN_WIDTH;
+use crate::limits::BlockLength;
 use avail_base::metrics::avail::{
 	HeaderExtensionBuilderMetrics as Metrics, MetricObserver, ObserveKind,
 };
@@ -15,15 +20,13 @@ use kate::{
 	pmp::m1_blst::M1NoPrecomp,
 	Seed,
 };
-
 use sp_core::H256;
 use sp_runtime::SaturatedConversion;
 use std::{sync::OnceLock, vec::Vec};
 
 static PMP: OnceLock<M1NoPrecomp> = OnceLock::new();
 
-#[cfg(feature = "std")]
-pub fn build_grid(
+fn build_grid(
 	submitted: Vec<AppExtrinsic>,
 	block_length: BlockLength,
 	seed: Seed,
@@ -42,8 +45,7 @@ pub fn build_grid(
 	Ok(grid)
 }
 
-#[cfg(feature = "std")]
-pub fn build_commitment(grid: &EvaluationGrid) -> Result<Vec<u8>, String> {
+fn build_commitment(grid: &EvaluationGrid) -> Result<Vec<u8>, String> {
 	let _metric_observer = MetricObserver::new(ObserveKind::HECommitment);
 
 	// couscous has pp for degree upto 1024
@@ -75,63 +77,6 @@ pub fn build_extension(
 	_block_number: u32,
 	seed: Seed,
 	version: HeaderVersion,
-) -> HeaderExtension {
-	let _metric_observer = MetricObserver::new(ObserveKind::HETotalExecutionTime);
-
-	// Build the grid
-	let maybe_grid = build_grid(submitted, block_length, seed);
-
-	// We get the grid or return an empty header in case of an error
-	let grid = match maybe_grid {
-		Ok(res) => res,
-		Err(message) => {
-			log::error!("NODE_CRITICAL_ERROR_001 - A critical error has occured: {message:?}.");
-			log::error!("NODE_CRITICAL_ERROR_001 - If you see this, please warn Avail team and raise an issue.");
-			return HeaderExtension::get_faulty_header(data_root, version);
-		},
-	};
-
-	let maybe_commitment = build_commitment(&grid);
-
-	// We get the commitment or return an empty header in case of an error
-	let commitment = match maybe_commitment {
-		Ok(res) => res,
-		Err(message) => {
-			log::error!("NODE_CRITICAL_ERROR_002 - A critical error has occured: {message:?}.");
-			log::error!("NODE_CRITICAL_ERROR_002 - If you see this, please warn Avail team and raise an issue.");
-			return HeaderExtension::get_faulty_header(data_root, version);
-		},
-	};
-
-	// Note that this uses the original dims, _not the extended ones_
-	let rows = grid.dims().rows().get();
-	let cols = grid.dims().cols().get();
-
-	// Grid Metrics
-	Metrics::observe_grid_rows(rows as f64);
-	Metrics::observe_grid_cols(cols as f64);
-
-	let app_lookup = grid.lookup().clone();
-
-	match version {
-		HeaderVersion::V3 => {
-			let commitment = kc::v3::KateCommitment::new(rows, cols, data_root, commitment);
-			he::v3::HeaderExtension {
-				app_lookup,
-				commitment,
-			}
-			.into()
-		},
-	}
-}
-
-pub fn build_extension_2(
-	submitted: Vec<AppExtrinsic>,
-	data_root: H256,
-	block_length: BlockLength,
-	_block_number: u32,
-	seed: Seed,
-	version: HeaderVersion,
 ) -> HeaderExtension {
 	// Blocks with non-DA extrinsics will have empty commitments
 	if submitted.is_empty() {