node integer delta usage (#412)

## Description
<!-- Describe what change this PR is implementing -->

## Types of Changes
Please select the branch type you are merging and fill in the relevant
template.
<!--- Check the following box with an x if the following applies: -->
- [ ] Hotfix
- [ ] Release
- [x] Fix or Feature

## Fix or Feature
<!--- Check the following box with an x if the following applies: -->

### Types of Changes
<!--- What types of changes does your code introduce? -->
- [x] Tech Debt (Code improvements)
- [x] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [x] Dependency upgrade (A change in substrate or any 3rd party crate
version)

### Migrations and Hooks
<!--- Check the following box with an x if the following applies: -->
- [ ] This change requires a runtime migration.
- [ ] Modifies `on_initialize`
- [ ] Modifies `on_finalize`

### Checklist for Fix or Feature
<!--- All boxes need to be checked. Follow this checklist before
requiring PR review -->
- [x] Change has been tested locally.
- [x] Change adds / updates tests if applicable.
- [ ] Changelog doc updated.
- [x] `spec_version` has been incremented.
- [ ] `network-relayer`'s
[events](https://github.com/Cerebellum-Network/network-relayer/blob/dev-cere/shared/substrate/events.go)
have been updated according to the blockchain events if applicable.
- [ ] All CI checks have been passed successfully

## Checklist for Hotfix
<!--- All boxes need to be checked. Follow this checklist before
requiring PR review -->
- [ ] Change has been deployed to Testnet.
- [ ] Change has been tested in Testnet.
- [ ] Changelog has been updated.
- [ ] Crate version has been updated.
- [ ] `spec_version` has been incremented.
- [ ] Transaction version has been updated if required.
- [ ] Pull Request to `dev` has been created.
- [ ] Pull Request to `staging` has been created.
- [ ] `network-relayer`'s
[events](https://github.com/Cerebellum-Network/network-relayer/blob/dev-cere/shared/substrate/events.go)
have been updated according to the blockchain events if applicable.
- [ ] All CI checks have been passed successfully

## Checklist for Release
<!--- All boxes need to be checked. Follow this checklist before
requiring PR review -->
- [ ] Change has been deployed to Devnet.
- [ ] Change has been tested in Devnet.
- [ ] Change has been deployed to Qanet.
- [ ] Change has been tested in Qanet.
- [ ] Change has been deployed to Testnet.
- [ ] Change has been tested in Testnet.
- [ ] Changelog has been updated.
- [ ] Crate version has been updated.
- [ ] Spec version has been updated.
- [ ] Transaction version has been updated if required.
- [ ] All CI checks have been passed successfully

---------

Co-authored-by: Ayush Kumar Mishra <[email protected]>

pallets/ddc-nodes/Cargo.toml

@@ -11,13 +11,15 @@ repository.workspace = true
 [dependencies]
 # 3rd-party dependencies
 codec = { workspace = true }
+log = { workspace = true }
 scale-info = { workspace = true }
 serde = { workspace = true }
 
 # Substrate dependencies
 frame-benchmarking = { workspace = true, optional = true }
 frame-support = { workspace = true }
 frame-system = { workspace = true }
+hex = { workspace = true }
 sp-runtime = { workspace = true }
 sp-std = { workspace = true }
 
@@ -35,6 +37,7 @@ substrate-test-utils = { workspace = true, default-features = true }
 [features]
 default = ["std"]
 std = [
+  "hex/std",
   "codec/std",
   "ddc-primitives/std",
   "frame-benchmarking/std",

pallets/ddc-nodes/src/lib.rs

@@ -32,12 +32,13 @@ use ddc_primitives::{
 		node::{NodeCreator, NodeVisitor},
 		staking::StakingVisitor,
 	},
-	ClusterId, NodeParams, NodePubKey, StorageNodeParams, StorageNodePubKey,
+	ClusterId, NodeParams, NodePubKey, NodeUsage, StorageNodeParams, StorageNodePubKey,
 };
 use frame_support::pallet_prelude::*;
 use frame_system::pallet_prelude::*;
 pub use pallet::*;
 use sp_std::prelude::*;
+pub mod migrations;
 mod node;
 mod storage_node;
 
@@ -53,7 +54,7 @@ pub mod pallet {
 
 	/// The current storage version.
 	const STORAGE_VERSION: frame_support::traits::StorageVersion =
-		frame_support::traits::StorageVersion::new(0);
+		frame_support::traits::StorageVersion::new(1);
 
 	#[pallet::pallet]
 	#[pallet::storage_version(STORAGE_VERSION)]
@@ -259,6 +260,13 @@ pub mod pallet {
 				},
 			}
 		}
+
+		fn get_total_usage(node_pub_key: &NodePubKey) -> Result<Option<NodeUsage>, DispatchError> {
+			let node = Self::get(node_pub_key.clone()).map_err(|_| Error::<T>::NodeDoesNotExist)?;
+			let total_usage = node.get_total_usage().clone();
+
+			Ok(total_usage)
+		}
 	}
 
 	impl<T: Config> NodeCreator<T> for Pallet<T> {

pallets/ddc-nodes/src/migrations.rs

@@ -0,0 +1,203 @@
+#[cfg(feature = "try-runtime")]
+use ddc_primitives::StorageNodePubKey;
+#[cfg(feature = "try-runtime")]
+use frame_support::ensure;
+use frame_support::{
+	storage_alias,
+	traits::{Get, GetStorageVersion, OnRuntimeUpgrade, StorageVersion},
+	weights::Weight,
+};
+use log::info;
+use serde::{Deserialize, Serialize};
+
+use super::*;
+use crate::{
+	storage_node::{StorageNode, StorageNodeProps},
+	ClusterId,
+};
+
+const LOG_TARGET: &str = "ddc-customers";
+
+pub mod v0 {
+	use frame_support::pallet_prelude::*;
+
+	use super::*;
+
+	// Define the old storage node structure
+	#[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo, PartialEq, Serialize, Deserialize)]
+	#[scale_info(skip_type_params(T))]
+	pub struct StorageNode<T: frame_system::Config> {
+		pub pub_key: StorageNodePubKey,
+		pub provider_id: <T as frame_system::Config>::AccountId,
+		pub cluster_id: Option<ClusterId>,
+		pub props: StorageNodeProps,
+	}
+
+	#[storage_alias]
+	pub type StorageNodes<T: Config> =
+		StorageMap<crate::Pallet<T>, Blake2_128Concat, StorageNodePubKey, StorageNode<T>>;
+}
+pub fn migrate_to_v1<T: Config>() -> Weight {
+	let on_chain_version = Pallet::<T>::on_chain_storage_version();
+	let current_version = Pallet::<T>::current_storage_version();
+
+	info!(
+		target: LOG_TARGET,
+		"Running migration with current storage version {:?} / onchain {:?}",
+		current_version,
+		on_chain_version
+	);
+
+	if on_chain_version == 0 && current_version == 1 {
+		let weight = T::DbWeight::get().reads(1);
+
+		let count = v0::StorageNodes::<T>::iter().count();
+		info!(
+			target: LOG_TARGET,
+			" >>> Updating DDC Storage Nodes. Migrating {} nodes...", count
+		);
+		StorageNodes::<T>::translate::<v0::StorageNode<T>, _>(|_, old: v0::StorageNode<T>| {
+			let node_pub_key_ref: &[u8; 32] = old.pub_key.as_ref();
+			let node_pub_key_string = hex::encode(node_pub_key_ref);
+			info!(target: LOG_TARGET, "     Migrating node for node ID {:?}...", node_pub_key_string);
+
+			Some(StorageNode {
+				pub_key: old.pub_key,
+				provider_id: old.provider_id,
+				cluster_id: old.cluster_id,
+				props: old.props,
+				total_usage: None, // Default value for the new field
+			})
+		});
+
+		// Update storage version.
+		StorageVersion::new(1).put::<Pallet<T>>();
+		let count = StorageNodes::<T>::iter().count();
+		info!(
+			target: LOG_TARGET,
+			"Upgraded {} records, storage to version {:?}",
+			count,
+			current_version
+		);
+
+		weight.saturating_add(T::DbWeight::get().reads_writes(count as u64 + 1, count as u64 + 1))
+	} else {
+		info!(target: LOG_TARGET, " >>> Unused migration!");
+		T::DbWeight::get().reads(1)
+	}
+}
+
+pub struct MigrateToV1<T>(PhantomData<T>);
+
+impl<T: Config> OnRuntimeUpgrade for MigrateToV1<T> {
+	fn on_runtime_upgrade() -> Weight {
+		migrate_to_v1::<T>()
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn pre_upgrade() -> Result<Vec<u8>, sp_runtime::DispatchError> {
+		let prev_count = StorageNodes::<T>::iter().count();
+
+		Ok((prev_count as u64).encode())
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn post_upgrade(prev_state: Vec<u8>) -> Result<(), sp_runtime::DispatchError> {
+		let prev_count: u64 =
+			Decode::decode(&mut &prev_state[..]).expect("pre_upgrade provides a valid state; qed");
+
+		let post_count = StorageNodes::<T>::iter().count() as u64;
+		ensure!(
+			prev_count == post_count,
+			"the storage node count before and after the migration should be the same"
+		);
+
+		let current_version = Pallet::<T>::current_storage_version();
+		let on_chain_version = Pallet::<T>::on_chain_storage_version();
+
+		ensure!(current_version == 1, "must_upgrade");
+		ensure!(
+			current_version == on_chain_version,
+			"after migration, the current_version and on_chain_version should be the same"
+		);
+
+		// Ensure all nodes have total_usage set to None
+		for (_key, node) in StorageNodes::<T>::iter() {
+			ensure!(node.total_usage.is_none(), "total_usage should be None");
+		}
+
+		Ok(())
+	}
+}
+
+#[cfg(test)]
+#[cfg(feature = "try-runtime")]
+mod test {
+	use ddc_primitives::StorageNodeMode;
+	use frame_support::pallet_prelude::StorageVersion;
+
+	use super::*;
+	use crate::mock::{Test as T, *};
+
+	#[test]
+	fn storage_node_migration_works() {
+		ExtBuilder.build_and_execute(|| {
+			let node_pub_key0 = StorageNodePubKey::from([0; 32]);
+			let node_pub_key1 = StorageNodePubKey::from([1; 32]);
+			let node_pub_key2 = StorageNodePubKey::from([2; 32]);
+			let provider_id = AccountId::from([1; 32]);
+			let cluster_id = Some(ClusterId::from([1; 20]));
+
+			assert_eq!(StorageVersion::get::<Pallet<T>>(), 0);
+
+			let node1 = v0::StorageNode {
+				pub_key: node_pub_key1.clone(),
+				provider_id: provider_id.clone(),
+				cluster_id,
+				props: StorageNodeProps {
+					mode: StorageNodeMode::Storage,
+					host: vec![3u8; 255].try_into().unwrap(),
+					domain: vec![4u8; 255].try_into().unwrap(),
+					ssl: true,
+					http_port: 45000u16,
+					grpc_port: 55000u16,
+					p2p_port: 65000u16,
+				},
+			};
+
+			v0::StorageNodes::<T>::insert(node_pub_key1.clone(), node1);
+			let node2 = v0::StorageNode {
+				pub_key: node_pub_key2.clone(),
+				provider_id: provider_id.clone(),
+				cluster_id,
+				props: StorageNodeProps {
+					mode: StorageNodeMode::Storage,
+					host: vec![3u8; 255].try_into().unwrap(),
+					domain: vec![4u8; 255].try_into().unwrap(),
+					ssl: true,
+					http_port: 45000u16,
+					grpc_port: 55000u16,
+					p2p_port: 65000u16,
+				},
+			};
+
+			v0::StorageNodes::<T>::insert(node_pub_key2.clone(), node2);
+			let node_count = v0::StorageNodes::<T>::iter_values().count() as u32;
+
+			assert_eq!(node_count, 2);
+			let state = MigrateToV1::<T>::pre_upgrade().unwrap();
+			let _weight = MigrateToV1::<T>::on_runtime_upgrade();
+			MigrateToV1::<T>::post_upgrade(state).unwrap();
+
+			let node_count_after_upgrade = StorageNodes::<T>::iter_values().count() as u32;
+
+			assert_eq!(StorageVersion::get::<Pallet<T>>(), 1);
+			assert_eq!(node_count_after_upgrade, 2);
+			assert_eq!(StorageNodes::<T>::get(node_pub_key0), None);
+			assert!(StorageNodes::<T>::get(node_pub_key1.clone()).is_some());
+			assert!(StorageNodes::<T>::get(node_pub_key2.clone()).is_some());
+			assert_eq!(StorageNodes::<T>::get(node_pub_key1).unwrap().total_usage, None);
+			assert_eq!(StorageNodes::<T>::get(node_pub_key2).unwrap().total_usage, None);
+		});
+	}
+}

pallets/ddc-nodes/src/mock.rs

@@ -12,17 +12,18 @@ use frame_system::mocking::{MockBlock, MockUncheckedExtrinsic};
 use sp_core::H256;
 use sp_io::TestExternalities;
 use sp_runtime::{
-	traits::{BlakeTwo256, IdentityLookup},
-	BuildStorage,
+	traits::{BlakeTwo256, IdentifyAccount, IdentityLookup, Verify},
+	BuildStorage, MultiSignature,
 };
 
 use crate::{self as pallet_ddc_nodes, *};
 
 /// The AccountId alias in this test module.
-pub(crate) type AccountId = u64;
+pub type AccountId = <<Signature as Verify>::Signer as IdentifyAccount>::AccountId;
 pub(crate) type AccountIndex = u64;
 pub(crate) type BlockNumber = u64;
 pub(crate) type Balance = u128;
+pub type Signature = MultiSignature;
 
 type UncheckedExtrinsic = MockUncheckedExtrinsic<Test>;
 type Block = MockBlock<Test>;
@@ -125,9 +126,12 @@ impl ExtBuilder {
 		sp_tracing::try_init_simple();
 
 		let mut t = frame_system::GenesisConfig::<Test>::default().build_storage().unwrap();
-
-		let _ = pallet_balances::GenesisConfig::<Test> { balances: vec![(1, 100), (2, 100)] }
-			.assimilate_storage(&mut t);
+		let account_id1 = AccountId::from([1; 32]);
+		let account_id2 = AccountId::from([2; 32]);
+		let _ = pallet_balances::GenesisConfig::<Test> {
+			balances: vec![(account_id1, 100), (account_id2, 100)],
+		}
+		.assimilate_storage(&mut t);
 
 		TestExternalities::new(t)
 	}

pallets/ddc-nodes/src/node.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::needless_lifetimes)] // ToDo
 
 use codec::{Decode, Encode};
-use ddc_primitives::{NodeParams, NodePubKey, NodeType};
+use ddc_primitives::{NodeParams, NodePubKey, NodeType, NodeUsage};
 use scale_info::TypeInfo;
 use sp_runtime::RuntimeDebug;
 
@@ -31,6 +31,7 @@ pub trait NodeTrait<T: frame_system::Config> {
 	fn get_cluster_id(&self) -> &Option<ClusterId>;
 	fn set_cluster_id(&mut self, cluster_id: Option<ClusterId>);
 	fn get_type(&self) -> NodeType;
+	fn get_total_usage(&self) -> &Option<NodeUsage>;
 }
 
 impl<T: frame_system::Config> Node<T> {
@@ -77,6 +78,11 @@ impl<T: frame_system::Config> NodeTrait<T> for Node<T> {
 			Node::Storage(node) => node.get_cluster_id(),
 		}
 	}
+	fn get_total_usage(&self) -> &Option<NodeUsage> {
+		match &self {
+			Node::Storage(node) => node.get_total_usage(),
+		}
+	}
 	fn set_cluster_id(&mut self, cluster_id: Option<ClusterId>) {
 		match self {
 			Node::Storage(node) => node.set_cluster_id(cluster_id),

pallets/ddc-nodes/src/storage_node.rs

@@ -1,6 +1,6 @@
 use codec::{Decode, Encode};
 use ddc_primitives::{
-	ClusterId, NodeParams, NodePubKey, NodeType, StorageNodeMode, StorageNodePubKey,
+	ClusterId, NodeParams, NodePubKey, NodeType, NodeUsage, StorageNodeMode, StorageNodePubKey,
 };
 use frame_support::{parameter_types, BoundedVec};
 use scale_info::TypeInfo;
@@ -21,6 +21,7 @@ pub struct StorageNode<T: frame_system::Config> {
 	pub provider_id: T::AccountId,
 	pub cluster_id: Option<ClusterId>,
 	pub props: StorageNodeProps,
+	pub total_usage: Option<NodeUsage>,
 }
 
 #[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo, PartialEq, Serialize, Deserialize)]
@@ -61,6 +62,7 @@ impl<T: frame_system::Config> StorageNode<T> {
 						grpc_port: node_params.grpc_port,
 						p2p_port: node_params.p2p_port,
 					},
+					total_usage: None,
 				}),
 			},
 		}
@@ -106,6 +108,9 @@ impl<T: frame_system::Config> NodeTrait<T> for StorageNode<T> {
 	fn get_cluster_id(&self) -> &Option<ClusterId> {
 		&self.cluster_id
 	}
+	fn get_total_usage(&self) -> &Option<NodeUsage> {
+		&self.total_usage
+	}
 	fn set_cluster_id(&mut self, cluster_id: Option<ClusterId>) {
 		self.cluster_id = cluster_id;
 	}