From c18ca2533736d3f2674dcd8fbf093083408c1b96 Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Wed, 19 Jun 2024 11:09:59 +0200 Subject: [PATCH 1/7] initial add_snapshot_test_data setup --- .../examples/snapshot_test_data.rs | 44 +++++++++++++++++++ .../iota-genesis-builder/src/stardust/mod.rs | 1 + .../src/stardust/test_data/mod.rs | 40 +++++++++++++++++ .../vesting_schedule_multiple_addresses.rs | 12 +++++ 4 files changed, 97 insertions(+) create mode 100644 crates/iota-genesis-builder/examples/snapshot_test_data.rs create mode 100644 crates/iota-genesis-builder/src/stardust/test_data/mod.rs create mode 100644 crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs diff --git a/crates/iota-genesis-builder/examples/snapshot_test_data.rs b/crates/iota-genesis-builder/examples/snapshot_test_data.rs new file mode 100644 index 00000000000..788ff0ae014 --- /dev/null +++ b/crates/iota-genesis-builder/examples/snapshot_test_data.rs @@ -0,0 +1,44 @@ +// Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +//! + +use std::{fs::File, path::Path}; + +use iota_genesis_builder::stardust::{ + parse::FullSnapshotParser, test_data::add_snapshot_test_data, + types::snapshot::TOTAL_SUPPLY_IOTA, +}; + +fn parse_snapshot>(path: P) -> anyhow::Result<()> { + let file = File::open(path)?; + let parser = FullSnapshotParser::new(file)?; + + println!("Output count: {}", parser.header.output_count()); + + let total_supply = parser.outputs().try_fold(0, |acc, output| { + Ok::<_, anyhow::Error>(acc + output?.1.amount()) + })?; + + assert_eq!(total_supply, TOTAL_SUPPLY_IOTA); + + println!("Total supply: {total_supply}"); + + Ok(()) +} + +fn main() -> anyhow::Result<()> { + let Some(current_path) = std::env::args().nth(1) else { + anyhow::bail!("please provide path to the full-snapshot file"); + }; + let mut new_path = String::from("test-"); + new_path.push_str(¤t_path); + + parse_snapshot(¤t_path)?; + + add_snapshot_test_data(¤t_path, &new_path)?; + + parse_snapshot(&new_path)?; + + Ok(()) +} diff --git a/crates/iota-genesis-builder/src/stardust/mod.rs b/crates/iota-genesis-builder/src/stardust/mod.rs index 617b442df70..31f036d9527 100644 --- a/crates/iota-genesis-builder/src/stardust/mod.rs +++ b/crates/iota-genesis-builder/src/stardust/mod.rs @@ -9,4 +9,5 @@ pub mod error; pub mod migration; pub mod native_token; pub mod parse; +pub mod test_data; pub mod types; diff --git a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs new file mode 100644 index 00000000000..607d0ba6ab2 --- /dev/null +++ b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs @@ -0,0 +1,40 @@ +// Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +mod vesting_schedule_multiple_addresses; + +use std::{ + fs::{File, OpenOptions}, + io::BufWriter, + path::Path, +}; + +use packable::{packer::IoPacker, Packable}; + +use crate::stardust::parse::FullSnapshotParser; + +pub fn add_snapshot_test_data + core::fmt::Debug>( + current_path: P, + new_path: P, +) -> anyhow::Result<()> { + let current_file = File::open(current_path)?; + let new_file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(new_path)?; + let mut writer = IoPacker::new(BufWriter::new(new_file)); + + let parser = FullSnapshotParser::new(current_file)?; + parser.header.pack(&mut writer)?; + + parser.outputs().for_each(|res| { + let (output_header, output) = res.unwrap(); + output_header.pack(&mut writer).unwrap(); + output.pack(&mut writer).unwrap(); + }); + + // let outputs = vesting_schedule_multiple_addresses::outputs(); + + Ok(()) +} diff --git a/crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs b/crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs new file mode 100644 index 00000000000..24d1ca449be --- /dev/null +++ b/crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs @@ -0,0 +1,12 @@ +// Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use iota_sdk::types::block::output::Output; + +use crate::stardust::types::snapshot::OutputHeader; + +pub(crate) fn outputs() -> Vec<(OutputHeader, Output)> { + let outputs = Vec::new(); + + outputs +} From 67aa756c63a457dff914c5488383dfe2abc3f9bc Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Wed, 19 Jun 2024 11:13:53 +0200 Subject: [PATCH 2/7] increment snapshot header output count --- .../iota-genesis-builder/src/stardust/test_data/mod.rs | 9 ++++++--- .../iota-genesis-builder/src/stardust/types/snapshot.rs | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs index 607d0ba6ab2..c8d53d92202 100644 --- a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs +++ b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs @@ -25,7 +25,12 @@ pub fn add_snapshot_test_data + core::fmt::Debug>( .open(new_path)?; let mut writer = IoPacker::new(BufWriter::new(new_file)); - let parser = FullSnapshotParser::new(current_file)?; + let mut parser = FullSnapshotParser::new(current_file)?; + + let new_outputs = vesting_schedule_multiple_addresses::outputs(); + + parser.header.output_count += new_outputs.len() as u64; + parser.header.pack(&mut writer)?; parser.outputs().for_each(|res| { @@ -34,7 +39,5 @@ pub fn add_snapshot_test_data + core::fmt::Debug>( output.pack(&mut writer).unwrap(); }); - // let outputs = vesting_schedule_multiple_addresses::outputs(); - Ok(()) } diff --git a/crates/iota-genesis-builder/src/stardust/types/snapshot.rs b/crates/iota-genesis-builder/src/stardust/types/snapshot.rs index b77ef8d2208..c514a968805 100644 --- a/crates/iota-genesis-builder/src/stardust/types/snapshot.rs +++ b/crates/iota-genesis-builder/src/stardust/types/snapshot.rs @@ -110,7 +110,7 @@ pub struct FullSnapshotHeader { treasury_output_milestone_id: MilestoneId, treasury_output_amount: u64, parameters_milestone_option: MilestoneOption, - output_count: u64, + pub(crate) output_count: u64, milestone_diff_count: u32, sep_count: u16, } From 50e15a16745f8da72fb7d22ebcc3a66e74cce900 Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Wed, 19 Jun 2024 11:43:13 +0200 Subject: [PATCH 3/7] add new outputs to snapshot --- .../src/stardust/test_data/mod.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs index c8d53d92202..6cacc89c4ae 100644 --- a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs +++ b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs @@ -33,11 +33,14 @@ pub fn add_snapshot_test_data + core::fmt::Debug>( parser.header.pack(&mut writer)?; - parser.outputs().for_each(|res| { - let (output_header, output) = res.unwrap(); - output_header.pack(&mut writer).unwrap(); - output.pack(&mut writer).unwrap(); - }); + parser + .outputs() + .filter_map(|o| o.ok()) + .chain(new_outputs) + .for_each(|(output_header, output)| { + output_header.pack(&mut writer).unwrap(); + output.pack(&mut writer).unwrap(); + }); Ok(()) } From 54ed7eb587ede5ce2af91a690a9c449ba0754372 Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Wed, 19 Jun 2024 14:27:52 +0200 Subject: [PATCH 4/7] change the header total supply --- .../examples/snapshot_test_data.rs | 8 ++-- .../src/stardust/parse.rs | 13 +++++-- .../src/stardust/test_data/dummy.rs | 38 +++++++++++++++++++ .../src/stardust/test_data/mod.rs | 30 +++++++++++++-- .../vesting_schedule_multiple_addresses.rs | 12 ------ .../src/stardust/types/snapshot.rs | 2 +- 6 files changed, 79 insertions(+), 24 deletions(-) create mode 100644 crates/iota-genesis-builder/src/stardust/test_data/dummy.rs delete mode 100644 crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs diff --git a/crates/iota-genesis-builder/examples/snapshot_test_data.rs b/crates/iota-genesis-builder/examples/snapshot_test_data.rs index 788ff0ae014..5610bac82d7 100644 --- a/crates/iota-genesis-builder/examples/snapshot_test_data.rs +++ b/crates/iota-genesis-builder/examples/snapshot_test_data.rs @@ -7,7 +7,6 @@ use std::{fs::File, path::Path}; use iota_genesis_builder::stardust::{ parse::FullSnapshotParser, test_data::add_snapshot_test_data, - types::snapshot::TOTAL_SUPPLY_IOTA, }; fn parse_snapshot>(path: P) -> anyhow::Result<()> { @@ -16,13 +15,14 @@ fn parse_snapshot>(path: P) -> anyhow::Result<()> { println!("Output count: {}", parser.header.output_count()); - let total_supply = parser.outputs().try_fold(0, |acc, output| { + let total_supply_header = parser.total_supply()?; + let total_supply_outputs = parser.outputs().try_fold(0, |acc, output| { Ok::<_, anyhow::Error>(acc + output?.1.amount()) })?; - assert_eq!(total_supply, TOTAL_SUPPLY_IOTA); + assert_eq!(total_supply_header, total_supply_outputs); - println!("Total supply: {total_supply}"); + println!("Total supply: {total_supply_header}"); Ok(()) } diff --git a/crates/iota-genesis-builder/src/stardust/parse.rs b/crates/iota-genesis-builder/src/stardust/parse.rs index aaf481424f8..0e0d47386c3 100644 --- a/crates/iota-genesis-builder/src/stardust/parse.rs +++ b/crates/iota-genesis-builder/src/stardust/parse.rs @@ -48,17 +48,22 @@ impl FullSnapshotParser { self.header.target_milestone_timestamp() } - /// Provide the network main token total supply through the snapshot - /// protocol parameters. - pub fn total_supply(&self) -> Result { + /// Provide the protocol parameters extracted from the snapshot header. + pub fn protocol_parameters(&self) -> Result { if let MilestoneOption::Parameters(params) = self.header.parameters_milestone_option() { let protocol_params = ::unpack_unverified( params.binary_parameters(), ) .expect("invalid protocol params"); - Ok(protocol_params.token_supply()) + Ok(protocol_params) } else { Err(StardustError::HornetSnapshotParametersNotFound.into()) } } + + /// Provide the network main token total supply through the snapshot + /// protocol parameters. + pub fn total_supply(&self) -> Result { + self.protocol_parameters().map(|p| p.token_supply()) + } } diff --git a/crates/iota-genesis-builder/src/stardust/test_data/dummy.rs b/crates/iota-genesis-builder/src/stardust/test_data/dummy.rs new file mode 100644 index 00000000000..033dea3a28d --- /dev/null +++ b/crates/iota-genesis-builder/src/stardust/test_data/dummy.rs @@ -0,0 +1,38 @@ +// Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use std::str::FromStr; + +use iota_sdk::types::block::{ + address::Ed25519Address, + output::{unlock_condition::AddressUnlockCondition, BasicOutputBuilder, Output}, + payload::transaction::TransactionId, +}; + +use crate::stardust::types::snapshot::OutputHeader; + +pub(crate) fn outputs() -> Vec<(OutputHeader, Output)> { + let mut outputs = Vec::new(); + + let output_header = OutputHeader::new_testing( + *TransactionId::from_str( + "0xb191c4bc825ac6983789e50545d5ef07a1d293a98ad974fc9498cb1812345678", + ) + .unwrap(), + rand::random(), + rand::random(), + rand::random(), + ); + let output = Output::from( + BasicOutputBuilder::new_with_amount(1_000_000) + .add_unlock_condition(AddressUnlockCondition::new(Ed25519Address::from( + rand::random::<[u8; Ed25519Address::LENGTH]>(), + ))) + .finish() + .unwrap(), + ); + + outputs.push((output_header, output)); + + outputs +} diff --git a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs index 6cacc89c4ae..2b95bd28801 100644 --- a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs +++ b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs @@ -1,7 +1,7 @@ // Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -mod vesting_schedule_multiple_addresses; +mod dummy; use std::{ fs::{File, OpenOptions}, @@ -9,7 +9,11 @@ use std::{ path::Path, }; -use packable::{packer::IoPacker, Packable}; +use iota_sdk::types::block::{ + payload::milestone::{MilestoneOption, ParametersMilestoneOption}, + protocol::ProtocolParameters, +}; +use packable::{packer::IoPacker, Packable, PackableExt}; use crate::stardust::parse::FullSnapshotParser; @@ -27,10 +31,30 @@ pub fn add_snapshot_test_data + core::fmt::Debug>( let mut parser = FullSnapshotParser::new(current_file)?; - let new_outputs = vesting_schedule_multiple_addresses::outputs(); + let new_outputs = dummy::outputs(); parser.header.output_count += new_outputs.len() as u64; + let params = parser.protocol_parameters()?; + let new_params = ProtocolParameters::new( + params.protocol_version(), + params.network_name().to_owned(), + params.bech32_hrp(), + params.min_pow_score(), + params.below_max_depth(), + *params.rent_structure(), + params.token_supply() + new_outputs.iter().map(|o| o.1.amount()).sum::(), + )?; + + if let MilestoneOption::Parameters(params) = &parser.header.parameters_milestone_option { + parser.header.parameters_milestone_option = + MilestoneOption::Parameters(ParametersMilestoneOption::new( + params.target_milestone_index(), + params.protocol_version(), + new_params.pack_to_vec(), + )?); + } + parser.header.pack(&mut writer)?; parser diff --git a/crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs b/crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs deleted file mode 100644 index 24d1ca449be..00000000000 --- a/crates/iota-genesis-builder/src/stardust/test_data/vesting_schedule_multiple_addresses.rs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use iota_sdk::types::block::output::Output; - -use crate::stardust::types::snapshot::OutputHeader; - -pub(crate) fn outputs() -> Vec<(OutputHeader, Output)> { - let outputs = Vec::new(); - - outputs -} diff --git a/crates/iota-genesis-builder/src/stardust/types/snapshot.rs b/crates/iota-genesis-builder/src/stardust/types/snapshot.rs index c514a968805..33df6f6df33 100644 --- a/crates/iota-genesis-builder/src/stardust/types/snapshot.rs +++ b/crates/iota-genesis-builder/src/stardust/types/snapshot.rs @@ -109,7 +109,7 @@ pub struct FullSnapshotHeader { ledger_milestone_index: MilestoneIndex, treasury_output_milestone_id: MilestoneId, treasury_output_amount: u64, - parameters_milestone_option: MilestoneOption, + pub(crate) parameters_milestone_option: MilestoneOption, pub(crate) output_count: u64, milestone_diff_count: u32, sep_count: u16, From 62d7e4671aae82e975636ab87e03aa25161e0efb Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Wed, 19 Jun 2024 15:20:13 +0200 Subject: [PATCH 5/7] add some comments --- crates/iota-genesis-builder/src/stardust/test_data/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs index 2b95bd28801..fe9c7f0863a 100644 --- a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs +++ b/crates/iota-genesis-builder/src/stardust/test_data/mod.rs @@ -17,6 +17,7 @@ use packable::{packer::IoPacker, Packable, PackableExt}; use crate::stardust::parse::FullSnapshotParser; +/// Adds outputs to test specific and intricate scenario in the full snapshot. pub fn add_snapshot_test_data + core::fmt::Debug>( current_path: P, new_path: P, @@ -28,13 +29,14 @@ pub fn add_snapshot_test_data + core::fmt::Debug>( .truncate(true) .open(new_path)?; let mut writer = IoPacker::new(BufWriter::new(new_file)); - let mut parser = FullSnapshotParser::new(current_file)?; let new_outputs = dummy::outputs(); + // Increments the output count according to newly generated outputs. parser.header.output_count += new_outputs.len() as u64; + // Creates new protocol parameters to increase the total supply according to newly generated outputs. let params = parser.protocol_parameters()?; let new_params = ProtocolParameters::new( params.protocol_version(), @@ -45,7 +47,6 @@ pub fn add_snapshot_test_data + core::fmt::Debug>( *params.rent_structure(), params.token_supply() + new_outputs.iter().map(|o| o.1.amount()).sum::(), )?; - if let MilestoneOption::Parameters(params) = &parser.header.parameters_milestone_option { parser.header.parameters_milestone_option = MilestoneOption::Parameters(ParametersMilestoneOption::new( @@ -55,8 +56,10 @@ pub fn add_snapshot_test_data + core::fmt::Debug>( )?); } + // Writes the new header. parser.header.pack(&mut writer)?; + // Writes previous and new outputs. parser .outputs() .filter_map(|o| o.ok()) From 1176bae9556dd90ecf4d956a19e71c4eca8638e3 Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Wed, 19 Jun 2024 15:23:34 +0200 Subject: [PATCH 6/7] rename and add comment --- crates/iota-genesis-builder/examples/snapshot_test_data.rs | 6 +++--- crates/iota-genesis-builder/src/stardust/mod.rs | 2 +- .../src/stardust/{test_data => test_outputs}/dummy.rs | 0 .../src/stardust/{test_data => test_outputs}/mod.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename crates/iota-genesis-builder/src/stardust/{test_data => test_outputs}/dummy.rs (100%) rename crates/iota-genesis-builder/src/stardust/{test_data => test_outputs}/mod.rs (97%) diff --git a/crates/iota-genesis-builder/examples/snapshot_test_data.rs b/crates/iota-genesis-builder/examples/snapshot_test_data.rs index 5610bac82d7..2a0ef37afba 100644 --- a/crates/iota-genesis-builder/examples/snapshot_test_data.rs +++ b/crates/iota-genesis-builder/examples/snapshot_test_data.rs @@ -1,12 +1,12 @@ // Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -//! +//! Example to add test outputs to a full snapshot. use std::{fs::File, path::Path}; use iota_genesis_builder::stardust::{ - parse::FullSnapshotParser, test_data::add_snapshot_test_data, + parse::FullSnapshotParser, test_outputs::add_snapshot_test_outputs, }; fn parse_snapshot>(path: P) -> anyhow::Result<()> { @@ -36,7 +36,7 @@ fn main() -> anyhow::Result<()> { parse_snapshot(¤t_path)?; - add_snapshot_test_data(¤t_path, &new_path)?; + add_snapshot_test_outputs(¤t_path, &new_path)?; parse_snapshot(&new_path)?; diff --git a/crates/iota-genesis-builder/src/stardust/mod.rs b/crates/iota-genesis-builder/src/stardust/mod.rs index 31f036d9527..bd31ffcc634 100644 --- a/crates/iota-genesis-builder/src/stardust/mod.rs +++ b/crates/iota-genesis-builder/src/stardust/mod.rs @@ -9,5 +9,5 @@ pub mod error; pub mod migration; pub mod native_token; pub mod parse; -pub mod test_data; +pub mod test_outputs; pub mod types; diff --git a/crates/iota-genesis-builder/src/stardust/test_data/dummy.rs b/crates/iota-genesis-builder/src/stardust/test_outputs/dummy.rs similarity index 100% rename from crates/iota-genesis-builder/src/stardust/test_data/dummy.rs rename to crates/iota-genesis-builder/src/stardust/test_outputs/dummy.rs diff --git a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs b/crates/iota-genesis-builder/src/stardust/test_outputs/mod.rs similarity index 97% rename from crates/iota-genesis-builder/src/stardust/test_data/mod.rs rename to crates/iota-genesis-builder/src/stardust/test_outputs/mod.rs index fe9c7f0863a..1349f96ae02 100644 --- a/crates/iota-genesis-builder/src/stardust/test_data/mod.rs +++ b/crates/iota-genesis-builder/src/stardust/test_outputs/mod.rs @@ -18,7 +18,7 @@ use packable::{packer::IoPacker, Packable, PackableExt}; use crate::stardust::parse::FullSnapshotParser; /// Adds outputs to test specific and intricate scenario in the full snapshot. -pub fn add_snapshot_test_data + core::fmt::Debug>( +pub fn add_snapshot_test_outputs + core::fmt::Debug>( current_path: P, new_path: P, ) -> anyhow::Result<()> { From e8a73330fc556d2f46e953cf86ddf021069b0a72 Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Wed, 19 Jun 2024 15:26:52 +0200 Subject: [PATCH 7/7] rename example --- .../examples/{snapshot_test_data.rs => snapshot_test_outputs.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crates/iota-genesis-builder/examples/{snapshot_test_data.rs => snapshot_test_outputs.rs} (100%) diff --git a/crates/iota-genesis-builder/examples/snapshot_test_data.rs b/crates/iota-genesis-builder/examples/snapshot_test_outputs.rs similarity index 100% rename from crates/iota-genesis-builder/examples/snapshot_test_data.rs rename to crates/iota-genesis-builder/examples/snapshot_test_outputs.rs