From 62c377c6ae298665598b83aabb859dbaadb728a3 Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Tue, 4 Jul 2023 10:42:45 +0300 Subject: [PATCH 1/8] feat: Proptests for Committee --- Cargo.lock | 115 +++++++++++++++++++++++++- node/narwhal/Cargo.toml | 5 ++ node/narwhal/src/helpers/committee.rs | 90 ++++++++++++++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index bc20d6bc78..ceba0bec7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,6 +454,21 @@ dependencies = [ "syn 2.0.23", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -1621,6 +1636,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + [[package]] name = "librocksdb-sys" version = "0.11.0+8.1.1" @@ -1954,6 +1975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", + "libm 0.2.7", ] [[package]] @@ -2064,7 +2086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282" dependencies = [ "cfg-if", - "libm", + "libm 0.1.4", ] [[package]] @@ -2243,6 +2265,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" +dependencies = [ + "bit-set", + "bitflags 1.3.2", + "byteorder", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.6.29", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "quanta" version = "0.11.1" @@ -2259,6 +2301,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.23.1" @@ -2552,6 +2600,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "rusty-hook" version = "0.11.2" @@ -3010,14 +3070,17 @@ dependencies = [ "indexmap 2.0.0", "open", "parking_lot", + "proptest", "rand", "rand_chacha", + "rand_xorshift", "serde", "sha2", "snarkos-account", "snarkos-node-messages", "snarkos-node-tcp", "snarkvm", + "test-strategy", "time", "tokio", "tokio-stream", @@ -3833,6 +3896,29 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "structmeta" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "104842d6278bf64aa9d2f182ba4bde31e8aec7a131d29b7f444bb9b344a09e2a" +dependencies = [ + "proc-macro2", + "quote 1.0.29", + "structmeta-derive", + "syn 1.0.109", +] + +[[package]] +name = "structmeta-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24420be405b590e2d746d83b01f09af673270cf80e9b003a5fa7b651c58c7d93" +dependencies = [ + "proc-macro2", + "quote 1.0.29", + "syn 1.0.109", +] + [[package]] name = "subtle" version = "2.5.0" @@ -3901,6 +3987,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "test-strategy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61348cf95c0dfa12f0b6155f5b564d2806f518620a28263280ae357b41c96b4a" +dependencies = [ + "proc-macro2", + "quote 1.0.29", + "structmeta", + "syn 1.0.109", +] + [[package]] name = "thiserror" version = "1.0.40" @@ -4300,6 +4398,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.6.0" @@ -4429,6 +4533,15 @@ version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.3.3" diff --git a/node/narwhal/Cargo.toml b/node/narwhal/Cargo.toml index 2b54217555..e15936f5b3 100644 --- a/node/narwhal/Cargo.toml +++ b/node/narwhal/Cargo.toml @@ -107,3 +107,8 @@ features = [ "fs", "trace" ] [dev-dependencies.tracing-subscriber] version = "0.3" features = [ "env-filter" ] + +[dev-dependencies] +test-strategy = "0.3.0" +proptest = "1.0.0" +rand_xorshift = "0.3.0" \ No newline at end of file diff --git a/node/narwhal/src/helpers/committee.rs b/node/narwhal/src/helpers/committee.rs index 70f804102a..e68ddaf30b 100644 --- a/node/narwhal/src/helpers/committee.rs +++ b/node/narwhal/src/helpers/committee.rs @@ -117,3 +117,93 @@ impl Committee { Ok(power) } } + +#[cfg(test)] +pub mod tests { + use crate::helpers::Committee; + use anyhow::Result; + use indexmap::IndexMap; + use proptest::sample::size_range; + use rand::SeedableRng; + use snarkos_account::Account; + use snarkvm::prelude::Testnet3; + use test_strategy::{proptest, Arbitrary}; + + type N = Testnet3; + + #[derive(Arbitrary, Debug)] + pub struct CommitteeInput { + #[strategy(0u64..)] + pub round: u64, + #[any(size_range(0..32).lift())] + pub validators: Vec, + } + + #[derive(Arbitrary, Debug, Clone)] + pub struct Validator { + #[strategy(..5_000_000_000u64)] + pub stake: u64, + account_seed: u64, + } + + impl Validator { + pub fn get_account(&self) -> Account { + match Account::new(&mut rand_chacha::ChaChaRng::seed_from_u64(self.account_seed)) { + Ok(account) => account, + Err(err) => panic!("Failed to create account {err}"), + } + } + } + + impl CommitteeInput { + pub fn to_committee(&self) -> Result> { + let mut index_map = IndexMap::new(); + for validator in self.validators.iter() { + index_map.insert(validator.get_account().address(), validator.stake); + } + Committee::new(self.round, index_map) + } + + pub fn is_valid(&self) -> bool { + self.round > 0 && self.validators.len() >= 4 + } + } + + #[proptest] + fn committee_members(input: CommitteeInput) { + let committee = match input.to_committee() { + Ok(committee) => { + assert!(input.is_valid()); + committee + } + Err(err) => { + assert!(!input.is_valid()); + match err.to_string().as_str() { + "Round must be nonzero" => assert_eq!(input.round, 0), + "Committee must have at least 4 members" => assert!(input.validators.len() < 4), + _ => panic!("Unexpected error: {err}"), + } + return Ok(()); + } + }; + + let validators = input.validators; + + let mut total_stake = 0; + for v in validators.iter() { + total_stake += v.stake; + } + + assert_eq!(committee.committee_size(), validators.len()); + assert_eq!(committee.total_stake().unwrap(), total_stake); + for v in validators.iter() { + let address = v.get_account().address(); + assert!(committee.is_committee_member(address)); + assert_eq!(committee.get_stake(address), v.stake); + } + let quorum_threshold = committee.quorum_threshold().unwrap(); + let availability_threshold = committee.availability_threshold().unwrap(); + // (2f + 1) + (f + 1) - 1 = 3f + 1 = N + assert_eq!(quorum_threshold + availability_threshold - 1, total_stake); + } +} From 05e6978f7d82d099d65b2b616cb8752572075453 Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Wed, 5 Jul 2023 11:11:46 +0300 Subject: [PATCH 2/8] feat: first proptests for Gateway --- node/narwhal/src/gateway.rs | 59 +++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/node/narwhal/src/gateway.rs b/node/narwhal/src/gateway.rs index 9fafabfd5e..c5647a866c 100644 --- a/node/narwhal/src/gateway.rs +++ b/node/narwhal/src/gateway.rs @@ -76,6 +76,7 @@ impl Gateway { pub fn new(committee: Arc>>, account: Account, dev: Option) -> Result { // Initialize the gateway IP. let ip = match dev { + // TODO change dev to Option, otherwise there is potential overflow Some(dev) => SocketAddr::from_str(&format!("127.0.0.1:{}", MEMORY_POOL_PORT + dev)), None => SocketAddr::from_str(&format!("0.0.0.0:{}", MEMORY_POOL_PORT)), }?; @@ -793,3 +794,61 @@ impl Gateway { None } } + +mod tests { + use crate::{ + helpers::tests::{CommitteeInput, Validator}, + Gateway, + MAX_COMMITTEE_SIZE, + MEMORY_POOL_PORT, + }; + use parking_lot::RwLock; + use snarkos_node_tcp::P2P; + use snarkvm::prelude::Testnet3; + use std::{ + net::{IpAddr, Ipv4Addr}, + sync::Arc, + }; + use test_strategy::{proptest, Arbitrary}; + + type N = Testnet3; + + #[derive(Arbitrary, Debug)] + struct GatewayInput { + #[filter(CommitteeInput::is_valid)] + committee_input: CommitteeInput, + node_validator: Validator, + dev: Option, + } + + #[proptest] + fn gateway_initialization(input: GatewayInput) { + let GatewayInput { committee_input, node_validator, dev } = input; + let committee = committee_input.to_committee().unwrap(); + let account = node_validator.get_account(); + let address = account.address(); + + let gateway = match dev { + Some(dev) => { + let dev_option = Some(dev as u16); + let gateway = Gateway::new(Arc::new(RwLock::new(committee)), account, dev_option).unwrap(); + let tcp_config = gateway.tcp().config(); + let expected_port = MEMORY_POOL_PORT + (dev as u16); + assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::LOCALHOST))); + assert_eq!(tcp_config.desired_listening_port, Some(expected_port)); + gateway + } + None => { + let gateway = Gateway::new(Arc::new(RwLock::new(committee)), account, None).unwrap(); + + let tcp_config = gateway.tcp().config(); + assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED))); + assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT)); + gateway + } + }; + let tcp_config = gateway.tcp().config(); + assert_eq!(tcp_config.max_connections, MAX_COMMITTEE_SIZE); + assert_eq!(gateway.account().address(), address); + } +} From bca259fb792d71f9d8f7bbae7ab2a885dfa0685a Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Wed, 5 Jul 2023 11:53:02 +0300 Subject: [PATCH 3/8] chore: some test cleanup for Gateway tests --- node/narwhal/src/gateway.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/node/narwhal/src/gateway.rs b/node/narwhal/src/gateway.rs index c5647a866c..41296f0779 100644 --- a/node/narwhal/src/gateway.rs +++ b/node/narwhal/src/gateway.rs @@ -821,26 +821,33 @@ mod tests { dev: Option, } + impl GatewayInput { + pub fn to_gateway(&self) -> Gateway { + let committee = self.committee_input.to_committee().unwrap(); + let account = self.node_validator.get_account(); + let dev = match self.dev { + None => None, + Some(dev) => Some(dev as u16), + }; + Gateway::new(Arc::new(RwLock::new(committee)), account, dev).unwrap() + } + } + #[proptest] fn gateway_initialization(input: GatewayInput) { - let GatewayInput { committee_input, node_validator, dev } = input; - let committee = committee_input.to_committee().unwrap(); - let account = node_validator.get_account(); + let account = input.node_validator.get_account(); let address = account.address(); - let gateway = match dev { + let gateway = match input.dev { Some(dev) => { - let dev_option = Some(dev as u16); - let gateway = Gateway::new(Arc::new(RwLock::new(committee)), account, dev_option).unwrap(); + let gateway = input.to_gateway(); let tcp_config = gateway.tcp().config(); - let expected_port = MEMORY_POOL_PORT + (dev as u16); assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::LOCALHOST))); - assert_eq!(tcp_config.desired_listening_port, Some(expected_port)); + assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT + (dev as u16))); gateway } None => { - let gateway = Gateway::new(Arc::new(RwLock::new(committee)), account, None).unwrap(); - + let gateway = input.to_gateway(); let tcp_config = gateway.tcp().config(); assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::UNSPECIFIED))); assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT)); From ff67ccba920b2dae0b4e7ee324979da876149c54 Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Wed, 5 Jul 2023 12:08:13 +0300 Subject: [PATCH 4/8] feat: First async proptest, for Gateway::run --- Cargo.lock | 18 ++++++++---------- node/narwhal/Cargo.toml | 5 ++--- node/narwhal/src/gateway.rs | 25 ++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ceba0bec7e..1b6ba7070d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3073,7 +3073,6 @@ dependencies = [ "proptest", "rand", "rand_chacha", - "rand_xorshift", "serde", "sha2", "snarkos-account", @@ -3898,25 +3897,25 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "structmeta" -version = "0.1.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104842d6278bf64aa9d2f182ba4bde31e8aec7a131d29b7f444bb9b344a09e2a" +checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" dependencies = [ "proc-macro2", "quote 1.0.29", "structmeta-derive", - "syn 1.0.109", + "syn 2.0.23", ] [[package]] name = "structmeta-derive" -version = "0.1.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24420be405b590e2d746d83b01f09af673270cf80e9b003a5fa7b651c58c7d93" +checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote 1.0.29", - "syn 1.0.109", + "syn 2.0.23", ] [[package]] @@ -3990,13 +3989,12 @@ dependencies = [ [[package]] name = "test-strategy" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61348cf95c0dfa12f0b6155f5b564d2806f518620a28263280ae357b41c96b4a" +source = "git+https://github.com/frozenlib/test-strategy/?branch=master#885013160c1431970a80faa43a5c96a9e05385ce" dependencies = [ "proc-macro2", "quote 1.0.29", "structmeta", - "syn 1.0.109", + "syn 2.0.23", ] [[package]] diff --git a/node/narwhal/Cargo.toml b/node/narwhal/Cargo.toml index e15936f5b3..ecb8811ce8 100644 --- a/node/narwhal/Cargo.toml +++ b/node/narwhal/Cargo.toml @@ -109,6 +109,5 @@ version = "0.3" features = [ "env-filter" ] [dev-dependencies] -test-strategy = "0.3.0" -proptest = "1.0.0" -rand_xorshift = "0.3.0" \ No newline at end of file +test-strategy = { git = "https://github.com/frozenlib/test-strategy/", branch = "master"} +proptest = "1.0.0" \ No newline at end of file diff --git a/node/narwhal/src/gateway.rs b/node/narwhal/src/gateway.rs index 41296f0779..fdfc2d2ddd 100644 --- a/node/narwhal/src/gateway.rs +++ b/node/narwhal/src/gateway.rs @@ -802,11 +802,12 @@ mod tests { MAX_COMMITTEE_SIZE, MEMORY_POOL_PORT, }; + use indexmap::IndexMap; use parking_lot::RwLock; use snarkos_node_tcp::P2P; use snarkvm::prelude::Testnet3; use std::{ - net::{IpAddr, Ipv4Addr}, + net::{IpAddr, Ipv4Addr, SocketAddr}, sync::Arc, }; use test_strategy::{proptest, Arbitrary}; @@ -858,4 +859,26 @@ mod tests { assert_eq!(tcp_config.max_connections, MAX_COMMITTEE_SIZE); assert_eq!(gateway.account().address(), address); } + + #[proptest(async = "tokio")] + async fn gateway_start(#[filter(|x| x.dev.is_some())] input: GatewayInput) { + let Some(dev) = input.dev else { unreachable!() }; + let mut gateway = input.to_gateway(); + let tcp_config = gateway.tcp().config(); + assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::LOCALHOST))); + assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT + (dev as u16))); + + match gateway.run(IndexMap::new()).await { + Ok(_) => { + assert_eq!( + gateway.local_ip(), + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + (dev as u16)) + ); + assert_eq!(gateway.num_workers(), 0); + } + Err(err) => { + panic!("Shouldn't fail because of {err}"); + } + } + } } From 7a896a6535f70acd2f986ff5711fed1ebc2d963c Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Wed, 5 Jul 2023 14:09:02 +0300 Subject: [PATCH 5/8] feat: Add missing test for Committee::to_next_round() --- node/narwhal/src/helpers/committee.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/node/narwhal/src/helpers/committee.rs b/node/narwhal/src/helpers/committee.rs index e68ddaf30b..74d96015ef 100644 --- a/node/narwhal/src/helpers/committee.rs +++ b/node/narwhal/src/helpers/committee.rs @@ -169,6 +169,18 @@ pub mod tests { } } + #[proptest] + fn committee_advance(#[filter(CommitteeInput::is_valid)] input: CommitteeInput) { + let committee = input.to_committee().unwrap(); + let current_round = input.round; + let current_members = committee.members(); + assert_eq!(committee.round(), current_round); + + let committee = committee.to_next_round().unwrap(); + assert_eq!(committee.round(), current_round + 1); + assert_eq!(committee.members(), current_members); + } + #[proptest] fn committee_members(input: CommitteeInput) { let committee = match input.to_committee() { From 326d4128ac18a69d0259920dcb576deeddd3f426 Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Wed, 5 Jul 2023 15:05:24 +0300 Subject: [PATCH 6/8] feat: generate real workers to proptest, test module chores --- node/narwhal/src/gateway.rs | 55 ++++++++++++++++++++++----- node/narwhal/src/helpers/committee.rs | 4 +- node/narwhal/src/helpers/storage.rs | 14 ++++++- node/narwhal/src/worker.rs | 15 ++++++++ 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/node/narwhal/src/gateway.rs b/node/narwhal/src/gateway.rs index fdfc2d2ddd..63be37a554 100644 --- a/node/narwhal/src/gateway.rs +++ b/node/narwhal/src/gateway.rs @@ -795,11 +795,19 @@ impl Gateway { } } -mod tests { +#[cfg(test)] +pub mod gateway_tests { use crate::{ - helpers::tests::{CommitteeInput, Validator}, + helpers::{ + committee_tests::{CommitteeInput, Validator}, + init_worker_channels, + storage_tests::StorageInput, + WorkerSender, + }, Gateway, + Worker, MAX_COMMITTEE_SIZE, + MAX_WORKERS, MEMORY_POOL_PORT, }; use indexmap::IndexMap; @@ -814,12 +822,15 @@ mod tests { type N = Testnet3; - #[derive(Arbitrary, Debug)] - struct GatewayInput { + #[derive(Arbitrary, Debug, Clone)] + pub struct GatewayInput { #[filter(CommitteeInput::is_valid)] - committee_input: CommitteeInput, - node_validator: Validator, - dev: Option, + pub committee_input: CommitteeInput, + pub node_validator: Validator, + pub dev: Option, + #[strategy(0..MAX_WORKERS)] + pub workers_count: u8, + pub worker_storage: StorageInput, } impl GatewayInput { @@ -832,6 +843,30 @@ mod tests { }; Gateway::new(Arc::new(RwLock::new(committee)), account, dev).unwrap() } + + pub async fn generate_workers( + &self, + gateway: &Gateway, + ) -> (IndexMap>, IndexMap>) { + // Construct a map of the worker senders. + let mut tx_workers = IndexMap::new(); + let mut workers = IndexMap::new(); + + // Initialize the workers. + for id in 0..self.workers_count { + // Construct the worker channels. + let (tx_worker, rx_worker) = init_worker_channels(); + // Construct the worker instance. + let mut worker = Worker::new(id, gateway.clone(), self.worker_storage.to_storage()).unwrap(); + // Run the worker instance. + worker.run(rx_worker).await.unwrap(); + + // Add the worker and the worker sender to maps + workers.insert(id, worker); + tx_workers.insert(id, tx_worker); + } + (workers, tx_workers) + } } #[proptest] @@ -863,18 +898,20 @@ mod tests { #[proptest(async = "tokio")] async fn gateway_start(#[filter(|x| x.dev.is_some())] input: GatewayInput) { let Some(dev) = input.dev else { unreachable!() }; + println!("{input:?}"); let mut gateway = input.to_gateway(); let tcp_config = gateway.tcp().config(); assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::LOCALHOST))); assert_eq!(tcp_config.desired_listening_port, Some(MEMORY_POOL_PORT + (dev as u16))); - match gateway.run(IndexMap::new()).await { + let (workers, worker_senders) = input.generate_workers(&gateway).await; + match gateway.run(worker_senders).await { Ok(_) => { assert_eq!( gateway.local_ip(), SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + (dev as u16)) ); - assert_eq!(gateway.num_workers(), 0); + assert_eq!(gateway.num_workers(), workers.len() as u8); } Err(err) => { panic!("Shouldn't fail because of {err}"); diff --git a/node/narwhal/src/helpers/committee.rs b/node/narwhal/src/helpers/committee.rs index 74d96015ef..43d9c2d950 100644 --- a/node/narwhal/src/helpers/committee.rs +++ b/node/narwhal/src/helpers/committee.rs @@ -119,7 +119,7 @@ impl Committee { } #[cfg(test)] -pub mod tests { +pub mod committee_tests { use crate::helpers::Committee; use anyhow::Result; use indexmap::IndexMap; @@ -131,7 +131,7 @@ pub mod tests { type N = Testnet3; - #[derive(Arbitrary, Debug)] + #[derive(Arbitrary, Debug, Clone)] pub struct CommitteeInput { #[strategy(0u64..)] pub round: u64, diff --git a/node/narwhal/src/helpers/storage.rs b/node/narwhal/src/helpers/storage.rs index 42ee309df9..f31fb32c45 100644 --- a/node/narwhal/src/helpers/storage.rs +++ b/node/narwhal/src/helpers/storage.rs @@ -406,7 +406,7 @@ impl Storage { } #[cfg(test)] -mod tests { +pub mod storage_tests { use super::*; use snarkvm::{ ledger::narwhal::Data, @@ -416,6 +416,7 @@ mod tests { use ::bytes::Bytes; use indexmap::indexset; + use test_strategy::Arbitrary; type CurrentNetwork = snarkvm::prelude::Testnet3; @@ -560,4 +561,15 @@ mod tests { // Ensure the certificate is no longer stored in the round. assert!(storage.get_certificates_for_round(round).is_empty()); } + + #[derive(Arbitrary, Debug, Clone)] + pub struct StorageInput { + pub gc_rounds: u64, + } + + impl StorageInput { + pub fn to_storage(&self) -> Storage { + Storage::new(self.gc_rounds) + } + } } diff --git a/node/narwhal/src/worker.rs b/node/narwhal/src/worker.rs index 0bd3095a35..a6f665973c 100644 --- a/node/narwhal/src/worker.rs +++ b/node/narwhal/src/worker.rs @@ -266,3 +266,18 @@ impl Worker { self.handles.lock().iter().for_each(|handle| handle.abort()); } } + +#[cfg(test)] +mod worker_tests { + use crate::{gateway_tests::GatewayInput, helpers::storage_tests::StorageInput, Gateway, Worker, MAX_WORKERS}; + use snarkvm::prelude::Testnet3; + use test_strategy::Arbitrary; + + type N = Testnet3; + + #[derive(Arbitrary, Debug, Clone)] + pub struct WorkerInput { + pub id: u8, + pub storage: StorageInput, + } +} From bb6c259229d102f278d0b177927e4da0a903b809 Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Thu, 6 Jul 2023 16:27:47 +0300 Subject: [PATCH 7/8] fix: remove accidental println!() --- node/narwhal/src/gateway.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/node/narwhal/src/gateway.rs b/node/narwhal/src/gateway.rs index 63be37a554..6bace4ad67 100644 --- a/node/narwhal/src/gateway.rs +++ b/node/narwhal/src/gateway.rs @@ -898,7 +898,6 @@ pub mod gateway_tests { #[proptest(async = "tokio")] async fn gateway_start(#[filter(|x| x.dev.is_some())] input: GatewayInput) { let Some(dev) = input.dev else { unreachable!() }; - println!("{input:?}"); let mut gateway = input.to_gateway(); let tcp_config = gateway.tcp().config(); assert_eq!(tcp_config.listener_ip, Some(IpAddr::V4(Ipv4Addr::LOCALHOST))); From 9165fabd1d304cfceb495339678181159e920f08 Mon Sep 17 00:00:00 2001 From: Vesa-Ville Date: Thu, 6 Jul 2023 16:28:41 +0300 Subject: [PATCH 8/8] fix: clippy --- node/narwhal/src/gateway.rs | 7 ++----- node/narwhal/src/worker.rs | 5 +---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/node/narwhal/src/gateway.rs b/node/narwhal/src/gateway.rs index 6bace4ad67..308b503c85 100644 --- a/node/narwhal/src/gateway.rs +++ b/node/narwhal/src/gateway.rs @@ -837,10 +837,7 @@ pub mod gateway_tests { pub fn to_gateway(&self) -> Gateway { let committee = self.committee_input.to_committee().unwrap(); let account = self.node_validator.get_account(); - let dev = match self.dev { - None => None, - Some(dev) => Some(dev as u16), - }; + let dev = self.dev.map(|dev| dev as u16); Gateway::new(Arc::new(RwLock::new(committee)), account, dev).unwrap() } @@ -913,7 +910,7 @@ pub mod gateway_tests { assert_eq!(gateway.num_workers(), workers.len() as u8); } Err(err) => { - panic!("Shouldn't fail because of {err}"); + unreachable!("Unexpected {err}"); } } } diff --git a/node/narwhal/src/worker.rs b/node/narwhal/src/worker.rs index a6f665973c..484cde1b62 100644 --- a/node/narwhal/src/worker.rs +++ b/node/narwhal/src/worker.rs @@ -269,12 +269,9 @@ impl Worker { #[cfg(test)] mod worker_tests { - use crate::{gateway_tests::GatewayInput, helpers::storage_tests::StorageInput, Gateway, Worker, MAX_WORKERS}; - use snarkvm::prelude::Testnet3; + use crate::helpers::storage_tests::StorageInput; use test_strategy::Arbitrary; - type N = Testnet3; - #[derive(Arbitrary, Debug, Clone)] pub struct WorkerInput { pub id: u8,