From 7b7e4674ad2ef5524acd3d067be9b340b96bb556 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Thu, 7 Sep 2023 10:18:47 -0400 Subject: [PATCH 1/9] merge gossip engine, remove use of parallelism for message handling (not needed) --- dkg-gadget/src/dkg_modules/mp_ecdsa.rs | 2 - .../gossip_messages/misbehaviour_report.rs | 2 +- dkg-gadget/src/lib.rs | 31 +--- dkg-gadget/src/worker.rs | 139 +++++++----------- dkg-test-orchestrator/src/main.rs | 15 +- 5 files changed, 60 insertions(+), 129 deletions(-) diff --git a/dkg-gadget/src/dkg_modules/mp_ecdsa.rs b/dkg-gadget/src/dkg_modules/mp_ecdsa.rs index 2a7849f87..2a2f622b8 100644 --- a/dkg-gadget/src/dkg_modules/mp_ecdsa.rs +++ b/dkg-gadget/src/dkg_modules/mp_ecdsa.rs @@ -58,7 +58,6 @@ where party_i, session_id, stage, - crate::DKG_KEYGEN_PROTOCOL_NAME, associated_block, KEYGEN_SSID, ) { @@ -166,7 +165,6 @@ where party_i, session_id, stage, - crate::DKG_SIGNING_PROTOCOL_NAME, associated_block_id, ssid, )?; diff --git a/dkg-gadget/src/gossip_messages/misbehaviour_report.rs b/dkg-gadget/src/gossip_messages/misbehaviour_report.rs index d71a4f8ac..b051bd177 100644 --- a/dkg-gadget/src/gossip_messages/misbehaviour_report.rs +++ b/dkg-gadget/src/gossip_messages/misbehaviour_report.rs @@ -173,7 +173,7 @@ where report.session_id, encoded_signed_dkg_message.len() )); - if let Err(e) = dkg_worker.keygen_gossip_engine.gossip(signed_dkg_message) { + if let Err(e) = dkg_worker.gossip_engine.gossip(signed_dkg_message) { dkg_worker.logger.error(format!( "💀 (Round: {:?}) Failed to gossip misbehaviour message: {:?}", report.session_id, e diff --git a/dkg-gadget/src/lib.rs b/dkg-gadget/src/lib.rs index 7f172784b..321513c44 100644 --- a/dkg-gadget/src/lib.rs +++ b/dkg-gadget/src/lib.rs @@ -142,11 +142,6 @@ where dkg_keystore.clone(), ); - let signing_gossip_protocol = NetworkGossipEngineBuilder::new( - DKG_SIGNING_PROTOCOL_NAME.to_string().into(), - dkg_keystore.clone(), - ); - let logger_prometheus = debug_logger.clone(); let metrics = @@ -165,7 +160,7 @@ where let latest_header = Arc::new(RwLock::new(None)); - let (keygen_gossip_handler, keygen_gossip_engine) = keygen_gossip_protocol + let (gossip_handler, gossip_engine) = keygen_gossip_protocol .build( network.clone(), sync_service.clone(), @@ -175,27 +170,13 @@ where ) .expect("Keygen : Failed to build gossip engine"); - let (signing_gossip_handler, signing_gossip_engine) = signing_gossip_protocol - .build( - network.clone(), - sync_service.clone(), - metrics.clone(), - latest_header.clone(), - debug_logger.clone(), - ) - .expect("Signing : Failed to build gossip engine"); - // enable the gossip - keygen_gossip_engine.set_gossip_enabled(true); - signing_gossip_engine.set_gossip_enabled(true); + gossip_engine.set_gossip_enabled(true); // keygen_gossip_engine.set_processing_already_seen_messages_enabled(false); // signing_gossip_engine.set_processing_already_seen_messages_enabled(false); - let keygen_handle = - crate::utils::ExplicitPanicFuture::new(tokio::spawn(keygen_gossip_handler.run())); - let signing_handle = - crate::utils::ExplicitPanicFuture::new(tokio::spawn(signing_gossip_handler.run())); + let gossip_handle = crate::utils::ExplicitPanicFuture::new(tokio::spawn(gossip_handler.run())); // In memory backend, not used for now // let db_backend = Arc::new(db::DKGInMemoryDb::new()); @@ -211,8 +192,7 @@ where client, backend, key_store: dkg_keystore, - keygen_gossip_engine, - signing_gossip_engine, + gossip_engine, db_backend, metrics, local_keystore, @@ -225,8 +205,7 @@ where let worker = worker::DKGWorker::<_, _, _, _>::new(worker_params, debug_logger); worker.run().await; - keygen_handle.abort(); - signing_handle.abort(); + gossip_handle.abort(); } pub mod deadlock_detection { diff --git a/dkg-gadget/src/worker.rs b/dkg-gadget/src/worker.rs index 5c31052a1..63248ce44 100644 --- a/dkg-gadget/src/worker.rs +++ b/dkg-gadget/src/worker.rs @@ -83,8 +83,7 @@ where pub client: Arc, pub backend: Arc, pub key_store: DKGKeystore, - pub keygen_gossip_engine: GE, - pub signing_gossip_engine: GE, + pub gossip_engine: GE, pub db_backend: Arc, pub metrics: Option, pub local_keystore: Option>, @@ -106,8 +105,7 @@ where pub client: Arc, pub backend: Arc, pub key_store: DKGKeystore, - pub keygen_gossip_engine: Arc, - pub signing_gossip_engine: Arc, + pub gossip_engine: Arc, pub db: Arc, pub metrics: Arc>, /// Cached best authorities @@ -169,8 +167,7 @@ where backend: self.backend.clone(), key_store: self.key_store.clone(), db: self.db.clone(), - keygen_gossip_engine: self.keygen_gossip_engine.clone(), - signing_gossip_engine: self.signing_gossip_engine.clone(), + gossip_engine: self.gossip_engine.clone(), metrics: self.metrics.clone(), best_authorities: self.best_authorities.clone(), next_best_authorities: self.next_best_authorities.clone(), @@ -220,8 +217,7 @@ where backend, key_store, db_backend, - keygen_gossip_engine, - signing_gossip_engine, + gossip_engine, metrics, local_keystore, latest_header, @@ -246,8 +242,7 @@ where key_store, db: db_backend, keygen_manager, - keygen_gossip_engine: Arc::new(keygen_gossip_engine), - signing_gossip_engine: Arc::new(signing_gossip_engine), + gossip_engine: Arc::new(gossip_engine), metrics: Arc::new(metrics), best_authorities: Arc::new(RwLock::new(vec![])), next_best_authorities: Arc::new(RwLock::new(vec![])), @@ -305,7 +300,6 @@ where party_i: KeygenPartyId, session_id: SessionId, stage: ProtoStageType, - protocol_name: &str, associated_block: NumberFor, ssid: SSID, ) -> Result< @@ -365,7 +359,7 @@ where client: self.client.clone(), keystore: self.key_store.clone(), db: self.db.clone(), - gossip_engine: self.get_gossip_engine_from_protocol_name(protocol_name), + gossip_engine: self.gossip_engine.clone(), aggregated_public_keys: self.aggregated_public_keys.clone(), best_authorities: best_authorities.clone(), authority_public_key: authority_public_key.clone(), @@ -407,15 +401,6 @@ where } } - /// Returns the gossip engine based on the protocol_name - fn get_gossip_engine_from_protocol_name(&self, protocol_name: &str) -> Arc { - match protocol_name { - crate::DKG_KEYGEN_PROTOCOL_NAME => self.keygen_gossip_engine.clone(), - crate::DKG_SIGNING_PROTOCOL_NAME => self.signing_gossip_engine.clone(), - _ => panic!("Protocol name not found!"), - } - } - /// Fetch the stored local keys if they exist. fn fetch_local_keys( &self, @@ -1050,90 +1035,68 @@ where // We run all these tasks in parallel and wait for any of them to complete. // If any of them completes, we stop all the other tasks since this means a fatal error has // occurred and we need to shut down. - let (first, n, ..) = futures::future::select_all(vec![ - self.spawn_finality_notification_task(), - self.spawn_keygen_messages_stream_task(), - self.spawn_signing_messages_stream_task(), - self.spawn_error_handling_task(), - ]) - .await; + let finality_notification_task = self.finality_notification_task(); + let gossip_engine_stream_task = self.gossip_engine_message_stream_task(); + let error_handling_task = self.spawn_error_handling_task(); + + let res = tokio::select! { + res0 = finality_notification_task => res0, + res1 = gossip_engine_stream_task => res1, + res2 = error_handling_task => res2, + }; + self.logger - .error(format!("DKG Worker finished; the reason that task({n}) ended with: {first:?}")); + .error(format!("DKG Worker finished prematurely. The cause: {res:?}")); } - fn spawn_finality_notification_task(&self) -> tokio::task::JoinHandle<()> { + async fn finality_notification_task(&self) -> Result<(), DKGError> { let mut stream = self.client.finality_notification_stream(); - let self_ = self.clone(); - tokio::spawn(async move { - while let Some(notification) = stream.next().await { - dkg_logging::debug!("Going to handle Finality notification"); - self_.handle_finality_notification(notification).await; - } - self_.logger.error("Finality notification stream ended"); - }) - } + while let Some(notification) = stream.next().await { + dkg_logging::debug!("Going to handle Finality notification"); + self.handle_finality_notification(notification).await; + } - fn spawn_keygen_messages_stream_task(&self) -> tokio::task::JoinHandle<()> { - let keygen_gossip_engine = self.keygen_gossip_engine.clone(); - let mut keygen_stream = - keygen_gossip_engine.get_stream().expect("keygen gossip stream already taken"); - let self_ = self.clone(); - tokio::spawn(async move { - while let Some(msg) = keygen_stream.recv().await { - let msg_hash = crate::debug_logger::raw_message_to_hash(msg.msg.payload.payload()); - self_.logger.debug(format!( - "Going to handle keygen message for session {} | hash: {msg_hash}", - msg.msg.session_id - )); - self_.logger.checkpoint_message_raw(msg.msg.payload.payload(), "CP1-keygen"); - match self_.process_incoming_dkg_message(msg).await { - Ok(_) => {}, - Err(e) => { - self_.logger.error(format!("Error processing keygen message: {e:?}")); - }, - } - } - }) + self.logger.error("Finality notification stream ended"); + + Err(DKGError::CriticalError { reason: "Finality notification stream ended".to_string() }) } - fn spawn_signing_messages_stream_task(&self) -> tokio::task::JoinHandle<()> { - let signing_gossip_engine = self.signing_gossip_engine.clone(); - let mut signing_stream = - signing_gossip_engine.get_stream().expect("signing gossip stream already taken"); - let self_ = self.clone(); - tokio::spawn(async move { - while let Some(msg) = signing_stream.recv().await { - self_.logger.debug(format!( - "Going to handle signing message for session {}", - msg.msg.session_id - )); - self_.logger.checkpoint_message_raw(msg.msg.payload.payload(), "CP1-signing"); - match self_.process_incoming_dkg_message(msg).await { - Ok(_) => {}, - Err(e) => { - self_.logger.error(format!("Error processing signing message: {e:?}")); - }, - } + async fn gossip_engine_message_stream_task(&self) -> Result<(), DKGError> { + let mut stream = + self.gossip_engine.get_stream().expect("keygen gossip stream already taken"); + + while let Some(msg) = stream.recv().await { + let msg_hash = crate::debug_logger::raw_message_to_hash(msg.msg.payload.payload()); + self.logger.debug(format!( + "Going to handle message for session {} | hash: {msg_hash}", + msg.msg.session_id + )); + self.logger.checkpoint_message_raw(msg.msg.payload.payload(), "CP1-incoming"); + match self.process_incoming_dkg_message(msg).await { + Ok(_) => {}, + Err(e) => { + self.logger.error(format!("Error processing keygen message: {e:?}")); + }, } - }) + } + + Err(DKGError::CriticalError { reason: "Gossip engine stream ended".to_string() }) } - fn spawn_error_handling_task(&self) -> tokio::task::JoinHandle<()> { - let self_ = self.clone(); + async fn spawn_error_handling_task(&self) -> Result<(), DKGError> { let mut error_handler_rx = self .error_handler_channel .rx .lock() .take() .expect("Error handler tx already taken"); - let logger = self.logger.clone(); - tokio::spawn(async move { - while let Some(error) = error_handler_rx.recv().await { - logger.debug("Going to handle Error"); - self_.handle_dkg_error(error).await; - } - }) + while let Some(error) = error_handler_rx.recv().await { + self.logger.debug("Going to handle Error"); + self.handle_dkg_error(error).await; + } + + Err(DKGError::CriticalError { reason: "Error handler stream ended".to_string() }) } } diff --git a/dkg-test-orchestrator/src/main.rs b/dkg-test-orchestrator/src/main.rs index eb856f20f..1e7025e4c 100644 --- a/dkg-test-orchestrator/src/main.rs +++ b/dkg-test-orchestrator/src/main.rs @@ -70,8 +70,7 @@ async fn main() -> Result<(), Box> { // the gossip engine and the dummy api share a state between ALL clients in this process // we will use the SAME gossip engine for both keygen and signing - let keygen_gossip_engine = &InMemoryGossipEngine::new(); - let signing_gossip_engine = &InMemoryGossipEngine::new(); + let gossip_engine = &InMemoryGossipEngine::new(); let keygen_t = t as u16; let keygen_n = n_clients as u16; let signing_t = t as u16; @@ -134,20 +133,13 @@ async fn main() -> Result<(), Box> { // output the logs for this specific peer to a file let output = args.tmp_path.join(format!("{peer_id}.log")); let logger = dkg_gadget::debug_logger::DebugLogger::new(peer_id, Some(output))?; - let keygen_gossip_engine = keygen_gossip_engine.clone_for_new_peer( + let gossip_engine = gossip_engine.clone_for_new_peer( api, n_blocks as _, peer_id, public_key.clone(), &logger, ); - let signing_gossip_engine = signing_gossip_engine.clone_for_new_peer( - api, - n_blocks as _, - peer_id, - public_key, - &logger, - ); key_store.set_logger(logger.clone()); @@ -179,8 +171,7 @@ async fn main() -> Result<(), Box> { client, backend, key_store, - keygen_gossip_engine, - signing_gossip_engine, + gossip_engine, db_backend, metrics, local_keystore, From 3d224f8f29dc30fab3db0762018a04a1847b90f4 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Thu, 7 Sep 2023 17:24:54 -0400 Subject: [PATCH 2/9] Add integration tests --- Cargo.lock | 236 ++++++++++++++++++++++++++++-- Cargo.toml | 6 +- Makefile.toml | 15 +- dkg-gadget/Cargo.toml | 2 +- dkg-integration-tests/Cargo.toml | 20 +++ dkg-integration-tests/src/main.rs | 204 ++++++++++++++++++++++++++ dkg-logging/Cargo.toml | 18 ++- dkg-logging/src/lib.rs | 1 + dkg-mock-blockchain/Cargo.toml | 2 +- dkg-test-orchestrator/Cargo.toml | 4 +- standalone/node/Cargo.toml | 2 +- 11 files changed, 481 insertions(+), 29 deletions(-) create mode 100644 dkg-integration-tests/Cargo.toml create mode 100644 dkg-integration-tests/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 046ad7d8e..32dcde7d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,66 @@ dependencies = [ "regex", ] +[[package]] +name = "ac-compose-macros" +version = "0.4.2" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "ac-primitives", + "log", + "maybe-async", +] + +[[package]] +name = "ac-node-api" +version = "0.5.1" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "ac-primitives", + "bitvec 1.0.1", + "derive_more", + "either", + "frame-metadata 15.1.0", + "hex", + "log", + "parity-scale-codec", + "scale-bits 0.4.0", + "scale-decode 0.8.0", + "scale-encode 0.4.0", + "scale-info", + "serde", + "serde_json", + "sp-application-crypto 23.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime-interface 17.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-storage 13.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", +] + +[[package]] +name = "ac-primitives" +version = "0.9.0" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "frame-system", + "impl-serde", + "pallet-assets", + "pallet-balances", + "parity-scale-codec", + "primitive-types", + "scale-info", + "serde", + "serde_json", + "sp-application-crypto 23.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-core-hashing 9.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime-interface 17.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-staking", + "sp-version", + "sp-weights 20.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", +] + [[package]] name = "addr2line" version = "0.19.0" @@ -2455,6 +2515,18 @@ dependencies = [ "webb-proposals 0.6.0", ] +[[package]] +name = "dkg-integration-tests" +version = "0.1.0" +dependencies = [ + "ac-primitives", + "dkg-logging", + "dkg-standalone-runtime", + "substrate-api-client", + "tokio", + "tokio-websockets", +] + [[package]] name = "dkg-logging" version = "0.1.0" @@ -5686,6 +5758,17 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "md-5" version = "0.10.5" @@ -6439,6 +6522,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "pallet-assets" +version = "4.0.0-dev" +source = "git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0#948fbd2fd1233dc26dbb9f9bbc1d2cca2c03945d" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "parity-scale-codec", + "scale-info", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-std 8.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", +] + [[package]] name = "pallet-aura" version = "4.0.0-dev" @@ -9568,6 +9666,17 @@ dependencies = [ "serde", ] +[[package]] +name = "scale-bits" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "036575c29af9b6e4866ffb7fa055dbf623fe7a9cc159b33786de6013a6969d89" +dependencies = [ + "parity-scale-codec", + "scale-info", + "serde", +] + [[package]] name = "scale-decode" version = "0.7.0" @@ -9576,13 +9685,27 @@ checksum = "f0459d00b0dbd2e765009924a78ef36b2ff7ba116292d732f00eb0ed8e465d15" dependencies = [ "parity-scale-codec", "primitive-types", - "scale-bits", - "scale-decode-derive", + "scale-bits 0.3.0", + "scale-decode-derive 0.7.0", "scale-info", "smallvec", "thiserror", ] +[[package]] +name = "scale-decode" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea509715113edab351e1f4d51fba6b186653259049a1155b52e2e994dd2f0e6d" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "scale-bits 0.4.0", + "scale-decode-derive 0.8.0", + "scale-info", + "smallvec", +] + [[package]] name = "scale-decode-derive" version = "0.7.0" @@ -9596,6 +9719,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scale-decode-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c9d7a1341497e9d016722144310de3dc6c933909c0376017c88f65092fff37" +dependencies = [ + "darling 0.14.4", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "scale-encode" version = "0.3.0" @@ -9604,13 +9740,27 @@ checksum = "b0401b7cdae8b8aa33725f3611a051358d5b32887ecaa0fda5953a775b2d4d76" dependencies = [ "parity-scale-codec", "primitive-types", - "scale-bits", - "scale-encode-derive", + "scale-bits 0.3.0", + "scale-encode-derive 0.3.0", "scale-info", "smallvec", "thiserror", ] +[[package]] +name = "scale-encode" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6f51bc8cd927dab2f4567b1a8a8e9d7fd5d0866f2dbc7c84fc97cfa9383a26" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "scale-bits 0.4.0", + "scale-encode-derive 0.4.0", + "scale-info", + "smallvec", +] + [[package]] name = "scale-encode-derive" version = "0.3.0" @@ -9624,6 +9774,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scale-encode-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28be1877787156a2df01be3c029b92bdffa6b6a9748d4996e383fff218c88f3" +dependencies = [ + "darling 0.14.4", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "scale-info" version = "2.9.0" @@ -9661,9 +9824,9 @@ dependencies = [ "either", "frame-metadata 15.1.0", "parity-scale-codec", - "scale-bits", - "scale-decode", - "scale-encode", + "scale-bits 0.3.0", + "scale-decode 0.7.0", + "scale-encode 0.3.0", "scale-info", "serde", "thiserror", @@ -10055,6 +10218,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.8.2" @@ -11503,6 +11672,34 @@ dependencies = [ "webrtc-util", ] +[[package]] +name = "substrate-api-client" +version = "0.14.0" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "ac-compose-macros", + "ac-node-api", + "ac-primitives", + "async-trait", + "derive_more", + "frame-metadata 15.1.0", + "frame-support", + "futures", + "hex", + "jsonrpsee", + "log", + "maybe-async", + "parity-scale-codec", + "serde", + "serde_json", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime-interface 17.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-storage 13.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-version", + "url", +] + [[package]] name = "substrate-bip39" version = "0.4.4" @@ -11687,9 +11884,9 @@ dependencies = [ "jsonrpsee", "parity-scale-codec", "primitive-types", - "scale-bits", - "scale-decode", - "scale-encode", + "scale-bits 0.3.0", + "scale-decode 0.7.0", + "scale-encode 0.3.0", "scale-info", "scale-value", "serde", @@ -12157,6 +12354,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "tokio-websockets" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd056b4095b3ec42b9a13d6985f3b1f7d2ee1a216440a9b25e122c15edfdce3" +dependencies = [ + "base64 0.21.2", + "bytes", + "fastrand 2.0.0", + "futures-util", + "http", + "httparse", + "sha1_smol", + "tokio", + "tokio-util", +] + [[package]] name = "toml" version = "0.5.11" @@ -12552,7 +12766,7 @@ checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", "digest 0.10.7", - "rand 0.8.5", + "rand 0.4.6", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index cc8d8128f..af93812fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ 'dkg-test-orchestrator', 'relayer-gadget', 'relayer-gadget/cli', + 'dkg-integration-tests' ] resolver = "2" @@ -32,7 +33,7 @@ pallet-bridge-registry = { path = "pallets/bridge-registry", default-features = dkg-gadget = { path = "dkg-gadget", default-features = false } dkg-primitives = { path = "dkg-primitives", default-features = false } dkg-standalone-runtime = { version = "3.0.0", path = "standalone/runtime" } -dkg-logging = { path = "dkg-logging" } +dkg-logging = { path = "dkg-logging", default-features = false } dkg-rococo-runtime = { default-features = false, path = "parachain/runtime/rococo" } dkg-mock-blockchain = { path = "dkg-mock-blockchain", default-features = false } dkg-test-orchestrator = { path = "dkg-test-orchestrator", default-features = false } @@ -75,6 +76,9 @@ tracing-subscriber = "0.3.5" sync_wrapper = "0.1.2" async-stream = "0.3.5" lazy_static = "1.4.0" +substrate-api-client = { git = "https://github.com/webb-tools/substrate-api-client" } +ac-primitives = { git = "https://github.com/webb-tools/substrate-api-client" } +tokio-websockets = "0.3.3" scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } diff --git a/Makefile.toml b/Makefile.toml index eab7b3a28..5ba8afada 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -16,32 +16,37 @@ command = "cargo" args = ["build", "-rp", "dkg-standalone-node", "--features=integration-tests,testing"] [tasks.alice] -dependencies = ["build-test"] +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } description = "Run Alice node (the bootnode)" command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--alice", "--output-path=./tmp/alice/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30333", "--rpc-port", "9944", "--node-key", "0000000000000000000000000000000000000000000000000000000000000001"] [tasks.bob] -dependencies = ["build-test"] description = "Run Bob node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--bob", "--output-path=./tmp/bob/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30305", "--rpc-port", "9945", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.charlie] -dependencies = ["build-test"] description = "Run Charlie node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--charlie", "--output-path=./tmp/charlie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30306", "--rpc-port", "9946", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.dave] -dependencies = ["build-test"] description = "Run Dave node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--dave", "--output-path=./tmp/dave/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30307", "--rpc-port", "9947", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.eve] -dependencies = ["build-test"] description = "Run Eve node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--eve", "--output-path=./tmp/eve/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30308", "--rpc-port", "9948", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] +[tasks.ferdie] +description = "Run Ferdie node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +command = "./target/release/dkg-standalone-node" +args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--ferdie", "--output-path=./tmp/ferdie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30309", "--rpc-port", "9949", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] diff --git a/dkg-gadget/Cargo.toml b/dkg-gadget/Cargo.toml index bd9acb4c4..70e44ca5e 100644 --- a/dkg-gadget/Cargo.toml +++ b/dkg-gadget/Cargo.toml @@ -64,7 +64,7 @@ webb-proposals = { workspace = true } # Local dependencies dkg-runtime-primitives = { workspace = true } dkg-primitives = { workspace = true } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } dkg-mock-blockchain = { workspace = true } tracing = { workspace = true, optional = true } diff --git a/dkg-integration-tests/Cargo.toml b/dkg-integration-tests/Cargo.toml new file mode 100644 index 000000000..1aefc73b6 --- /dev/null +++ b/dkg-integration-tests/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "dkg-integration-tests" +version = "0.1.0" +description.workspace = true +authors.workspace = true +license.workspace = true +publish.workspace = true +homepage.workspace = true +repository.workspace = true +edition.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { workspace = true, features = ["full"] } +dkg-logging = { workspace = true, default-features = false } +dkg-standalone-runtime = { workspace = true } +substrate-api-client = { workspace = true, features = ["std"] } +ac-primitives = { workspace = true } +tokio-websockets = { workspace = true } diff --git a/dkg-integration-tests/src/main.rs b/dkg-integration-tests/src/main.rs new file mode 100644 index 000000000..d092a4c25 --- /dev/null +++ b/dkg-integration-tests/src/main.rs @@ -0,0 +1,204 @@ +use ac_primitives::{AssetRuntimeConfig, H256}; +use std::{collections::HashSet, error::Error, time::Duration}; +use substrate_api_client::{rpc::JsonrpseeClient, Api, SubscribeEvents}; +use tokio::{net::TcpStream, process::Child}; +use tokio_websockets::{ClientBuilder, MaybeTlsStream, WebsocketStream}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let mut shutdown = Shutdown { members: vec![] }; + let res = main_inner(&mut shutdown).await; + + for member in shutdown.members.iter_mut() { + if let Some(pid) = member.id() { + dkg_logging::info!(target: "dkg", "Killing member {pid}"); + let _exit = tokio::process::Command::new("kill") + .arg("-9") + .arg(pid.to_string()) + .stderr(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::inherit()) + .spawn() + .expect("Should spawn") + .wait() + .await + .unwrap(); + } + + member.kill().await.expect("Should kill member"); + } + + res +} + +async fn main_inner(shutdown: &mut Shutdown) -> Result<(), Box> { + dkg_logging::setup_log(); + dkg_logging::info!(target: "dkg", "Will test getting to session 3"); + + let alice = generate_process("alice", false)?; + dkg_logging::info!(target: "dkg", "Spawned Bootnode, now awaiting for initialization"); + // With alice spawned, we now must wait for her to be ready to accept connections + wait_for_bootnode_to_init().await?; + + // with the bootnode ready, load the rest of the nodes + let bob = generate_process("bob", false)?; + let charlie = generate_process("charlie", false)?; + let dave = generate_process("dave", false)?; + let eve = generate_process("eve", false)?; + + shutdown.members = vec![alice, bob, charlie, dave, eve]; + + // Setup API + let client = JsonrpseeClient::with_default_url().unwrap(); + let mut api = Api::::new(client).unwrap(); + + // Wait for the bootnode to get to session 5 + wait_for_session_3(&mut api).await; + Ok(()) +} + +async fn wait_for_bootnode_to_init() -> Result<(), Box> { + loop { + match tokio::time::timeout( + Duration::from_millis(10000), + generate_websocket_connection_to_bootnode(), + ) + .await + { + Ok(Ok(_ws)) => { + dkg_logging::info!(target: "dkg", "Obtained websocket connection to bootnode"); + return Ok(()) + }, + + Ok(Err(_err)) => {}, + + Err(_err) => { + dkg_logging::info!(target: "dkg", "Still waiting for bootnode to init ..."); + }, + } + } +} + +async fn generate_websocket_connection_to_bootnode( +) -> Result>, Box> { + Ok(ClientBuilder::new() + .uri("ws://127.0.0.1:9944")? // connect to the bootnode's WS port to listen to events + .connect() + .await?) +} + +async fn wait_for_session_3(api: &mut Api) { + let mut has_seen = HashSet::new(); + let mut listener = DKGEventListener::new(); + loop { + let mut subscription = api.subscribe_events().unwrap(); + let records = tokio::task::spawn_blocking(move || { + let events = subscription.next_events::(); + if let Some(events) = events { + events.unwrap_or_default() + } else { + vec![] + } + }) + .await + .expect("JoinError"); + + for record in records { + let record_str = format!("{record:?}"); + if has_seen.insert(record_str.clone()) { + dkg_logging::info!(target: "dkg", "decoded: {:?}", record.event); + let session = listener.on_event_received(&record_str); + dkg_logging::info!(target: "dkg", "State: {listener:?}"); + if session == 3 { + return + } + } + } + } +} + +fn generate_process(node: &str, inherit: bool) -> Result> { + let mut command = tokio::process::Command::new("cargo"); + + if inherit { + command.stdout(std::process::Stdio::inherit()); + command.stderr(std::process::Stdio::inherit()); + } else { + command.stdout(std::process::Stdio::piped()); + command.stderr(std::process::Stdio::piped()); + } + + Ok(command + .arg("make") + .arg(node) + .env("RUST_LOG", "dkg=info") + .kill_on_drop(true) + .spawn()?) +} + +struct Shutdown { + members: Vec, +} + +const EVENT_DKG_PUBLIC_KEY_SUBMITTED: &str = "Event::PublicKeySubmitted"; +const EVENT_NEXT_DKG_PUBLIC_KEY_SUBMITTED: &str = "Event::NextPublicKeySubmitted"; +const EVENT_DKG_PROPOSAL_ADDED: &str = "Event::ProposalAdded"; +const EVENT_DKG_PROPOSAL_BATCH_SIGNED: &str = "Event::ProposalBatchSigned"; +const EVENT_NEXT_DKG_PUBLIC_KEY_SIGNATURE_SUBMITTED: &str = + "Event::NextPublicKeySignatureSubmitted"; +const EVENT_NEW_SESSION: &str = "Event::NewSession"; +const EVENT_PUBLIC_KEY_CHANGED: &str = "Event::PublicKeyChanged"; +const EVENT_PUBLIC_KEY_SIGNATURE_CHANGED: &str = "Event::PublicKeySignatureChanged"; + +#[derive(Debug)] +struct DKGEventListener { + current_session: u64, + current_session_events_required: HashSet<&'static str>, +} + +impl DKGEventListener { + fn new() -> Self { + Self { + current_session: 0, + current_session_events_required: [ + EVENT_DKG_PUBLIC_KEY_SUBMITTED, + EVENT_NEXT_DKG_PUBLIC_KEY_SUBMITTED, + EVENT_DKG_PROPOSAL_ADDED, + EVENT_DKG_PROPOSAL_BATCH_SIGNED, + EVENT_NEXT_DKG_PUBLIC_KEY_SIGNATURE_SUBMITTED, + EVENT_NEW_SESSION, + EVENT_PUBLIC_KEY_CHANGED, + EVENT_PUBLIC_KEY_SIGNATURE_CHANGED, + ] + .into_iter() + .collect(), + } + } + // Returns the current session + fn on_event_received(&mut self, event: &str) -> u64 { + let maybe_matched_event = self + .current_session_events_required + .iter() + .find(|exact| event.contains(**exact)); + if let Some(matched_event) = maybe_matched_event { + self.current_session_events_required.remove(*matched_event); + if self.current_session_events_required.is_empty() { + self.current_session += 1; + self.current_session_events_required = [ + // EVENT_DKG_PUBLIC_KEY_SUBMITTED, omit this since this is only needed in the + // zeroth session (genesis) + EVENT_NEXT_DKG_PUBLIC_KEY_SUBMITTED, + EVENT_DKG_PROPOSAL_ADDED, + EVENT_DKG_PROPOSAL_BATCH_SIGNED, + EVENT_NEXT_DKG_PUBLIC_KEY_SIGNATURE_SUBMITTED, + EVENT_NEW_SESSION, + EVENT_PUBLIC_KEY_CHANGED, + EVENT_PUBLIC_KEY_SIGNATURE_CHANGED, + ] + .into_iter() + .collect(); + } + } + + self.current_session + } +} diff --git a/dkg-logging/Cargo.toml b/dkg-logging/Cargo.toml index b34394070..d6a86ab83 100644 --- a/dkg-logging/Cargo.toml +++ b/dkg-logging/Cargo.toml @@ -9,14 +9,18 @@ edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = ["full"] +full = ["parking_lot", "serde", "sp-core", "lazy_static", "tokio", "serde_json", "hex"] + [dependencies] tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } tracing-filter = "0.1.0-alpha.2" -parking_lot = { workspace = true } -serde = { workspace = true } -sp-core = { workspace = true } -lazy_static = { workspace = true } -tokio = { workspace = true } -serde_json = { workspace = true } -hex = { workspace = true } +parking_lot = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +sp-core = { workspace = true, optional = true } +lazy_static = { workspace = true, optional = true } +tokio = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +hex = { workspace = true, optional = true } diff --git a/dkg-logging/src/lib.rs b/dkg-logging/src/lib.rs index f7213b876..18905dcf8 100644 --- a/dkg-logging/src/lib.rs +++ b/dkg-logging/src/lib.rs @@ -5,6 +5,7 @@ use tracing_subscriber::{ EnvFilter, }; +#[cfg(feature = "full")] pub mod debug_logger; pub fn setup_log() { diff --git a/dkg-mock-blockchain/Cargo.toml b/dkg-mock-blockchain/Cargo.toml index 6c330678c..5ce87a04d 100644 --- a/dkg-mock-blockchain/Cargo.toml +++ b/dkg-mock-blockchain/Cargo.toml @@ -23,4 +23,4 @@ sc-network = { workspace = true } sc-utils = { workspace = true } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } dkg-runtime-primitives = { workspace = true, features = ["testing", "std"] } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } diff --git a/dkg-test-orchestrator/Cargo.toml b/dkg-test-orchestrator/Cargo.toml index abfaf7066..aeca6fc47 100644 --- a/dkg-test-orchestrator/Cargo.toml +++ b/dkg-test-orchestrator/Cargo.toml @@ -12,7 +12,7 @@ debug-tracing = ["dkg-gadget/debug-tracing"] tokio = { workspace = true, features = ["rt-multi-thread", "fs", "macros", "process"] } toml = { workspace = true, features = ["parse"] } dkg-mock-blockchain = { workspace = true } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } dkg-gadget = { workspace = true, features = ["testing"] } structopt = { workspace = true } log = { workspace = true } @@ -34,4 +34,4 @@ sp-core = { workspace = true } sp-keystore = { workspace = true } uuid = { workspace = true, features = ["v4"] } frame-support = { workspace = true } -rand = "0.8.5" \ No newline at end of file +rand = "0.8.5" diff --git a/standalone/node/Cargo.toml b/standalone/node/Cargo.toml index 3214cdef9..d88c23267 100644 --- a/standalone/node/Cargo.toml +++ b/standalone/node/Cargo.toml @@ -65,7 +65,7 @@ dkg-gadget = { workspace = true } dkg-runtime-primitives = { workspace = true } dkg-primitives = { workspace = true } dkg-standalone-runtime = { workspace = true } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } webb-relayer-gadget = { workspace = true } webb-relayer-gadget-cli = { workspace = true } tokio = { workspace = true, features = ["rt"] } From 42d4cb66f82f058c0d766eae59a206717dfcabd5 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 8 Sep 2023 05:52:48 -0400 Subject: [PATCH 3/9] Add workflow file --- .github/workflows/integration_tests.yml | 67 +++++++++++++++++++++++++ Makefile.toml | 5 ++ dkg-integration-tests/src/main.rs | 9 ++-- 3 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/integration_tests.yml diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 000000000..fbdd98ff4 --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,67 @@ +name: integration tests + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +env: + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git + RUST_LOG: "dkg=trace" + +jobs: + # dkg-substrate integration tests + local_chain: + name: Local Chain Tests + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Restore Cache + if: always() + uses: actions/cache/restore@v3 + with: + path: | + ~/.cargo/registry + target/release + target/debug + key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-harness-stress-tests + + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly + + - name: Install cargo make + run: cargo install --force cargo-make + + - name: Build Release Binary + run: cargo make build-test + + - name: Get To Session 3 + run: cargo make integration-tests + + - name: Save Cache + if: ${{ !cancelled() }} + uses: actions/cache/save@v3 + with: + path: | + ~/.cargo/registry + target/release + target/debug + key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-harness-stress-tests diff --git a/Makefile.toml b/Makefile.toml index 5ba8afada..0b3711cd2 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -15,6 +15,11 @@ args = ["clean"] command = "cargo" args = ["build", "-rp", "dkg-standalone-node", "--features=integration-tests,testing"] +[tasks.integration-tests] +command = "cargo" +dependencies = ["clean-tmp"] +args = ["run", "--bin", "dkg-integration-tests"] + [tasks.alice] condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } description = "Run Alice node (the bootnode)" diff --git a/dkg-integration-tests/src/main.rs b/dkg-integration-tests/src/main.rs index d092a4c25..ac3dce0df 100644 --- a/dkg-integration-tests/src/main.rs +++ b/dkg-integration-tests/src/main.rs @@ -21,7 +21,7 @@ async fn main() -> Result<(), Box> { .expect("Should spawn") .wait() .await - .unwrap(); + .expect("Should exec"); } member.kill().await.expect("Should kill member"); @@ -48,8 +48,9 @@ async fn main_inner(shutdown: &mut Shutdown) -> Result<(), Box> { shutdown.members = vec![alice, bob, charlie, dave, eve]; // Setup API - let client = JsonrpseeClient::with_default_url().unwrap(); - let mut api = Api::::new(client).unwrap(); + let client = JsonrpseeClient::with_default_url().expect("Should create client"); + let mut api = + Api::::new(client).expect("Should create API"); // Wait for the bootnode to get to session 5 wait_for_session_3(&mut api).await; @@ -90,7 +91,7 @@ async fn wait_for_session_3(api: &mut Api) let mut has_seen = HashSet::new(); let mut listener = DKGEventListener::new(); loop { - let mut subscription = api.subscribe_events().unwrap(); + let mut subscription = api.subscribe_events().expect("Should subscribe to events"); let records = tokio::task::spawn_blocking(move || { let events = subscription.next_events::(); if let Some(events) = events { From 7cb9f3da4a618c55d7c559b10b374484d41af195 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 8 Sep 2023 05:54:12 -0400 Subject: [PATCH 4/9] fix cache --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index fbdd98ff4..e95b575b4 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -40,7 +40,7 @@ jobs: ~/.cargo/registry target/release target/debug - key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-harness-stress-tests + key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-local-chain-tests - name: Install toolchain uses: dtolnay/rust-toolchain@stable @@ -64,4 +64,4 @@ jobs: ~/.cargo/registry target/release target/debug - key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-harness-stress-tests + key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-local-chain-tests From cd6d08b4fe6f43dcaa7e93b979ac0ea5824a448e Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 8 Sep 2023 06:07:53 -0400 Subject: [PATCH 5/9] Add documentation, switch from release to debug build --- Makefile.toml | 26 +++++++++++++------------- README.md | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 0b3711cd2..20fa89b01 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -13,7 +13,7 @@ args = ["clean"] [tasks.build-test] command = "cargo" -args = ["build", "-rp", "dkg-standalone-node", "--features=integration-tests,testing"] +args = ["build", "-p", "dkg-standalone-node", "--features=integration-tests,testing"] [tasks.integration-tests] command = "cargo" @@ -21,37 +21,37 @@ dependencies = ["clean-tmp"] args = ["run", "--bin", "dkg-integration-tests"] [tasks.alice] -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } description = "Run Alice node (the bootnode)" -command = "./target/release/dkg-standalone-node" +command = "./target/debug/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--alice", "--output-path=./tmp/alice/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30333", "--rpc-port", "9944", "--node-key", "0000000000000000000000000000000000000000000000000000000000000001"] [tasks.bob] description = "Run Bob node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } -command = "./target/release/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } +command = "./target/debug/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--bob", "--output-path=./tmp/bob/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30305", "--rpc-port", "9945", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.charlie] description = "Run Charlie node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } -command = "./target/release/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } +command = "./target/debug/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--charlie", "--output-path=./tmp/charlie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30306", "--rpc-port", "9946", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.dave] description = "Run Dave node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } -command = "./target/release/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } +command = "./target/debug/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--dave", "--output-path=./tmp/dave/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30307", "--rpc-port", "9947", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.eve] description = "Run Eve node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } -command = "./target/release/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } +command = "./target/debug/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--eve", "--output-path=./tmp/eve/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30308", "--rpc-port", "9948", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.ferdie] description = "Run Ferdie node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } -command = "./target/release/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } +command = "./target/debug/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--ferdie", "--output-path=./tmp/ferdie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30309", "--rpc-port", "9949", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] diff --git a/README.md b/README.md index 8f132a8d1..01e52a674 100644 --- a/README.md +++ b/README.md @@ -92,13 +92,15 @@ This should start the local testnet, you can view the logs in `/tmp` directory f The following instructions outline how to run dkg-substrate's base test suite and E2E test suite. -### To run base tests +### Unit Tests ``` cargo test ``` -### To run the test orchestrator E2E tests (recommended) +### Stress Tests + +When debugging the internal mechanics of the DKG, it is useful to run the stress tests. These tests are designed to run the DKG with a small number of authorities and a small number of participants. This allows you to quickly debug the DKG without having to setup a local chain. ``` # Build the dkg-standalone node @@ -108,6 +110,39 @@ cargo build --release -p dkg-standalone-node --features=integration-tests,testin cargo run --package dkg-test-orchestrator --release --features=testing -- --config /path/to/orchestrator_config.toml ``` +### Local Chain Tests (Automatic) + +To test the DKG against a local testnet, run: + +``` +cargo make integration-tests +``` + +### Local Chain Tests (Manual) + +If manual control is needed over the nodes (e.g., for debugging and/or shutting down nodes to test fault-tolerance), first build the binary: + +``` +cargo make build-test +``` + +Then, start the bootnode (Alice): + +``` +cargo make alice +``` + +Once Alice is running, start the other nodes each in separate terminals/processes: + +``` +cargo make bob +cargo make charlie +cargo make dave +cargo make eve +``` + + + ### Setting up debugging logs If you would like to run the dkg with verbose logs you may add the following arguments during initial setup. You may change the target to include `debug | error | info| trace | warn`. You may also want to review [Substrate runtime debugging](https://docs.substrate.io/v3/runtime/debugging/). From 946841c91075b7eac0bfbd8b8fe87511394869bf Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 8 Sep 2023 06:11:47 -0400 Subject: [PATCH 6/9] clean up --- .github/workflows/harness_stress_tests.yml | 7 +++---- .github/workflows/integration_tests.yml | 4 ++-- README.md | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/harness_stress_tests.yml b/.github/workflows/harness_stress_tests.yml index c436d3d8f..844f3b4ad 100644 --- a/.github/workflows/harness_stress_tests.yml +++ b/.github/workflows/harness_stress_tests.yml @@ -1,4 +1,4 @@ -name: harness stress tests +name: Stress Tests on: push: @@ -11,9 +11,8 @@ env: RUST_LOG: "dkg=trace" jobs: - # dkg-substrate integration tests harness: - name: harness stress tests + name: Harness runs-on: ubuntu-latest steps: - name: Cancel Previous Runs @@ -46,7 +45,7 @@ jobs: uses: dtolnay/rust-toolchain@stable with: toolchain: nightly - + - name: Install Protobuf run: sudo apt-get install protobuf-compiler diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index e95b575b4..1dbc8b011 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -50,10 +50,10 @@ jobs: - name: Install cargo make run: cargo install --force cargo-make - - name: Build Release Binary + - name: Build Binary run: cargo make build-test - - name: Get To Session 3 + - name: Run Tests run: cargo make integration-tests - name: Save Cache diff --git a/README.md b/README.md index 01e52a674..e1eef43d4 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ cargo test ### Stress Tests -When debugging the internal mechanics of the DKG, it is useful to run the stress tests. These tests are designed to run the DKG with a small number of authorities and a small number of participants. This allows you to quickly debug the DKG without having to setup a local chain. +When debugging the internal mechanics of the DKG, it is useful to run the stress tests. These tests are designed to run the DKG with a custom number of authorities. This allows you to quickly debug the DKG without having to setup a local chain. ``` # Build the dkg-standalone node From f8b061a4644e248588e222fad7c0ed831f6f5a7e Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 8 Sep 2023 07:50:47 -0400 Subject: [PATCH 7/9] add protobuf --- .github/workflows/integration_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 1dbc8b011..bddb04381 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -47,6 +47,9 @@ jobs: with: toolchain: nightly + - name: Install Protobuf + run: sudo apt-get install protobuf-compiler + - name: Install cargo make run: cargo install --force cargo-make From 815fdfc45f5c26cc35eccbd8cea3fc349feaa81d Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 8 Sep 2023 08:35:25 -0400 Subject: [PATCH 8/9] Switch to release build --- Makefile.toml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 20fa89b01..f8e1523d7 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -13,45 +13,45 @@ args = ["clean"] [tasks.build-test] command = "cargo" -args = ["build", "-p", "dkg-standalone-node", "--features=integration-tests,testing"] +args = ["build", "-rp", "dkg-standalone-node", "--features=integration-tests,testing"] [tasks.integration-tests] command = "cargo" -dependencies = ["clean-tmp"] +dependencies = ["clean-tmp", "build-test"] args = ["run", "--bin", "dkg-integration-tests"] [tasks.alice] -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } description = "Run Alice node (the bootnode)" -command = "./target/debug/dkg-standalone-node" +command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--alice", "--output-path=./tmp/alice/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30333", "--rpc-port", "9944", "--node-key", "0000000000000000000000000000000000000000000000000000000000000001"] [tasks.bob] description = "Run Bob node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } -command = "./target/debug/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--bob", "--output-path=./tmp/bob/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30305", "--rpc-port", "9945", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.charlie] description = "Run Charlie node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } -command = "./target/debug/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--charlie", "--output-path=./tmp/charlie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30306", "--rpc-port", "9946", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.dave] description = "Run Dave node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } -command = "./target/debug/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--dave", "--output-path=./tmp/dave/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30307", "--rpc-port", "9947", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.eve] description = "Run Eve node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } -command = "./target/debug/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--eve", "--output-path=./tmp/eve/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30308", "--rpc-port", "9948", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.ferdie] description = "Run Ferdie node" -condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/debug/dkg-standalone-node"] } -command = "./target/debug/dkg-standalone-node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--ferdie", "--output-path=./tmp/ferdie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30309", "--rpc-port", "9949", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] From c90e7d4bbb1f1048a5335782a193073a0b067449 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 8 Sep 2023 12:52:45 -0400 Subject: [PATCH 9/9] for integration test, get to session 2 --- dkg-integration-tests/src/main.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/dkg-integration-tests/src/main.rs b/dkg-integration-tests/src/main.rs index ac3dce0df..003759811 100644 --- a/dkg-integration-tests/src/main.rs +++ b/dkg-integration-tests/src/main.rs @@ -31,8 +31,10 @@ async fn main() -> Result<(), Box> { } async fn main_inner(shutdown: &mut Shutdown) -> Result<(), Box> { + const SESSION_N: u64 = 2; dkg_logging::setup_log(); - dkg_logging::info!(target: "dkg", "Will test getting to session 3"); + + dkg_logging::info!(target: "dkg", "Will test getting to session {SESSION_N}"); let alice = generate_process("alice", false)?; dkg_logging::info!(target: "dkg", "Spawned Bootnode, now awaiting for initialization"); @@ -52,8 +54,8 @@ async fn main_inner(shutdown: &mut Shutdown) -> Result<(), Box> { let mut api = Api::::new(client).expect("Should create API"); - // Wait for the bootnode to get to session 5 - wait_for_session_3(&mut api).await; + // Wait for the bootnode to get to SESSION_N + wait_for_session_n(&mut api, SESSION_N).await; Ok(()) } @@ -87,7 +89,10 @@ async fn generate_websocket_connection_to_bootnode( .await?) } -async fn wait_for_session_3(api: &mut Api) { +async fn wait_for_session_n( + api: &mut Api, + session_needed: u64, +) { let mut has_seen = HashSet::new(); let mut listener = DKGEventListener::new(); loop { @@ -109,7 +114,7 @@ async fn wait_for_session_3(api: &mut Api) dkg_logging::info!(target: "dkg", "decoded: {:?}", record.event); let session = listener.on_event_received(&record_str); dkg_logging::info!(target: "dkg", "State: {listener:?}"); - if session == 3 { + if session == session_needed { return } }