diff --git a/.github/workflows/harness_stress_tests.yml b/.github/workflows/harness_stress_tests.yml index c436d3d8f..844f3b4ad 100644 --- a/.github/workflows/harness_stress_tests.yml +++ b/.github/workflows/harness_stress_tests.yml @@ -1,4 +1,4 @@ -name: harness stress tests +name: Stress Tests on: push: @@ -11,9 +11,8 @@ env: RUST_LOG: "dkg=trace" jobs: - # dkg-substrate integration tests harness: - name: harness stress tests + name: Harness runs-on: ubuntu-latest steps: - name: Cancel Previous Runs @@ -46,7 +45,7 @@ jobs: uses: dtolnay/rust-toolchain@stable with: toolchain: nightly - + - name: Install Protobuf run: sudo apt-get install protobuf-compiler diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 000000000..bddb04381 --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,70 @@ +name: integration tests + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +env: + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git + RUST_LOG: "dkg=trace" + +jobs: + # dkg-substrate integration tests + local_chain: + name: Local Chain Tests + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Restore Cache + if: always() + uses: actions/cache/restore@v3 + with: + path: | + ~/.cargo/registry + target/release + target/debug + key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-local-chain-tests + + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly + + - name: Install Protobuf + run: sudo apt-get install protobuf-compiler + + - name: Install cargo make + run: cargo install --force cargo-make + + - name: Build Binary + run: cargo make build-test + + - name: Run Tests + run: cargo make integration-tests + + - name: Save Cache + if: ${{ !cancelled() }} + uses: actions/cache/save@v3 + with: + path: | + ~/.cargo/registry + target/release + target/debug + key: ${{ runner.os }}-cargo-index-${{ github.ref_name }}-local-chain-tests diff --git a/Cargo.lock b/Cargo.lock index ce0a868b0..2b38a17a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,66 @@ dependencies = [ "regex", ] +[[package]] +name = "ac-compose-macros" +version = "0.4.2" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "ac-primitives", + "log", + "maybe-async", +] + +[[package]] +name = "ac-node-api" +version = "0.5.1" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "ac-primitives", + "bitvec 1.0.1", + "derive_more", + "either", + "frame-metadata 15.1.0", + "hex", + "log", + "parity-scale-codec", + "scale-bits 0.4.0", + "scale-decode 0.8.0", + "scale-encode 0.4.0", + "scale-info", + "serde", + "serde_json", + "sp-application-crypto 23.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime-interface 17.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-storage 13.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", +] + +[[package]] +name = "ac-primitives" +version = "0.9.0" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "frame-system", + "impl-serde", + "pallet-assets", + "pallet-balances", + "parity-scale-codec", + "primitive-types", + "scale-info", + "serde", + "serde_json", + "sp-application-crypto 23.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-core-hashing 9.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime-interface 17.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-staking", + "sp-version", + "sp-weights 20.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", +] + [[package]] name = "addr2line" version = "0.19.0" @@ -2455,6 +2515,18 @@ dependencies = [ "webb-proposals", ] +[[package]] +name = "dkg-integration-tests" +version = "0.1.0" +dependencies = [ + "ac-primitives", + "dkg-logging", + "dkg-standalone-runtime", + "substrate-api-client", + "tokio", + "tokio-websockets", +] + [[package]] name = "dkg-logging" version = "0.1.0" @@ -5686,6 +5758,17 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "md-5" version = "0.10.5" @@ -6439,6 +6522,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "pallet-assets" +version = "4.0.0-dev" +source = "git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0#948fbd2fd1233dc26dbb9f9bbc1d2cca2c03945d" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "parity-scale-codec", + "scale-info", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-std 8.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", +] + [[package]] name = "pallet-aura" version = "4.0.0-dev" @@ -9568,6 +9666,17 @@ dependencies = [ "serde", ] +[[package]] +name = "scale-bits" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "036575c29af9b6e4866ffb7fa055dbf623fe7a9cc159b33786de6013a6969d89" +dependencies = [ + "parity-scale-codec", + "scale-info", + "serde", +] + [[package]] name = "scale-decode" version = "0.7.0" @@ -9576,13 +9685,27 @@ checksum = "f0459d00b0dbd2e765009924a78ef36b2ff7ba116292d732f00eb0ed8e465d15" dependencies = [ "parity-scale-codec", "primitive-types", - "scale-bits", - "scale-decode-derive", + "scale-bits 0.3.0", + "scale-decode-derive 0.7.0", "scale-info", "smallvec", "thiserror", ] +[[package]] +name = "scale-decode" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea509715113edab351e1f4d51fba6b186653259049a1155b52e2e994dd2f0e6d" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "scale-bits 0.4.0", + "scale-decode-derive 0.8.0", + "scale-info", + "smallvec", +] + [[package]] name = "scale-decode-derive" version = "0.7.0" @@ -9596,6 +9719,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scale-decode-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c9d7a1341497e9d016722144310de3dc6c933909c0376017c88f65092fff37" +dependencies = [ + "darling 0.14.4", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "scale-encode" version = "0.3.0" @@ -9604,13 +9740,27 @@ checksum = "b0401b7cdae8b8aa33725f3611a051358d5b32887ecaa0fda5953a775b2d4d76" dependencies = [ "parity-scale-codec", "primitive-types", - "scale-bits", - "scale-encode-derive", + "scale-bits 0.3.0", + "scale-encode-derive 0.3.0", "scale-info", "smallvec", "thiserror", ] +[[package]] +name = "scale-encode" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6f51bc8cd927dab2f4567b1a8a8e9d7fd5d0866f2dbc7c84fc97cfa9383a26" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "scale-bits 0.4.0", + "scale-encode-derive 0.4.0", + "scale-info", + "smallvec", +] + [[package]] name = "scale-encode-derive" version = "0.3.0" @@ -9624,6 +9774,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scale-encode-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28be1877787156a2df01be3c029b92bdffa6b6a9748d4996e383fff218c88f3" +dependencies = [ + "darling 0.14.4", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "scale-info" version = "2.9.0" @@ -9661,9 +9824,9 @@ dependencies = [ "either", "frame-metadata 15.1.0", "parity-scale-codec", - "scale-bits", - "scale-decode", - "scale-encode", + "scale-bits 0.3.0", + "scale-decode 0.7.0", + "scale-encode 0.3.0", "scale-info", "serde", "thiserror", @@ -10020,6 +10183,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.8.2" @@ -11468,6 +11637,34 @@ dependencies = [ "webrtc-util", ] +[[package]] +name = "substrate-api-client" +version = "0.14.0" +source = "git+https://github.com/webb-tools/substrate-api-client#6653a8d12d9ddfa2cd0301ab6c2aa50262da73e1" +dependencies = [ + "ac-compose-macros", + "ac-node-api", + "ac-primitives", + "async-trait", + "derive_more", + "frame-metadata 15.1.0", + "frame-support", + "futures", + "hex", + "jsonrpsee", + "log", + "maybe-async", + "parity-scale-codec", + "serde", + "serde_json", + "sp-core 21.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime 24.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-runtime-interface 17.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-storage 13.0.0 (git+https://github.com/paritytech/substrate?branch=polkadot-v1.0.0)", + "sp-version", + "url", +] + [[package]] name = "substrate-bip39" version = "0.4.4" @@ -11652,9 +11849,9 @@ dependencies = [ "jsonrpsee", "parity-scale-codec", "primitive-types", - "scale-bits", - "scale-decode", - "scale-encode", + "scale-bits 0.3.0", + "scale-decode 0.7.0", + "scale-encode 0.3.0", "scale-info", "scale-value", "serde", @@ -12122,6 +12319,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "tokio-websockets" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd056b4095b3ec42b9a13d6985f3b1f7d2ee1a216440a9b25e122c15edfdce3" +dependencies = [ + "base64 0.21.2", + "bytes", + "fastrand 2.0.0", + "futures-util", + "http", + "httparse", + "sha1_smol", + "tokio", + "tokio-util", +] + [[package]] name = "toml" version = "0.5.11" @@ -12517,7 +12731,7 @@ checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", "digest 0.10.7", - "rand 0.8.5", + "rand 0.4.6", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index cc8d8128f..af93812fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ 'dkg-test-orchestrator', 'relayer-gadget', 'relayer-gadget/cli', + 'dkg-integration-tests' ] resolver = "2" @@ -32,7 +33,7 @@ pallet-bridge-registry = { path = "pallets/bridge-registry", default-features = dkg-gadget = { path = "dkg-gadget", default-features = false } dkg-primitives = { path = "dkg-primitives", default-features = false } dkg-standalone-runtime = { version = "3.0.0", path = "standalone/runtime" } -dkg-logging = { path = "dkg-logging" } +dkg-logging = { path = "dkg-logging", default-features = false } dkg-rococo-runtime = { default-features = false, path = "parachain/runtime/rococo" } dkg-mock-blockchain = { path = "dkg-mock-blockchain", default-features = false } dkg-test-orchestrator = { path = "dkg-test-orchestrator", default-features = false } @@ -75,6 +76,9 @@ tracing-subscriber = "0.3.5" sync_wrapper = "0.1.2" async-stream = "0.3.5" lazy_static = "1.4.0" +substrate-api-client = { git = "https://github.com/webb-tools/substrate-api-client" } +ac-primitives = { git = "https://github.com/webb-tools/substrate-api-client" } +tokio-websockets = "0.3.3" scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } diff --git a/Makefile.toml b/Makefile.toml index eab7b3a28..f8e1523d7 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -15,33 +15,43 @@ args = ["clean"] command = "cargo" args = ["build", "-rp", "dkg-standalone-node", "--features=integration-tests,testing"] +[tasks.integration-tests] +command = "cargo" +dependencies = ["clean-tmp", "build-test"] +args = ["run", "--bin", "dkg-integration-tests"] + [tasks.alice] -dependencies = ["build-test"] +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } description = "Run Alice node (the bootnode)" command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--alice", "--output-path=./tmp/alice/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30333", "--rpc-port", "9944", "--node-key", "0000000000000000000000000000000000000000000000000000000000000001"] [tasks.bob] -dependencies = ["build-test"] description = "Run Bob node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--bob", "--output-path=./tmp/bob/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30305", "--rpc-port", "9945", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.charlie] -dependencies = ["build-test"] description = "Run Charlie node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--charlie", "--output-path=./tmp/charlie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30306", "--rpc-port", "9946", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.dave] -dependencies = ["build-test"] description = "Run Dave node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--dave", "--output-path=./tmp/dave/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30307", "--rpc-port", "9947", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] [tasks.eve] -dependencies = ["build-test"] description = "Run Eve node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } command = "./target/release/dkg-standalone-node" args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--eve", "--output-path=./tmp/eve/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30308", "--rpc-port", "9948", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] +[tasks.ferdie] +description = "Run Ferdie node" +condition = { files_exist = ["${CARGO_MAKE_WORKING_DIRECTORY}/target/release/dkg-standalone-node"] } +command = "./target/release/dkg-standalone-node" +args = ["--tmp", "--chain", "local", "--validator", "-lerror", "--ferdie", "--output-path=./tmp/ferdie/output.log", "--rpc-cors", "all", "--rpc-external", "--rpc-methods=unsafe", "--port", "30309", "--rpc-port", "9949", "--bootnodes", "/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"] diff --git a/README.md b/README.md index 8f132a8d1..e1eef43d4 100644 --- a/README.md +++ b/README.md @@ -92,13 +92,15 @@ This should start the local testnet, you can view the logs in `/tmp` directory f The following instructions outline how to run dkg-substrate's base test suite and E2E test suite. -### To run base tests +### Unit Tests ``` cargo test ``` -### To run the test orchestrator E2E tests (recommended) +### Stress Tests + +When debugging the internal mechanics of the DKG, it is useful to run the stress tests. These tests are designed to run the DKG with a custom number of authorities. This allows you to quickly debug the DKG without having to setup a local chain. ``` # Build the dkg-standalone node @@ -108,6 +110,39 @@ cargo build --release -p dkg-standalone-node --features=integration-tests,testin cargo run --package dkg-test-orchestrator --release --features=testing -- --config /path/to/orchestrator_config.toml ``` +### Local Chain Tests (Automatic) + +To test the DKG against a local testnet, run: + +``` +cargo make integration-tests +``` + +### Local Chain Tests (Manual) + +If manual control is needed over the nodes (e.g., for debugging and/or shutting down nodes to test fault-tolerance), first build the binary: + +``` +cargo make build-test +``` + +Then, start the bootnode (Alice): + +``` +cargo make alice +``` + +Once Alice is running, start the other nodes each in separate terminals/processes: + +``` +cargo make bob +cargo make charlie +cargo make dave +cargo make eve +``` + + + ### Setting up debugging logs If you would like to run the dkg with verbose logs you may add the following arguments during initial setup. You may change the target to include `debug | error | info| trace | warn`. You may also want to review [Substrate runtime debugging](https://docs.substrate.io/v3/runtime/debugging/). diff --git a/dkg-gadget/Cargo.toml b/dkg-gadget/Cargo.toml index bd9acb4c4..70e44ca5e 100644 --- a/dkg-gadget/Cargo.toml +++ b/dkg-gadget/Cargo.toml @@ -64,7 +64,7 @@ webb-proposals = { workspace = true } # Local dependencies dkg-runtime-primitives = { workspace = true } dkg-primitives = { workspace = true } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } dkg-mock-blockchain = { workspace = true } tracing = { workspace = true, optional = true } diff --git a/dkg-gadget/src/dkg_modules/mp_ecdsa.rs b/dkg-gadget/src/dkg_modules/mp_ecdsa.rs index 2a7849f87..2a2f622b8 100644 --- a/dkg-gadget/src/dkg_modules/mp_ecdsa.rs +++ b/dkg-gadget/src/dkg_modules/mp_ecdsa.rs @@ -58,7 +58,6 @@ where party_i, session_id, stage, - crate::DKG_KEYGEN_PROTOCOL_NAME, associated_block, KEYGEN_SSID, ) { @@ -166,7 +165,6 @@ where party_i, session_id, stage, - crate::DKG_SIGNING_PROTOCOL_NAME, associated_block_id, ssid, )?; diff --git a/dkg-gadget/src/gossip_messages/misbehaviour_report.rs b/dkg-gadget/src/gossip_messages/misbehaviour_report.rs index d71a4f8ac..b051bd177 100644 --- a/dkg-gadget/src/gossip_messages/misbehaviour_report.rs +++ b/dkg-gadget/src/gossip_messages/misbehaviour_report.rs @@ -173,7 +173,7 @@ where report.session_id, encoded_signed_dkg_message.len() )); - if let Err(e) = dkg_worker.keygen_gossip_engine.gossip(signed_dkg_message) { + if let Err(e) = dkg_worker.gossip_engine.gossip(signed_dkg_message) { dkg_worker.logger.error(format!( "💀 (Round: {:?}) Failed to gossip misbehaviour message: {:?}", report.session_id, e diff --git a/dkg-gadget/src/lib.rs b/dkg-gadget/src/lib.rs index 7f172784b..321513c44 100644 --- a/dkg-gadget/src/lib.rs +++ b/dkg-gadget/src/lib.rs @@ -142,11 +142,6 @@ where dkg_keystore.clone(), ); - let signing_gossip_protocol = NetworkGossipEngineBuilder::new( - DKG_SIGNING_PROTOCOL_NAME.to_string().into(), - dkg_keystore.clone(), - ); - let logger_prometheus = debug_logger.clone(); let metrics = @@ -165,7 +160,7 @@ where let latest_header = Arc::new(RwLock::new(None)); - let (keygen_gossip_handler, keygen_gossip_engine) = keygen_gossip_protocol + let (gossip_handler, gossip_engine) = keygen_gossip_protocol .build( network.clone(), sync_service.clone(), @@ -175,27 +170,13 @@ where ) .expect("Keygen : Failed to build gossip engine"); - let (signing_gossip_handler, signing_gossip_engine) = signing_gossip_protocol - .build( - network.clone(), - sync_service.clone(), - metrics.clone(), - latest_header.clone(), - debug_logger.clone(), - ) - .expect("Signing : Failed to build gossip engine"); - // enable the gossip - keygen_gossip_engine.set_gossip_enabled(true); - signing_gossip_engine.set_gossip_enabled(true); + gossip_engine.set_gossip_enabled(true); // keygen_gossip_engine.set_processing_already_seen_messages_enabled(false); // signing_gossip_engine.set_processing_already_seen_messages_enabled(false); - let keygen_handle = - crate::utils::ExplicitPanicFuture::new(tokio::spawn(keygen_gossip_handler.run())); - let signing_handle = - crate::utils::ExplicitPanicFuture::new(tokio::spawn(signing_gossip_handler.run())); + let gossip_handle = crate::utils::ExplicitPanicFuture::new(tokio::spawn(gossip_handler.run())); // In memory backend, not used for now // let db_backend = Arc::new(db::DKGInMemoryDb::new()); @@ -211,8 +192,7 @@ where client, backend, key_store: dkg_keystore, - keygen_gossip_engine, - signing_gossip_engine, + gossip_engine, db_backend, metrics, local_keystore, @@ -225,8 +205,7 @@ where let worker = worker::DKGWorker::<_, _, _, _>::new(worker_params, debug_logger); worker.run().await; - keygen_handle.abort(); - signing_handle.abort(); + gossip_handle.abort(); } pub mod deadlock_detection { diff --git a/dkg-gadget/src/worker.rs b/dkg-gadget/src/worker.rs index 5c31052a1..63248ce44 100644 --- a/dkg-gadget/src/worker.rs +++ b/dkg-gadget/src/worker.rs @@ -83,8 +83,7 @@ where pub client: Arc, pub backend: Arc, pub key_store: DKGKeystore, - pub keygen_gossip_engine: GE, - pub signing_gossip_engine: GE, + pub gossip_engine: GE, pub db_backend: Arc, pub metrics: Option, pub local_keystore: Option>, @@ -106,8 +105,7 @@ where pub client: Arc, pub backend: Arc, pub key_store: DKGKeystore, - pub keygen_gossip_engine: Arc, - pub signing_gossip_engine: Arc, + pub gossip_engine: Arc, pub db: Arc, pub metrics: Arc>, /// Cached best authorities @@ -169,8 +167,7 @@ where backend: self.backend.clone(), key_store: self.key_store.clone(), db: self.db.clone(), - keygen_gossip_engine: self.keygen_gossip_engine.clone(), - signing_gossip_engine: self.signing_gossip_engine.clone(), + gossip_engine: self.gossip_engine.clone(), metrics: self.metrics.clone(), best_authorities: self.best_authorities.clone(), next_best_authorities: self.next_best_authorities.clone(), @@ -220,8 +217,7 @@ where backend, key_store, db_backend, - keygen_gossip_engine, - signing_gossip_engine, + gossip_engine, metrics, local_keystore, latest_header, @@ -246,8 +242,7 @@ where key_store, db: db_backend, keygen_manager, - keygen_gossip_engine: Arc::new(keygen_gossip_engine), - signing_gossip_engine: Arc::new(signing_gossip_engine), + gossip_engine: Arc::new(gossip_engine), metrics: Arc::new(metrics), best_authorities: Arc::new(RwLock::new(vec![])), next_best_authorities: Arc::new(RwLock::new(vec![])), @@ -305,7 +300,6 @@ where party_i: KeygenPartyId, session_id: SessionId, stage: ProtoStageType, - protocol_name: &str, associated_block: NumberFor, ssid: SSID, ) -> Result< @@ -365,7 +359,7 @@ where client: self.client.clone(), keystore: self.key_store.clone(), db: self.db.clone(), - gossip_engine: self.get_gossip_engine_from_protocol_name(protocol_name), + gossip_engine: self.gossip_engine.clone(), aggregated_public_keys: self.aggregated_public_keys.clone(), best_authorities: best_authorities.clone(), authority_public_key: authority_public_key.clone(), @@ -407,15 +401,6 @@ where } } - /// Returns the gossip engine based on the protocol_name - fn get_gossip_engine_from_protocol_name(&self, protocol_name: &str) -> Arc { - match protocol_name { - crate::DKG_KEYGEN_PROTOCOL_NAME => self.keygen_gossip_engine.clone(), - crate::DKG_SIGNING_PROTOCOL_NAME => self.signing_gossip_engine.clone(), - _ => panic!("Protocol name not found!"), - } - } - /// Fetch the stored local keys if they exist. fn fetch_local_keys( &self, @@ -1050,90 +1035,68 @@ where // We run all these tasks in parallel and wait for any of them to complete. // If any of them completes, we stop all the other tasks since this means a fatal error has // occurred and we need to shut down. - let (first, n, ..) = futures::future::select_all(vec![ - self.spawn_finality_notification_task(), - self.spawn_keygen_messages_stream_task(), - self.spawn_signing_messages_stream_task(), - self.spawn_error_handling_task(), - ]) - .await; + let finality_notification_task = self.finality_notification_task(); + let gossip_engine_stream_task = self.gossip_engine_message_stream_task(); + let error_handling_task = self.spawn_error_handling_task(); + + let res = tokio::select! { + res0 = finality_notification_task => res0, + res1 = gossip_engine_stream_task => res1, + res2 = error_handling_task => res2, + }; + self.logger - .error(format!("DKG Worker finished; the reason that task({n}) ended with: {first:?}")); + .error(format!("DKG Worker finished prematurely. The cause: {res:?}")); } - fn spawn_finality_notification_task(&self) -> tokio::task::JoinHandle<()> { + async fn finality_notification_task(&self) -> Result<(), DKGError> { let mut stream = self.client.finality_notification_stream(); - let self_ = self.clone(); - tokio::spawn(async move { - while let Some(notification) = stream.next().await { - dkg_logging::debug!("Going to handle Finality notification"); - self_.handle_finality_notification(notification).await; - } - self_.logger.error("Finality notification stream ended"); - }) - } + while let Some(notification) = stream.next().await { + dkg_logging::debug!("Going to handle Finality notification"); + self.handle_finality_notification(notification).await; + } - fn spawn_keygen_messages_stream_task(&self) -> tokio::task::JoinHandle<()> { - let keygen_gossip_engine = self.keygen_gossip_engine.clone(); - let mut keygen_stream = - keygen_gossip_engine.get_stream().expect("keygen gossip stream already taken"); - let self_ = self.clone(); - tokio::spawn(async move { - while let Some(msg) = keygen_stream.recv().await { - let msg_hash = crate::debug_logger::raw_message_to_hash(msg.msg.payload.payload()); - self_.logger.debug(format!( - "Going to handle keygen message for session {} | hash: {msg_hash}", - msg.msg.session_id - )); - self_.logger.checkpoint_message_raw(msg.msg.payload.payload(), "CP1-keygen"); - match self_.process_incoming_dkg_message(msg).await { - Ok(_) => {}, - Err(e) => { - self_.logger.error(format!("Error processing keygen message: {e:?}")); - }, - } - } - }) + self.logger.error("Finality notification stream ended"); + + Err(DKGError::CriticalError { reason: "Finality notification stream ended".to_string() }) } - fn spawn_signing_messages_stream_task(&self) -> tokio::task::JoinHandle<()> { - let signing_gossip_engine = self.signing_gossip_engine.clone(); - let mut signing_stream = - signing_gossip_engine.get_stream().expect("signing gossip stream already taken"); - let self_ = self.clone(); - tokio::spawn(async move { - while let Some(msg) = signing_stream.recv().await { - self_.logger.debug(format!( - "Going to handle signing message for session {}", - msg.msg.session_id - )); - self_.logger.checkpoint_message_raw(msg.msg.payload.payload(), "CP1-signing"); - match self_.process_incoming_dkg_message(msg).await { - Ok(_) => {}, - Err(e) => { - self_.logger.error(format!("Error processing signing message: {e:?}")); - }, - } + async fn gossip_engine_message_stream_task(&self) -> Result<(), DKGError> { + let mut stream = + self.gossip_engine.get_stream().expect("keygen gossip stream already taken"); + + while let Some(msg) = stream.recv().await { + let msg_hash = crate::debug_logger::raw_message_to_hash(msg.msg.payload.payload()); + self.logger.debug(format!( + "Going to handle message for session {} | hash: {msg_hash}", + msg.msg.session_id + )); + self.logger.checkpoint_message_raw(msg.msg.payload.payload(), "CP1-incoming"); + match self.process_incoming_dkg_message(msg).await { + Ok(_) => {}, + Err(e) => { + self.logger.error(format!("Error processing keygen message: {e:?}")); + }, } - }) + } + + Err(DKGError::CriticalError { reason: "Gossip engine stream ended".to_string() }) } - fn spawn_error_handling_task(&self) -> tokio::task::JoinHandle<()> { - let self_ = self.clone(); + async fn spawn_error_handling_task(&self) -> Result<(), DKGError> { let mut error_handler_rx = self .error_handler_channel .rx .lock() .take() .expect("Error handler tx already taken"); - let logger = self.logger.clone(); - tokio::spawn(async move { - while let Some(error) = error_handler_rx.recv().await { - logger.debug("Going to handle Error"); - self_.handle_dkg_error(error).await; - } - }) + while let Some(error) = error_handler_rx.recv().await { + self.logger.debug("Going to handle Error"); + self.handle_dkg_error(error).await; + } + + Err(DKGError::CriticalError { reason: "Error handler stream ended".to_string() }) } } diff --git a/dkg-integration-tests/Cargo.toml b/dkg-integration-tests/Cargo.toml new file mode 100644 index 000000000..1aefc73b6 --- /dev/null +++ b/dkg-integration-tests/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "dkg-integration-tests" +version = "0.1.0" +description.workspace = true +authors.workspace = true +license.workspace = true +publish.workspace = true +homepage.workspace = true +repository.workspace = true +edition.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { workspace = true, features = ["full"] } +dkg-logging = { workspace = true, default-features = false } +dkg-standalone-runtime = { workspace = true } +substrate-api-client = { workspace = true, features = ["std"] } +ac-primitives = { workspace = true } +tokio-websockets = { workspace = true } diff --git a/dkg-integration-tests/src/main.rs b/dkg-integration-tests/src/main.rs new file mode 100644 index 000000000..003759811 --- /dev/null +++ b/dkg-integration-tests/src/main.rs @@ -0,0 +1,210 @@ +use ac_primitives::{AssetRuntimeConfig, H256}; +use std::{collections::HashSet, error::Error, time::Duration}; +use substrate_api_client::{rpc::JsonrpseeClient, Api, SubscribeEvents}; +use tokio::{net::TcpStream, process::Child}; +use tokio_websockets::{ClientBuilder, MaybeTlsStream, WebsocketStream}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let mut shutdown = Shutdown { members: vec![] }; + let res = main_inner(&mut shutdown).await; + + for member in shutdown.members.iter_mut() { + if let Some(pid) = member.id() { + dkg_logging::info!(target: "dkg", "Killing member {pid}"); + let _exit = tokio::process::Command::new("kill") + .arg("-9") + .arg(pid.to_string()) + .stderr(std::process::Stdio::inherit()) + .stdout(std::process::Stdio::inherit()) + .spawn() + .expect("Should spawn") + .wait() + .await + .expect("Should exec"); + } + + member.kill().await.expect("Should kill member"); + } + + res +} + +async fn main_inner(shutdown: &mut Shutdown) -> Result<(), Box> { + const SESSION_N: u64 = 2; + dkg_logging::setup_log(); + + dkg_logging::info!(target: "dkg", "Will test getting to session {SESSION_N}"); + + let alice = generate_process("alice", false)?; + dkg_logging::info!(target: "dkg", "Spawned Bootnode, now awaiting for initialization"); + // With alice spawned, we now must wait for her to be ready to accept connections + wait_for_bootnode_to_init().await?; + + // with the bootnode ready, load the rest of the nodes + let bob = generate_process("bob", false)?; + let charlie = generate_process("charlie", false)?; + let dave = generate_process("dave", false)?; + let eve = generate_process("eve", false)?; + + shutdown.members = vec![alice, bob, charlie, dave, eve]; + + // Setup API + let client = JsonrpseeClient::with_default_url().expect("Should create client"); + let mut api = + Api::::new(client).expect("Should create API"); + + // Wait for the bootnode to get to SESSION_N + wait_for_session_n(&mut api, SESSION_N).await; + Ok(()) +} + +async fn wait_for_bootnode_to_init() -> Result<(), Box> { + loop { + match tokio::time::timeout( + Duration::from_millis(10000), + generate_websocket_connection_to_bootnode(), + ) + .await + { + Ok(Ok(_ws)) => { + dkg_logging::info!(target: "dkg", "Obtained websocket connection to bootnode"); + return Ok(()) + }, + + Ok(Err(_err)) => {}, + + Err(_err) => { + dkg_logging::info!(target: "dkg", "Still waiting for bootnode to init ..."); + }, + } + } +} + +async fn generate_websocket_connection_to_bootnode( +) -> Result>, Box> { + Ok(ClientBuilder::new() + .uri("ws://127.0.0.1:9944")? // connect to the bootnode's WS port to listen to events + .connect() + .await?) +} + +async fn wait_for_session_n( + api: &mut Api, + session_needed: u64, +) { + let mut has_seen = HashSet::new(); + let mut listener = DKGEventListener::new(); + loop { + let mut subscription = api.subscribe_events().expect("Should subscribe to events"); + let records = tokio::task::spawn_blocking(move || { + let events = subscription.next_events::(); + if let Some(events) = events { + events.unwrap_or_default() + } else { + vec![] + } + }) + .await + .expect("JoinError"); + + for record in records { + let record_str = format!("{record:?}"); + if has_seen.insert(record_str.clone()) { + dkg_logging::info!(target: "dkg", "decoded: {:?}", record.event); + let session = listener.on_event_received(&record_str); + dkg_logging::info!(target: "dkg", "State: {listener:?}"); + if session == session_needed { + return + } + } + } + } +} + +fn generate_process(node: &str, inherit: bool) -> Result> { + let mut command = tokio::process::Command::new("cargo"); + + if inherit { + command.stdout(std::process::Stdio::inherit()); + command.stderr(std::process::Stdio::inherit()); + } else { + command.stdout(std::process::Stdio::piped()); + command.stderr(std::process::Stdio::piped()); + } + + Ok(command + .arg("make") + .arg(node) + .env("RUST_LOG", "dkg=info") + .kill_on_drop(true) + .spawn()?) +} + +struct Shutdown { + members: Vec, +} + +const EVENT_DKG_PUBLIC_KEY_SUBMITTED: &str = "Event::PublicKeySubmitted"; +const EVENT_NEXT_DKG_PUBLIC_KEY_SUBMITTED: &str = "Event::NextPublicKeySubmitted"; +const EVENT_DKG_PROPOSAL_ADDED: &str = "Event::ProposalAdded"; +const EVENT_DKG_PROPOSAL_BATCH_SIGNED: &str = "Event::ProposalBatchSigned"; +const EVENT_NEXT_DKG_PUBLIC_KEY_SIGNATURE_SUBMITTED: &str = + "Event::NextPublicKeySignatureSubmitted"; +const EVENT_NEW_SESSION: &str = "Event::NewSession"; +const EVENT_PUBLIC_KEY_CHANGED: &str = "Event::PublicKeyChanged"; +const EVENT_PUBLIC_KEY_SIGNATURE_CHANGED: &str = "Event::PublicKeySignatureChanged"; + +#[derive(Debug)] +struct DKGEventListener { + current_session: u64, + current_session_events_required: HashSet<&'static str>, +} + +impl DKGEventListener { + fn new() -> Self { + Self { + current_session: 0, + current_session_events_required: [ + EVENT_DKG_PUBLIC_KEY_SUBMITTED, + EVENT_NEXT_DKG_PUBLIC_KEY_SUBMITTED, + EVENT_DKG_PROPOSAL_ADDED, + EVENT_DKG_PROPOSAL_BATCH_SIGNED, + EVENT_NEXT_DKG_PUBLIC_KEY_SIGNATURE_SUBMITTED, + EVENT_NEW_SESSION, + EVENT_PUBLIC_KEY_CHANGED, + EVENT_PUBLIC_KEY_SIGNATURE_CHANGED, + ] + .into_iter() + .collect(), + } + } + // Returns the current session + fn on_event_received(&mut self, event: &str) -> u64 { + let maybe_matched_event = self + .current_session_events_required + .iter() + .find(|exact| event.contains(**exact)); + if let Some(matched_event) = maybe_matched_event { + self.current_session_events_required.remove(*matched_event); + if self.current_session_events_required.is_empty() { + self.current_session += 1; + self.current_session_events_required = [ + // EVENT_DKG_PUBLIC_KEY_SUBMITTED, omit this since this is only needed in the + // zeroth session (genesis) + EVENT_NEXT_DKG_PUBLIC_KEY_SUBMITTED, + EVENT_DKG_PROPOSAL_ADDED, + EVENT_DKG_PROPOSAL_BATCH_SIGNED, + EVENT_NEXT_DKG_PUBLIC_KEY_SIGNATURE_SUBMITTED, + EVENT_NEW_SESSION, + EVENT_PUBLIC_KEY_CHANGED, + EVENT_PUBLIC_KEY_SIGNATURE_CHANGED, + ] + .into_iter() + .collect(); + } + } + + self.current_session + } +} diff --git a/dkg-logging/Cargo.toml b/dkg-logging/Cargo.toml index b34394070..d6a86ab83 100644 --- a/dkg-logging/Cargo.toml +++ b/dkg-logging/Cargo.toml @@ -9,14 +9,18 @@ edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = ["full"] +full = ["parking_lot", "serde", "sp-core", "lazy_static", "tokio", "serde_json", "hex"] + [dependencies] tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } tracing-filter = "0.1.0-alpha.2" -parking_lot = { workspace = true } -serde = { workspace = true } -sp-core = { workspace = true } -lazy_static = { workspace = true } -tokio = { workspace = true } -serde_json = { workspace = true } -hex = { workspace = true } +parking_lot = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +sp-core = { workspace = true, optional = true } +lazy_static = { workspace = true, optional = true } +tokio = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +hex = { workspace = true, optional = true } diff --git a/dkg-logging/src/lib.rs b/dkg-logging/src/lib.rs index f7213b876..18905dcf8 100644 --- a/dkg-logging/src/lib.rs +++ b/dkg-logging/src/lib.rs @@ -5,6 +5,7 @@ use tracing_subscriber::{ EnvFilter, }; +#[cfg(feature = "full")] pub mod debug_logger; pub fn setup_log() { diff --git a/dkg-mock-blockchain/Cargo.toml b/dkg-mock-blockchain/Cargo.toml index 6c330678c..5ce87a04d 100644 --- a/dkg-mock-blockchain/Cargo.toml +++ b/dkg-mock-blockchain/Cargo.toml @@ -23,4 +23,4 @@ sc-network = { workspace = true } sc-utils = { workspace = true } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } dkg-runtime-primitives = { workspace = true, features = ["testing", "std"] } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } diff --git a/dkg-test-orchestrator/Cargo.toml b/dkg-test-orchestrator/Cargo.toml index abfaf7066..aeca6fc47 100644 --- a/dkg-test-orchestrator/Cargo.toml +++ b/dkg-test-orchestrator/Cargo.toml @@ -12,7 +12,7 @@ debug-tracing = ["dkg-gadget/debug-tracing"] tokio = { workspace = true, features = ["rt-multi-thread", "fs", "macros", "process"] } toml = { workspace = true, features = ["parse"] } dkg-mock-blockchain = { workspace = true } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } dkg-gadget = { workspace = true, features = ["testing"] } structopt = { workspace = true } log = { workspace = true } @@ -34,4 +34,4 @@ sp-core = { workspace = true } sp-keystore = { workspace = true } uuid = { workspace = true, features = ["v4"] } frame-support = { workspace = true } -rand = "0.8.5" \ No newline at end of file +rand = "0.8.5" diff --git a/dkg-test-orchestrator/src/main.rs b/dkg-test-orchestrator/src/main.rs index eb856f20f..1e7025e4c 100644 --- a/dkg-test-orchestrator/src/main.rs +++ b/dkg-test-orchestrator/src/main.rs @@ -70,8 +70,7 @@ async fn main() -> Result<(), Box> { // the gossip engine and the dummy api share a state between ALL clients in this process // we will use the SAME gossip engine for both keygen and signing - let keygen_gossip_engine = &InMemoryGossipEngine::new(); - let signing_gossip_engine = &InMemoryGossipEngine::new(); + let gossip_engine = &InMemoryGossipEngine::new(); let keygen_t = t as u16; let keygen_n = n_clients as u16; let signing_t = t as u16; @@ -134,20 +133,13 @@ async fn main() -> Result<(), Box> { // output the logs for this specific peer to a file let output = args.tmp_path.join(format!("{peer_id}.log")); let logger = dkg_gadget::debug_logger::DebugLogger::new(peer_id, Some(output))?; - let keygen_gossip_engine = keygen_gossip_engine.clone_for_new_peer( + let gossip_engine = gossip_engine.clone_for_new_peer( api, n_blocks as _, peer_id, public_key.clone(), &logger, ); - let signing_gossip_engine = signing_gossip_engine.clone_for_new_peer( - api, - n_blocks as _, - peer_id, - public_key, - &logger, - ); key_store.set_logger(logger.clone()); @@ -179,8 +171,7 @@ async fn main() -> Result<(), Box> { client, backend, key_store, - keygen_gossip_engine, - signing_gossip_engine, + gossip_engine, db_backend, metrics, local_keystore, diff --git a/standalone/node/Cargo.toml b/standalone/node/Cargo.toml index 3214cdef9..d88c23267 100644 --- a/standalone/node/Cargo.toml +++ b/standalone/node/Cargo.toml @@ -65,7 +65,7 @@ dkg-gadget = { workspace = true } dkg-runtime-primitives = { workspace = true } dkg-primitives = { workspace = true } dkg-standalone-runtime = { workspace = true } -dkg-logging = { workspace = true } +dkg-logging = { workspace = true, features = ["full"] } webb-relayer-gadget = { workspace = true } webb-relayer-gadget-cli = { workspace = true } tokio = { workspace = true, features = ["rt"] }