From d2e068d76b882f95f2b9493cd1e61fd1feb2acbb Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 6 Jun 2023 17:21:17 +0200 Subject: [PATCH 01/48] [WIP] PVF: Split out worker binaries --- Cargo.toml | 8 ++++++++ node/core/candidate-validation/src/lib.rs | 6 +++--- node/core/pvf/execute-worker/src/main.rs | 22 +++++++++++++++++++++ node/core/pvf/prepare-worker/src/main.rs | 24 +++++++++++++++++++++++ node/core/pvf/src/host.rs | 18 +++++++++++++---- node/service/src/lib.rs | 13 ++++++++---- 6 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 node/core/pvf/execute-worker/src/main.rs create mode 100644 node/core/pvf/prepare-worker/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index cc3b4e4c1d35..6912d868ce78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,14 @@ name = "polkadot" path = "src/main.rs" +[[bin]] +name = "polkadot-execute-worker" +path = "node/core/pvf/execute-worker/src/main.rs" + +[[bin]] +name = "polkadot-prepare-worker" +path = "node/core/pvf/prepare-worker/src/main.rs" + [package] name = "polkadot" description = "Implementation of a `https://polkadot.network` node in Rust based on the Substrate framework." diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index b2aa0fd15105..cc218a1af32a 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -93,8 +93,8 @@ const DEFAULT_APPROVAL_EXECUTION_TIMEOUT: Duration = Duration::from_secs(12); pub struct Config { /// The path where candidate validation can store compiled artifacts for PVFs. pub artifacts_cache_path: PathBuf, - /// The path to the executable which can be used for spawning PVF compilation & validation - /// workers. + /// The path to the directory with the executables which can be used for spawning PVF + /// compilation & validation workers. pub program_path: PathBuf, } @@ -143,7 +143,7 @@ async fn run( metrics: Metrics, pvf_metrics: polkadot_node_core_pvf::Metrics, cache_path: PathBuf, - program_path: PathBuf, + program_path: Option, ) -> SubsystemResult<()> { let (validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), diff --git a/node/core/pvf/execute-worker/src/main.rs b/node/core/pvf/execute-worker/src/main.rs new file mode 100644 index 000000000000..5724b2b9b92a --- /dev/null +++ b/node/core/pvf/execute-worker/src/main.rs @@ -0,0 +1,22 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Execute worker. + +polkadot_node_core_pvf_common::decl_worker_main!( + "execute-worker", + polkadot_node_core_pvf_execute_worker::worker_entrypoint +); diff --git a/node/core/pvf/prepare-worker/src/main.rs b/node/core/pvf/prepare-worker/src/main.rs new file mode 100644 index 000000000000..1757dbde1893 --- /dev/null +++ b/node/core/pvf/prepare-worker/src/main.rs @@ -0,0 +1,24 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Prepare worker. + +pub use polkadot_node_core_pvf_common::decl_worker_main; + +polkadot_node_core_pvf_common::decl_worker_main!( + "prepare-worker", + polkadot_node_core_pvf_prepare_worker::worker_entrypoint +); diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index f9b00823625e..a5b5d78af680 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -52,6 +52,9 @@ pub const PREPARE_FAILURE_COOLDOWN: Duration = Duration::from_millis(200); /// The amount of times we will retry failed prepare jobs. pub const NUM_PREPARE_RETRIES: u32 = 5; +const PREPARE_BINARY_NAME: &str = "polkadot-prepare-worker"; +const EXECUTE_BINARY_NAME: &str = "polkadot-execute-worker"; + /// An alias to not spell the type for the oneshot sender for the PVF execution result. pub(crate) type ResultSender = oneshot::Sender>; @@ -162,18 +165,25 @@ pub struct Config { impl Config { /// Create a new instance of the configuration. - pub fn new(cache_path: std::path::PathBuf, program_path: std::path::PathBuf) -> Self { + pub fn new(cache_path: std::path::PathBuf, program_path: Option) -> Self { // Do not contaminate the other parts of the codebase with the types from `tokio`. let cache_path = PathBuf::from(cache_path); - let program_path = PathBuf::from(program_path); + let program_path = program_path.map(PathBuf::from); + + // If the worker directory is given, get the binaries from that directory. Otherwise, expect + // that they are in $PATH. + let (prepare_worker_program_path, execute_worker_program_path) = match program_path { + Some(p) => (p.join(PREPARE_BINARY_NAME), p.join(EXECUTE_BINARY_NAME)), + None => (PREPARE_BINARY_NAME.into(), EXECUTE_BINARY_NAME.into()), + }; Self { cache_path, - prepare_worker_program_path: program_path.clone(), + prepare_worker_program_path, prepare_worker_spawn_timeout: Duration::from_secs(3), prepare_workers_soft_max_num: 1, prepare_workers_hard_max_num: 1, - execute_worker_program_path: program_path, + execute_worker_program_path, execute_worker_spawn_timeout: Duration::from_secs(3), execute_workers_max_num: 2, } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index c29933732388..83dbc78007ef 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -676,6 +676,7 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { col_meta: parachains_db::REAL_COLUMNS.col_availability_meta, }; +// TODO: This function seems to be public? How to not break backwards compatibility? /// Create a new full node of arbitrary runtime and executor. /// /// This is an advanced feature and not recommended for general use. Generally, `build_full` is @@ -684,6 +685,12 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { /// `overseer_enable_anyways` always enables the overseer, based on the provided `OverseerGenerator`, /// regardless of the role the node has. The relay chain selection (longest or disputes-aware) is /// still determined based on the role of the node. Likewise for authority discovery. +/// +/// `program_path` and `worker_directory_path` are used to get the PVF worker locations. First, +/// `program_path` is considered, and if it is passed then that location will be used for both the +/// binaries. If it is not passed, `worker_directory_path` is considered, and if it is passed then +/// the binaries from that directory will be used. If neither is passed, the binaries will be taken +/// from `$PATH`. `program_path` is a deprecated option and is present for backwards compatibility. #[cfg(feature = "full-node")] pub fn new_full( mut config: Configuration, @@ -693,6 +700,7 @@ pub fn new_full( jaeger_agent: Option, telemetry_worker_handle: Option, program_path: Option, + worker_directory_path: Option, overseer_enable_anyways: bool, overseer_gen: OverseerGenerator, overseer_message_channel_capacity_override: Option, @@ -905,10 +913,7 @@ where .path() .ok_or(Error::DatabasePathRequired)? .join("pvf-artifacts"), - program_path: match program_path { - None => std::env::current_exe()?, - Some(p) => p, - }, + program_path, }; let chain_selection_config = ChainSelectionConfig { From 5bed7268f28e7fd57303efdb2d4c39f6f697bdd3 Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Sat, 10 Jun 2023 13:44:51 +0200 Subject: [PATCH 02/48] Address compilation problems and re-design a bit --- Cargo.lock | 4 ++ Cargo.toml | 3 ++ cli/src/cli.rs | 6 +++ cli/src/command.rs | 1 + node/core/candidate-validation/src/lib.rs | 8 ++-- node/core/pvf/src/host.rs | 15 +++---- node/core/pvf/src/lib.rs | 2 +- node/service/Cargo.toml | 2 + node/service/src/lib.rs | 45 +++++++++++++------ .../adder/collator/src/main.rs | 1 + .../undying/collator/src/main.rs | 1 + 11 files changed, 62 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 162bd76a76c0..88ceb81698a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6914,8 +6914,11 @@ dependencies = [ "nix 0.26.2", "polkadot-cli", "polkadot-core-primitives", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", + "sp-tracing", "substrate-rpc-client", "tempfile", "tikv-jemallocator", @@ -8230,6 +8233,7 @@ dependencies = [ "polkadot-node-core-dispute-coordinator", "polkadot-node-core-parachains-inherent", "polkadot-node-core-provisioner", + "polkadot-node-core-pvf", "polkadot-node-core-pvf-checker", "polkadot-node-core-runtime-api", "polkadot-node-network-protocol", diff --git a/Cargo.toml b/Cargo.toml index 6912d868ce78..73324f5aa2f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,9 +29,12 @@ version = "0.9.41" [dependencies] color-eyre = { version = "0.6.1", default-features = false } tikv-jemallocator = "0.5.0" +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } # Crates in our workspace, defined as dependencies so we can pass them feature flags. polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ] } +polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } polkadot-overseer = { path = "node/overseer" } diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 69c54b428a92..db2c4ff67a50 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -16,6 +16,7 @@ //! Polkadot CLI library. +use std::path::PathBuf; use clap::Parser; #[allow(missing_docs)] @@ -148,6 +149,11 @@ pub struct RunCmd { /// **Dangerous!** Do not touch unless explicitly adviced to. #[arg(long)] pub overseer_channel_capacity_override: Option, + + /// Path to auxiliary worker binaries. If not specified, the main binary's directory is + /// searched first, and then the `$PATH` is considered. + #[arg(long, value_name = "PATH")] + pub workers_path: Option, } #[allow(missing_docs)] diff --git a/cli/src/command.rs b/cli/src/command.rs index 378e8e7650c2..c37d1ad1f6f1 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -345,6 +345,7 @@ where grandpa_pause, enable_beefy, jaeger_agent, + cli.run.workers_path, None, false, overseer_gen, diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index cc218a1af32a..9c68b004caa9 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -95,7 +95,7 @@ pub struct Config { pub artifacts_cache_path: PathBuf, /// The path to the directory with the executables which can be used for spawning PVF /// compilation & validation workers. - pub program_path: PathBuf, + pub workers_path: Option, } /// The candidate validation subsystem. @@ -129,7 +129,7 @@ impl CandidateValidationSubsystem { self.metrics, self.pvf_metrics, self.config.artifacts_cache_path, - self.config.program_path, + self.config.workers_path, ) .map_err(|e| SubsystemError::with_origin("candidate-validation", e)) .boxed(); @@ -143,10 +143,10 @@ async fn run( metrics: Metrics, pvf_metrics: polkadot_node_core_pvf::Metrics, cache_path: PathBuf, - program_path: Option, + workers_path: Option, ) -> SubsystemResult<()> { let (validation_host, task) = polkadot_node_core_pvf::start( - polkadot_node_core_pvf::Config::new(cache_path, program_path), + polkadot_node_core_pvf::Config::new(cache_path, workers_path), pvf_metrics, ); ctx.spawn_blocking("pvf-validation-host", task.boxed())?; diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index a5b5d78af680..8bbe9f837f0c 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -52,8 +52,11 @@ pub const PREPARE_FAILURE_COOLDOWN: Duration = Duration::from_millis(200); /// The amount of times we will retry failed prepare jobs. pub const NUM_PREPARE_RETRIES: u32 = 5; -const PREPARE_BINARY_NAME: &str = "polkadot-prepare-worker"; -const EXECUTE_BINARY_NAME: &str = "polkadot-execute-worker"; +/// The name of binary spawned to prepare a PVF artifact +pub const PREPARE_BINARY_NAME: &str = "polkadot-prepare-worker"; + +/// The name of binary spawned to execute a PVF +pub const EXECUTE_BINARY_NAME: &str = "polkadot-execute-worker"; /// An alias to not spell the type for the oneshot sender for the PVF execution result. pub(crate) type ResultSender = oneshot::Sender>; @@ -165,14 +168,10 @@ pub struct Config { impl Config { /// Create a new instance of the configuration. - pub fn new(cache_path: std::path::PathBuf, program_path: Option) -> Self { - // Do not contaminate the other parts of the codebase with the types from `tokio`. - let cache_path = PathBuf::from(cache_path); - let program_path = program_path.map(PathBuf::from); - + pub fn new(cache_path: std::path::PathBuf, workers_path: Option) -> Self { // If the worker directory is given, get the binaries from that directory. Otherwise, expect // that they are in $PATH. - let (prepare_worker_program_path, execute_worker_program_path) = match program_path { + let (prepare_worker_program_path, execute_worker_program_path) = match workers_path { Some(p) => (p.join(PREPARE_BINARY_NAME), p.join(EXECUTE_BINARY_NAME)), None => (PREPARE_BINARY_NAME.into(), EXECUTE_BINARY_NAME.into()), }; diff --git a/node/core/pvf/src/lib.rs b/node/core/pvf/src/lib.rs index 8696119f801c..c9c7b8fe446f 100644 --- a/node/core/pvf/src/lib.rs +++ b/node/core/pvf/src/lib.rs @@ -105,7 +105,7 @@ pub mod testing; pub use sp_tracing; pub use error::{InvalidCandidate, ValidationError}; -pub use host::{start, Config, ValidationHost}; +pub use host::{start, Config, ValidationHost, PREPARE_BINARY_NAME, EXECUTE_BINARY_NAME}; pub use metrics::Metrics; pub use priority::Priority; pub use worker_intf::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index 34b69d045753..73a069b6dd34 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -126,6 +126,7 @@ polkadot-node-core-chain-api = { path = "../core/chain-api", optional = true } polkadot-node-core-chain-selection = { path = "../core/chain-selection", optional = true } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator", optional = true } polkadot-node-core-provisioner = { path = "../core/provisioner", optional = true } +polkadot-node-core-pvf = { path = "../core/pvf", optional = true } polkadot-node-core-pvf-checker = { path = "../core/pvf-checker", optional = true } polkadot-node-core-runtime-api = { path = "../core/runtime-api", optional = true } polkadot-statement-distribution = { path = "../network/statement-distribution", optional = true } @@ -166,6 +167,7 @@ full-node = [ "polkadot-node-core-runtime-api", "polkadot-statement-distribution", "polkadot-approval-distribution", + "polkadot-node-core-pvf", "polkadot-node-core-pvf-checker", "kvdb-rocksdb", "parity-db", diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 83dbc78007ef..c51edfc198ea 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -70,7 +70,7 @@ pub use { #[cfg(feature = "full-node")] use polkadot_node_subsystem::jaeger; -use std::{sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration, path::PathBuf}; use prometheus_endpoint::Registry; #[cfg(feature = "full-node")] @@ -686,11 +686,10 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { /// regardless of the role the node has. The relay chain selection (longest or disputes-aware) is /// still determined based on the role of the node. Likewise for authority discovery. /// -/// `program_path` and `worker_directory_path` are used to get the PVF worker locations. First, -/// `program_path` is considered, and if it is passed then that location will be used for both the -/// binaries. If it is not passed, `worker_directory_path` is considered, and if it is passed then -/// the binaries from that directory will be used. If neither is passed, the binaries will be taken -/// from `$PATH`. `program_path` is a deprecated option and is present for backwards compatibility. +/// `workers_path` is used to get the PVF worker locations. First, `workers_path` is considered, +/// and if it is passed then that location will be used for both the binaries. If it is not passed, +/// current exe's directory is considered and checked for pinaries presence. If not found, the +/// binaries will be taken from `$PATH`. #[cfg(feature = "full-node")] pub fn new_full( mut config: Configuration, @@ -699,8 +698,7 @@ pub fn new_full( enable_beefy: bool, jaeger_agent: Option, telemetry_worker_handle: Option, - program_path: Option, - worker_directory_path: Option, + workers_path: Option, overseer_enable_anyways: bool, overseer_gen: OverseerGenerator, overseer_message_channel_capacity_override: Option, @@ -907,13 +905,33 @@ where slot_duration_millis: slot_duration.as_millis() as u64, }; + let workers_path = if workers_path.is_some() { + log::trace!("Using explicitly provided workers path {:?}", workers_path); + workers_path + } else { + let mut exe_path = std::env::current_exe()?; + let _ = exe_path.pop(); + let mut prep_worker = exe_path.clone(); + prep_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + let mut exec_worker = exe_path.clone(); + exec_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + + if prep_worker.exists() && exec_worker.exists() { + log::trace!("Using current exe path as workers path: {:?}", exe_path); + Some(exe_path) + } else { + log::trace!("Workers path not found, considering `$PATH`"); + None + } + }; + let candidate_validation_config = CandidateValidationConfig { artifacts_cache_path: config .database .path() .ok_or(Error::DatabasePathRequired)? .join("pvf-artifacts"), - program_path, + workers_path, }; let chain_selection_config = ChainSelectionConfig { @@ -1351,6 +1369,7 @@ pub fn build_full( grandpa_pause: Option<(u32, u32)>, enable_beefy: bool, jaeger_agent: Option, + workers_path: Option, telemetry_worker_handle: Option, overseer_enable_anyways: bool, overseer_gen: impl OverseerGen, @@ -1370,7 +1389,7 @@ pub fn build_full( enable_beefy, jaeger_agent, telemetry_worker_handle, - None, + workers_path, overseer_enable_anyways, overseer_gen, overseer_message_channel_override, @@ -1389,7 +1408,7 @@ pub fn build_full( enable_beefy, jaeger_agent, telemetry_worker_handle, - None, + workers_path, overseer_enable_anyways, overseer_gen, overseer_message_channel_override, @@ -1408,7 +1427,7 @@ pub fn build_full( enable_beefy, jaeger_agent, telemetry_worker_handle, - None, + workers_path, overseer_enable_anyways, overseer_gen, overseer_message_channel_override, @@ -1427,7 +1446,7 @@ pub fn build_full( enable_beefy, jaeger_agent, telemetry_worker_handle, - None, + workers_path, overseer_enable_anyways, overseer_gen, overseer_message_channel_override.map(|capacity| { diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index 699cee202cb8..cc12e51f714b 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -63,6 +63,7 @@ fn main() -> Result<()> { false, None, None, + None, false, polkadot_service::RealOverseerGen, None, diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 189674b82a97..a301fef79f8d 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -63,6 +63,7 @@ fn main() -> Result<()> { false, None, None, + None, false, polkadot_service::RealOverseerGen, None, From d7a389cc951a449dcb9cd2287cddba63c8b23192 Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Sun, 11 Jun 2023 13:49:39 +0200 Subject: [PATCH 03/48] Reorganize once more, fix tests --- Cargo.lock | 10 ++ cli/src/cli.rs | 6 +- node/core/candidate-validation/src/lib.rs | 15 +-- .../dispute-coordinator/src/initialized.rs | 101 +++++++++--------- node/core/pvf/src/host.rs | 13 +-- node/core/pvf/src/lib.rs | 2 +- node/core/pvf/tests/it/main.rs | 3 +- node/service/Cargo.toml | 1 + node/service/src/lib.rs | 28 +++-- .../src/configuration/migration_ump.rs | 10 +- 10 files changed, 105 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88ceb81698a0..81932854a550 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3701,6 +3701,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "is_executable" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa9acdc6d67b75e626ad644734e8bc6df893d9cd2a834129065d3dd6158ea9c8" +dependencies = [ + "winapi", +] + [[package]] name = "itertools" version = "0.10.3" @@ -8201,6 +8210,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "futures", "hex-literal 0.4.1", + "is_executable", "kusama-runtime", "kusama-runtime-constants", "kvdb", diff --git a/cli/src/cli.rs b/cli/src/cli.rs index db2c4ff67a50..4d1589c626c4 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -16,8 +16,8 @@ //! Polkadot CLI library. -use std::path::PathBuf; use clap::Parser; +use std::path::PathBuf; #[allow(missing_docs)] #[derive(Debug, Parser)] @@ -150,8 +150,8 @@ pub struct RunCmd { #[arg(long)] pub overseer_channel_capacity_override: Option, - /// Path to auxiliary worker binaries. If not specified, the main binary's directory is - /// searched first, and then the `$PATH` is considered. + /// Path to directory where auxiliary worker binaries reside. If not specified, the main + /// binary's directory is searched first, and then the `$PATH` is considered. #[arg(long, value_name = "PATH")] pub workers_path: Option, } diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 9c68b004caa9..749bf410596e 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -93,9 +93,10 @@ const DEFAULT_APPROVAL_EXECUTION_TIMEOUT: Duration = Duration::from_secs(12); pub struct Config { /// The path where candidate validation can store compiled artifacts for PVFs. pub artifacts_cache_path: PathBuf, - /// The path to the directory with the executables which can be used for spawning PVF - /// compilation & validation workers. - pub workers_path: Option, + /// Path to the preparation worker binary + pub prep_worker_path: PathBuf, + /// Path to the execution worker binary + pub exec_worker_path: PathBuf, } /// The candidate validation subsystem. @@ -129,7 +130,8 @@ impl CandidateValidationSubsystem { self.metrics, self.pvf_metrics, self.config.artifacts_cache_path, - self.config.workers_path, + self.config.prep_worker_path, + self.config.exec_worker_path, ) .map_err(|e| SubsystemError::with_origin("candidate-validation", e)) .boxed(); @@ -143,10 +145,11 @@ async fn run( metrics: Metrics, pvf_metrics: polkadot_node_core_pvf::Metrics, cache_path: PathBuf, - workers_path: Option, + prep_worker_path: PathBuf, + exec_worker_path: PathBuf, ) -> SubsystemResult<()> { let (validation_host, task) = polkadot_node_core_pvf::start( - polkadot_node_core_pvf::Config::new(cache_path, workers_path), + polkadot_node_core_pvf::Config::new(cache_path, prep_worker_path, exec_worker_path), pvf_metrics, ); ctx.spawn_blocking("pvf-validation-host", task.boxed())?; diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index 25bbcde6ee7f..4498b522d079 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -217,61 +217,62 @@ impl Initialized { gum::trace!(target: LOG_TARGET, "Waiting for message"); let mut overlay_db = OverlayedBackend::new(backend); let default_confirm = Box::new(|| Ok(())); - let confirm_write = - match MuxedMessage::receive(ctx, &mut self.participation_receiver).await? { - MuxedMessage::Participation(msg) => { - gum::trace!(target: LOG_TARGET, "MuxedMessage::Participation"); - let ParticipationStatement { - session, + let confirm_write = match MuxedMessage::receive(ctx, &mut self.participation_receiver) + .await? + { + MuxedMessage::Participation(msg) => { + gum::trace!(target: LOG_TARGET, "MuxedMessage::Participation"); + let ParticipationStatement { + session, + candidate_hash, + candidate_receipt, + outcome, + } = self.participation.get_participation_result(ctx, msg).await?; + if let Some(valid) = outcome.validity() { + gum::trace!( + target: LOG_TARGET, + ?session, + ?candidate_hash, + ?valid, + "Issuing local statement based on participation outcome." + ); + self.issue_local_statement( + ctx, + &mut overlay_db, candidate_hash, candidate_receipt, - outcome, - } = self.participation.get_participation_result(ctx, msg).await?; - if let Some(valid) = outcome.validity() { - gum::trace!( - target: LOG_TARGET, - ?session, - ?candidate_hash, - ?valid, - "Issuing local statement based on participation outcome." - ); - self.issue_local_statement( - ctx, - &mut overlay_db, - candidate_hash, - candidate_receipt, - session, - valid, - clock.now(), - ) - .await?; - } else { - gum::warn!(target: LOG_TARGET, ?outcome, "Dispute participation failed"); - } + session, + valid, + clock.now(), + ) + .await?; + } else { + gum::warn!(target: LOG_TARGET, ?outcome, "Dispute participation failed"); + } + default_confirm + }, + MuxedMessage::Subsystem(msg) => match msg { + FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()), + FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => { + gum::trace!(target: LOG_TARGET, "OverseerSignal::ActiveLeaves"); + self.process_active_leaves_update( + ctx, + &mut overlay_db, + update, + clock.now(), + ) + .await?; default_confirm }, - MuxedMessage::Subsystem(msg) => match msg { - FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()), - FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => { - gum::trace!(target: LOG_TARGET, "OverseerSignal::ActiveLeaves"); - self.process_active_leaves_update( - ctx, - &mut overlay_db, - update, - clock.now(), - ) - .await?; - default_confirm - }, - FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, n)) => { - gum::trace!(target: LOG_TARGET, "OverseerSignal::BlockFinalized"); - self.scraper.process_finalized_block(&n); - default_confirm - }, - FromOrchestra::Communication { msg } => - self.handle_incoming(ctx, &mut overlay_db, msg, clock.now()).await?, + FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, n)) => { + gum::trace!(target: LOG_TARGET, "OverseerSignal::BlockFinalized"); + self.scraper.process_finalized_block(&n); + default_confirm }, - }; + FromOrchestra::Communication { msg } => + self.handle_incoming(ctx, &mut overlay_db, msg, clock.now()).await?, + }, + }; if !overlay_db.is_empty() { let ops = overlay_db.into_write_ops(); diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index 8bbe9f837f0c..ac35486773d5 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -168,14 +168,11 @@ pub struct Config { impl Config { /// Create a new instance of the configuration. - pub fn new(cache_path: std::path::PathBuf, workers_path: Option) -> Self { - // If the worker directory is given, get the binaries from that directory. Otherwise, expect - // that they are in $PATH. - let (prepare_worker_program_path, execute_worker_program_path) = match workers_path { - Some(p) => (p.join(PREPARE_BINARY_NAME), p.join(EXECUTE_BINARY_NAME)), - None => (PREPARE_BINARY_NAME.into(), EXECUTE_BINARY_NAME.into()), - }; - + pub fn new( + cache_path: PathBuf, + prepare_worker_program_path: PathBuf, + execute_worker_program_path: PathBuf, + ) -> Self { Self { cache_path, prepare_worker_program_path, diff --git a/node/core/pvf/src/lib.rs b/node/core/pvf/src/lib.rs index c9c7b8fe446f..772c0b430c1b 100644 --- a/node/core/pvf/src/lib.rs +++ b/node/core/pvf/src/lib.rs @@ -105,7 +105,7 @@ pub mod testing; pub use sp_tracing; pub use error::{InvalidCandidate, ValidationError}; -pub use host::{start, Config, ValidationHost, PREPARE_BINARY_NAME, EXECUTE_BINARY_NAME}; +pub use host::{start, Config, ValidationHost, EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME}; pub use metrics::Metrics; pub use priority::Priority; pub use worker_intf::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; diff --git a/node/core/pvf/tests/it/main.rs b/node/core/pvf/tests/it/main.rs index ef461a3531b5..0dc306e6bd53 100644 --- a/node/core/pvf/tests/it/main.rs +++ b/node/core/pvf/tests/it/main.rs @@ -48,7 +48,8 @@ impl TestHost { { let cache_dir = tempfile::tempdir().unwrap(); let program_path = std::path::PathBuf::from(PUPPET_EXE); - let mut config = Config::new(cache_dir.path().to_owned(), program_path); + let mut config = + Config::new(cache_dir.path().to_owned(), program_path.clone(), program_path); f(&mut config); let (host, task) = start(config, Metrics::default()); let _ = tokio::task::spawn(task); diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index 73a069b6dd34..aa4286fa3e0f 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -80,6 +80,7 @@ parity-db = { version = "0.4.8", optional = true } async-trait = "0.1.57" lru = "0.9" log = "0.4.17" +is_executable = "1.0.1" # Polkadot polkadot-node-core-parachains-inherent = { path = "../core/parachains-inherent" } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index c51edfc198ea..859faeeee2b7 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -36,6 +36,7 @@ mod tests; use { grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider}, gum::info, + is_executable::IsExecutable, polkadot_node_core_approval_voting::{ self as approval_voting_subsystem, Config as ApprovalVotingConfig, }, @@ -70,7 +71,7 @@ pub use { #[cfg(feature = "full-node")] use polkadot_node_subsystem::jaeger; -use std::{sync::Arc, time::Duration, path::PathBuf}; +use std::{path::PathBuf, sync::Arc, time::Duration}; use prometheus_endpoint::Registry; #[cfg(feature = "full-node")] @@ -905,9 +906,18 @@ where slot_duration_millis: slot_duration.as_millis() as u64, }; - let workers_path = if workers_path.is_some() { - log::trace!("Using explicitly provided workers path {:?}", workers_path); - workers_path + let (prep_worker_path, exec_worker_path) = if workers_path.is_some() { + let path = workers_path.expect("Workers path is checked to be provided"); + log::trace!("Using explicitly provided workers path {:?}", path); + if path.is_executable() { + (path.clone(), path) + } else { + let mut prep_worker = path.clone(); + let mut exec_worker = path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + (prep_worker, exec_worker) + } } else { let mut exe_path = std::env::current_exe()?; let _ = exe_path.pop(); @@ -918,10 +928,13 @@ where if prep_worker.exists() && exec_worker.exists() { log::trace!("Using current exe path as workers path: {:?}", exe_path); - Some(exe_path) + (prep_worker, exec_worker) } else { log::trace!("Workers path not found, considering `$PATH`"); - None + ( + PathBuf::from(polkadot_node_core_pvf::EXECUTE_BINARY_NAME), + PathBuf::from(polkadot_node_core_pvf::PREPARE_BINARY_NAME), + ) } }; @@ -931,7 +944,8 @@ where .path() .ok_or(Error::DatabasePathRequired)? .join("pvf-artifacts"), - workers_path, + prep_worker_path, + exec_worker_path, }; let chain_selection_config = ChainSelectionConfig { diff --git a/runtime/parachains/src/configuration/migration_ump.rs b/runtime/parachains/src/configuration/migration_ump.rs index c46f25108fa3..008a93142ee7 100644 --- a/runtime/parachains/src/configuration/migration_ump.rs +++ b/runtime/parachains/src/configuration/migration_ump.rs @@ -107,18 +107,12 @@ pub mod latest { "There must be exactly one new pending upgrade enqueued" ); if let Err(err) = last.1.check_consistency() { - log::error!( - target: LOG_TARGET, - "Last PendingConfig is invalidity {:?}", err, - ); + log::error!(target: LOG_TARGET, "Last PendingConfig is invalidity {:?}", err,); return Err("Pending upgrade must be sane but was not".into()) } if let Err(err) = ActiveConfig::::get().check_consistency() { - log::error!( - target: LOG_TARGET, - "ActiveConfig is invalid: {:?}", err, - ); + log::error!(target: LOG_TARGET, "ActiveConfig is invalid: {:?}", err,); return Err("Active upgrade must be sane but was not".into()) } From 818778ca7ce0a8bb366aec800757c9c98d4414c9 Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Sun, 11 Jun 2023 14:12:01 +0200 Subject: [PATCH 04/48] Reformat with new nightly to make `cargo fmt` test happy --- .../dispute-coordinator/src/initialized.rs | 101 +++++++++--------- 1 file changed, 50 insertions(+), 51 deletions(-) diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index 4498b522d079..25bbcde6ee7f 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -217,62 +217,61 @@ impl Initialized { gum::trace!(target: LOG_TARGET, "Waiting for message"); let mut overlay_db = OverlayedBackend::new(backend); let default_confirm = Box::new(|| Ok(())); - let confirm_write = match MuxedMessage::receive(ctx, &mut self.participation_receiver) - .await? - { - MuxedMessage::Participation(msg) => { - gum::trace!(target: LOG_TARGET, "MuxedMessage::Participation"); - let ParticipationStatement { - session, - candidate_hash, - candidate_receipt, - outcome, - } = self.participation.get_participation_result(ctx, msg).await?; - if let Some(valid) = outcome.validity() { - gum::trace!( - target: LOG_TARGET, - ?session, - ?candidate_hash, - ?valid, - "Issuing local statement based on participation outcome." - ); - self.issue_local_statement( - ctx, - &mut overlay_db, + let confirm_write = + match MuxedMessage::receive(ctx, &mut self.participation_receiver).await? { + MuxedMessage::Participation(msg) => { + gum::trace!(target: LOG_TARGET, "MuxedMessage::Participation"); + let ParticipationStatement { + session, candidate_hash, candidate_receipt, - session, - valid, - clock.now(), - ) - .await?; - } else { - gum::warn!(target: LOG_TARGET, ?outcome, "Dispute participation failed"); - } - default_confirm - }, - MuxedMessage::Subsystem(msg) => match msg { - FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()), - FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => { - gum::trace!(target: LOG_TARGET, "OverseerSignal::ActiveLeaves"); - self.process_active_leaves_update( - ctx, - &mut overlay_db, - update, - clock.now(), - ) - .await?; + outcome, + } = self.participation.get_participation_result(ctx, msg).await?; + if let Some(valid) = outcome.validity() { + gum::trace!( + target: LOG_TARGET, + ?session, + ?candidate_hash, + ?valid, + "Issuing local statement based on participation outcome." + ); + self.issue_local_statement( + ctx, + &mut overlay_db, + candidate_hash, + candidate_receipt, + session, + valid, + clock.now(), + ) + .await?; + } else { + gum::warn!(target: LOG_TARGET, ?outcome, "Dispute participation failed"); + } default_confirm }, - FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, n)) => { - gum::trace!(target: LOG_TARGET, "OverseerSignal::BlockFinalized"); - self.scraper.process_finalized_block(&n); - default_confirm + MuxedMessage::Subsystem(msg) => match msg { + FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()), + FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => { + gum::trace!(target: LOG_TARGET, "OverseerSignal::ActiveLeaves"); + self.process_active_leaves_update( + ctx, + &mut overlay_db, + update, + clock.now(), + ) + .await?; + default_confirm + }, + FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, n)) => { + gum::trace!(target: LOG_TARGET, "OverseerSignal::BlockFinalized"); + self.scraper.process_finalized_block(&n); + default_confirm + }, + FromOrchestra::Communication { msg } => + self.handle_incoming(ctx, &mut overlay_db, msg, clock.now()).await?, }, - FromOrchestra::Communication { msg } => - self.handle_incoming(ctx, &mut overlay_db, msg, clock.now()).await?, - }, - }; + }; if !overlay_db.is_empty() { let ops = overlay_db.into_write_ops(); From f9a9280a82d8b680e17a25160ceb9ce0eacf64db Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Sun, 11 Jun 2023 18:31:01 +0200 Subject: [PATCH 05/48] Address `clippy` warnings --- node/service/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 859faeeee2b7..cb18db617dcc 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -906,8 +906,7 @@ where slot_duration_millis: slot_duration.as_millis() as u64, }; - let (prep_worker_path, exec_worker_path) = if workers_path.is_some() { - let path = workers_path.expect("Workers path is checked to be provided"); + let (prep_worker_path, exec_worker_path) = if let Some(path) = workers_path { log::trace!("Using explicitly provided workers path {:?}", path); if path.is_executable() { (path.clone(), path) @@ -1323,7 +1322,7 @@ macro_rules! chain_ops { /// Builds a new object suitable for chain operations. #[cfg(feature = "full-node")] pub fn new_chain_ops( - mut config: &mut Configuration, + config: &mut Configuration, jaeger_agent: Option, ) -> Result< ( From b5733faee769d659aa10035ab34d26f067feac87 Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Wed, 14 Jun 2023 23:36:21 +0200 Subject: [PATCH 06/48] Add temporary trace to debug zombienet tests --- node/service/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index cb18db617dcc..2c4a15dcc508 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -919,11 +919,15 @@ where } } else { let mut exe_path = std::env::current_exe()?; + log::trace!("Exe path: {:?}", exe_path); let _ = exe_path.pop(); + log::trace!("Truncated exe path: {:?}", exe_path); let mut prep_worker = exe_path.clone(); - prep_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + log::trace!("Prep worker path: {:?}", prep_worker); let mut exec_worker = exe_path.clone(); - exec_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + log::trace!("Exec worker path: {:?}", exec_worker); if prep_worker.exists() && exec_worker.exists() { log::trace!("Using current exe path as workers path: {:?}", exe_path); From f5ba9650f565b4a7688e28cf1b95b6356fba167c Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Fri, 16 Jun 2023 13:26:19 +0200 Subject: [PATCH 07/48] Fix zombienet node upgrade test --- scripts/ci/gitlab/pipeline/zombienet.yml | 2 +- zombienet_tests/misc/0002-download-polkadot-from-pr.sh | 6 ++++-- zombienet_tests/misc/0002-upgrade-node.zndsl | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/ci/gitlab/pipeline/zombienet.yml b/scripts/ci/gitlab/pipeline/zombienet.yml index 5f51b06e2e78..cc4a7eb2ccc1 100644 --- a/scripts/ci/gitlab/pipeline/zombienet.yml +++ b/scripts/ci/gitlab/pipeline/zombienet.yml @@ -237,7 +237,7 @@ zombienet-tests-misc-upgrade-node: - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:latest" - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - BUILD_LINUX_JOB_ID="$(cat ./artifacts/BUILD_LINUX_JOB_ID)" - - export POLKADOT_PR_BIN_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts/polkadot" + - export POLKADOT_PR_ARTIFACTS_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts" script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" diff --git a/zombienet_tests/misc/0002-download-polkadot-from-pr.sh b/zombienet_tests/misc/0002-download-polkadot-from-pr.sh index 7ff323d9c41f..0d4b28075795 100644 --- a/zombienet_tests/misc/0002-download-polkadot-from-pr.sh +++ b/zombienet_tests/misc/0002-download-polkadot-from-pr.sh @@ -12,6 +12,8 @@ export PATH=$CFG_DIR:$PATH cd $CFG_DIR # see 0002-upgrade-node.zndsl to view the args. -curl -L -O $1 -chmod +x $CFG_DIR/polkadot +curl -L -O $1/polkadot +curl -L -O $1/polkadot-prepare-worker +curl -L -O $1/polkadot-execute-worker +chmod +x $CFG_DIR/polkadot $CFG_DIR/polkadot-prepare-worker $CFG_DIR/polkadot-execute-worker echo $(polkadot --version) diff --git a/zombienet_tests/misc/0002-upgrade-node.zndsl b/zombienet_tests/misc/0002-upgrade-node.zndsl index 7b23a2604989..fdf16b7286c9 100644 --- a/zombienet_tests/misc/0002-upgrade-node.zndsl +++ b/zombienet_tests/misc/0002-upgrade-node.zndsl @@ -7,12 +7,12 @@ dave: parachain 2001 block height is at least 10 within 200 seconds # upgrade both nodes # For testing using native provider you should set this env var -# POLKADOT_PR_BIN_URL=https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/1842869/artifacts/raw/artifacts/polkadot +# POLKADOT_PR_ARTIFACTS_URL=https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/1842869/artifacts/raw/artifacts # with the version of polkadot you want to download. # avg 30s in our infra -alice: run ./0002-download-polkadot-from-pr.sh with "{{POLKADOT_PR_BIN_URL}}" within 40 seconds -bob: run ./0002-download-polkadot-from-pr.sh with "{{POLKADOT_PR_BIN_URL}}" within 40 seconds +alice: run ./0002-download-polkadot-from-pr.sh with "{{POLKADOT_PR_ARTIFACTS_URL}}" within 40 seconds +bob: run ./0002-download-polkadot-from-pr.sh with "{{POLKADOT_PR_ARTIFACTS_URL}}" within 40 seconds alice: restart after 5 seconds bob: restart after 5 seconds From 7079e2fc43b418684f7bdd5974f009b2618ee509 Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Fri, 16 Jun 2023 15:20:11 +0200 Subject: [PATCH 08/48] Fix malus and its CI --- node/malus/src/malus.rs | 37 ------------------- .../ci/dockerfiles/malus_injected.Dockerfile | 2 +- scripts/ci/gitlab/pipeline/build.yml | 2 + 3 files changed, 3 insertions(+), 38 deletions(-) diff --git a/node/malus/src/malus.rs b/node/malus/src/malus.rs index d09f8be990a4..69dd7c869fc0 100644 --- a/node/malus/src/malus.rs +++ b/node/malus/src/malus.rs @@ -36,14 +36,6 @@ enum NemesisVariant { BackGarbageCandidate(BackGarbageCandidateOptions), /// Delayed disputing of ancestors that are perfectly fine. DisputeAncestor(DisputeAncestorOptions), - - #[allow(missing_docs)] - #[command(name = "prepare-worker", hide = true)] - PvfPrepareWorker(polkadot_cli::ValidationWorkerCommand), - - #[allow(missing_docs)] - #[command(name = "execute-worker", hide = true)] - PvfExecuteWorker(polkadot_cli::ValidationWorkerCommand), } #[derive(Debug, Parser)] @@ -88,35 +80,6 @@ impl MalusCli { finality_delay, )? }, - NemesisVariant::PvfPrepareWorker(cmd) => { - #[cfg(target_os = "android")] - { - return Err("PVF preparation workers are not supported under this platform") - .into() - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_prepare_worker::worker_entrypoint( - &cmd.socket_path, - None, - ); - } - }, - NemesisVariant::PvfExecuteWorker(cmd) => { - #[cfg(target_os = "android")] - { - return Err("PVF execution workers are not supported under this platform").into() - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_execute_worker::worker_entrypoint( - &cmd.socket_path, - None, - ); - } - }, } Ok(()) } diff --git a/scripts/ci/dockerfiles/malus_injected.Dockerfile b/scripts/ci/dockerfiles/malus_injected.Dockerfile index 3f7f1313b38f..e6aae8064509 100644 --- a/scripts/ci/dockerfiles/malus_injected.Dockerfile +++ b/scripts/ci/dockerfiles/malus_injected.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && \ # add adder-collator binary to docker image -COPY ./malus /usr/local/bin +COPY ./malus ./polkadot-*-worker /usr/local/bin USER nonroot diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index d07037626d65..b4e93c61cabb 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -83,6 +83,8 @@ build-malus: # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. 2>/dev/null || true + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. 2>/dev/null || true - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" From 95e643f73af29aa28096661c4035032a932ead5f Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Fri, 16 Jun 2023 15:52:25 +0200 Subject: [PATCH 09/48] Fix building worker binaries with malus --- Cargo.lock | 2 ++ node/malus/Cargo.toml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 26f765c339df..b63695825f29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8375,6 +8375,7 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", + "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", @@ -8386,6 +8387,7 @@ dependencies = [ "rand 0.8.5", "sp-core", "sp-keystore", + "sp-tracing", "tracing-gum", ] diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 8e23e623174f..1288f9be6b8c 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -12,6 +12,14 @@ publish = false name = "malus" path = "src/malus.rs" +[[bin]] +name = "polkadot-execute-worker" +path = "../core/pvf/execute-worker/src/main.rs" + +[[bin]] +name = "polkadot-prepare-worker" +path = "../core/pvf/prepare-worker/src/main.rs" + [dependencies] polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } polkadot-node-subsystem = { path = "../subsystem" } @@ -20,6 +28,7 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator" } polkadot-node-core-candidate-validation = { path = "../core/candidate-validation" } polkadot-node-core-backing = { path = "../core/backing" } +polkadot-node-core-pvf-common = { path = "../core/pvf/common" } polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } polkadot-node-primitives = { path = "../primitives" } @@ -29,6 +38,7 @@ assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } clap = { version = "4.0.9", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" From a1c093efc0b3cb87d20fe1a5eb226975f2aa6a7e Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Fri, 16 Jun 2023 17:24:41 +0200 Subject: [PATCH 10/48] More fixes for malus --- Cargo.lock | 4 ---- node/malus/Cargo.toml | 12 ------------ scripts/ci/gitlab/pipeline/build.yml | 6 +++--- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b63695825f29..1a4ba790d8a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8375,9 +8375,6 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -8387,7 +8384,6 @@ dependencies = [ "rand 0.8.5", "sp-core", "sp-keystore", - "sp-tracing", "tracing-gum", ] diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 1288f9be6b8c..53e80eb5bff5 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -12,14 +12,6 @@ publish = false name = "malus" path = "src/malus.rs" -[[bin]] -name = "polkadot-execute-worker" -path = "../core/pvf/execute-worker/src/main.rs" - -[[bin]] -name = "polkadot-prepare-worker" -path = "../core/pvf/prepare-worker/src/main.rs" - [dependencies] polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } polkadot-node-subsystem = { path = "../subsystem" } @@ -28,9 +20,6 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator" } polkadot-node-core-candidate-validation = { path = "../core/candidate-validation" } polkadot-node-core-backing = { path = "../core/backing" } -polkadot-node-core-pvf-common = { path = "../core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } color-eyre = { version = "0.6.1", default-features = false } @@ -38,7 +27,6 @@ assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } clap = { version = "4.0.9", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index b4e93c61cabb..c5c6b30fa398 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -79,12 +79,12 @@ build-malus: - .compiler-info - .collect-artifacts script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus + - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus -p polkadot-node-core-pvf-prepare-worker -p polkadot-node-core-pvf-execute-worker # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. - - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. 2>/dev/null || true - - mv ./target/testnet/polkadot-execute-worker ./artifacts/. 2>/dev/null || true + - mv ./target/testnet/polkadot-node-core-pvf-prepare-worker ./artifacts/polkadot-prepare-worker + - mv ./target/testnet/polkadot-node-core-pvf-execute-worker ./artifacts/polkadot-execute-worker - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" From 06de32dba4c34b051420518c5a2237c9a4d436a6 Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Sun, 18 Jun 2023 12:35:59 +0200 Subject: [PATCH 11/48] Remove unneeded cli subcommands --- cli/src/cli.rs | 8 -------- cli/src/command.rs | 44 -------------------------------------------- 2 files changed, 52 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 4d1589c626c4..0b7c940cc715 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -43,14 +43,6 @@ pub enum Subcommand { /// Revert the chain to a previous state. Revert(sc_cli::RevertCmd), - #[allow(missing_docs)] - #[command(name = "prepare-worker", hide = true)] - PvfPrepareWorker(ValidationWorkerCommand), - - #[allow(missing_docs)] - #[command(name = "execute-worker", hide = true)] - PvfExecuteWorker(ValidationWorkerCommand), - /// Sub-commands concerned with benchmarking. /// The pallet benchmarking moved to the `pallet` sub-command. #[command(subcommand)] diff --git a/cli/src/command.rs b/cli/src/command.rs index c37d1ad1f6f1..099bd58950ac 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -478,50 +478,6 @@ pub fn run() -> Result<()> { )) })?) }, - Some(Subcommand::PvfPrepareWorker(cmd)) => { - let mut builder = sc_cli::LoggerBuilder::new(""); - builder.with_colors(false); - let _ = builder.init(); - - #[cfg(target_os = "android")] - { - return Err(sc_cli::Error::Input( - "PVF preparation workers are not supported under this platform".into(), - ) - .into()) - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_prepare_worker::worker_entrypoint( - &cmd.socket_path, - Some(&cmd.node_impl_version), - ); - Ok(()) - } - }, - Some(Subcommand::PvfExecuteWorker(cmd)) => { - let mut builder = sc_cli::LoggerBuilder::new(""); - builder.with_colors(false); - let _ = builder.init(); - - #[cfg(target_os = "android")] - { - return Err(sc_cli::Error::Input( - "PVF execution workers are not supported under this platform".into(), - ) - .into()) - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_execute_worker::worker_entrypoint( - &cmd.socket_path, - Some(&cmd.node_impl_version), - ); - Ok(()) - } - }, Some(Subcommand::Benchmark(cmd)) => { let runner = cli.create_runner(cmd)?; let chain_spec = &runner.config().chain_spec; From d7b5a5a50f7c867bdcca5f03be6ceb680e652b3c Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Sun, 18 Jun 2023 13:50:25 +0200 Subject: [PATCH 12/48] Support placing auxiliary binaries to `/usr/libexec` --- Cargo.toml | 2 ++ cli/src/cli.rs | 5 ++++- node/service/src/lib.rs | 35 +++++++++++++++++++++-------------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d4b18667e4a5..8408679fd190 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -236,6 +236,8 @@ license-file = ["LICENSE", "0"] maintainer-scripts = "scripts/packaging/deb-maintainer-scripts" assets = [ ["target/release/polkadot", "/usr/bin/", "755"], + ["target/release/polkadot-prepare-worker", "/usr/libexec/", "755"], + ["target/release/polkadot-execute-worker", "/usr/libexec/", "755"], ["scripts/packaging/polkadot.service", "/lib/systemd/system/", "644"] ] conf-files = [ diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 0b7c940cc715..5c3195fac6af 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -143,7 +143,10 @@ pub struct RunCmd { pub overseer_channel_capacity_override: Option, /// Path to directory where auxiliary worker binaries reside. If not specified, the main - /// binary's directory is searched first, and then the `$PATH` is considered. + /// binary's directory is searched first, then /usr/libexec is searched, and then the + /// `$PATH` is considered. + /// If the path points to an executable rather then directory, that executable is used + /// both as preparation and execution worker (that supposed to be used for tests only). #[arg(long, value_name = "PATH")] pub workers_path: Option, } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 2c4a15dcc508..00f534a7fe3f 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -918,26 +918,33 @@ where (prep_worker, exec_worker) } } else { - let mut exe_path = std::env::current_exe()?; - log::trace!("Exe path: {:?}", exe_path); - let _ = exe_path.pop(); - log::trace!("Truncated exe path: {:?}", exe_path); - let mut prep_worker = exe_path.clone(); + let libexec = PathBuf::from("/usr/libexec"); + let mut prep_worker = libexec.clone(); prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - log::trace!("Prep worker path: {:?}", prep_worker); - let mut exec_worker = exe_path.clone(); + let mut exec_worker = libexec.clone(); exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - log::trace!("Exec worker path: {:?}", exec_worker); if prep_worker.exists() && exec_worker.exists() { - log::trace!("Using current exe path as workers path: {:?}", exe_path); + log::trace!("Using /usr/libexec as workers path"); (prep_worker, exec_worker) } else { - log::trace!("Workers path not found, considering `$PATH`"); - ( - PathBuf::from(polkadot_node_core_pvf::EXECUTE_BINARY_NAME), - PathBuf::from(polkadot_node_core_pvf::PREPARE_BINARY_NAME), - ) + let mut exe_path = std::env::current_exe()?; + let _ = exe_path.pop(); + let mut prep_worker = exe_path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + let mut exec_worker = exe_path.clone(); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + if prep_worker.exists() && exec_worker.exists() { + log::trace!("Using current exe path as workers path: {:?}", exe_path); + (prep_worker, exec_worker) + } else { + log::trace!("Workers path not found, considering `$PATH`"); + ( + PathBuf::from(polkadot_node_core_pvf::EXECUTE_BINARY_NAME), + PathBuf::from(polkadot_node_core_pvf::PREPARE_BINARY_NAME), + ) + } } }; From 49475c2314831208c881d154fced369e6f46721d Mon Sep 17 00:00:00 2001 From: Dmitry Sinyavin Date: Sun, 18 Jun 2023 15:17:24 +0200 Subject: [PATCH 13/48] Fix spelling --- cli/src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 5c3195fac6af..924e5d152266 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -143,7 +143,7 @@ pub struct RunCmd { pub overseer_channel_capacity_override: Option, /// Path to directory where auxiliary worker binaries reside. If not specified, the main - /// binary's directory is searched first, then /usr/libexec is searched, and then the + /// binary's directory is searched first, then `/usr/libexec` is searched, and then the /// `$PATH` is considered. /// If the path points to an executable rather then directory, that executable is used /// both as preparation and execution worker (that supposed to be used for tests only). From 4c72864def689b575958335f09569f3fef380677 Mon Sep 17 00:00:00 2001 From: s0me0ne-unkn0wn <48632512+s0me0ne-unkn0wn@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:52:36 +0200 Subject: [PATCH 14/48] Spelling Co-authored-by: Marcin S. --- cli/src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 924e5d152266..ef074645b07b 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -146,7 +146,7 @@ pub struct RunCmd { /// binary's directory is searched first, then `/usr/libexec` is searched, and then the /// `$PATH` is considered. /// If the path points to an executable rather then directory, that executable is used - /// both as preparation and execution worker (that supposed to be used for tests only). + /// both as preparation and execution worker (supposed to be used for tests only). #[arg(long, value_name = "PATH")] pub workers_path: Option, } From 07d876e417e4791f545620e175967ea4e409755b Mon Sep 17 00:00:00 2001 From: Marcin S Date: Sun, 9 Jul 2023 13:51:40 +0200 Subject: [PATCH 15/48] Implement review comments (mostly nits) --- Cargo.lock | 4 ++++ Cargo.toml | 8 +++++--- cli/src/cli.rs | 9 ++++----- node/malus/Cargo.toml | 16 ++++++++++++++++ node/service/src/lib.rs | 9 +++++---- scripts/ci/dockerfiles/malus_injected.Dockerfile | 2 +- .../polkadot_injected_debug.Dockerfile | 2 +- scripts/ci/gitlab/pipeline/build.yml | 10 +++++----- 8 files changed, 41 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bd42316db13..c37ad1ce4173 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8375,6 +8375,9 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", + "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -8384,6 +8387,7 @@ dependencies = [ "rand 0.8.5", "sp-core", "sp-keystore", + "sp-tracing", "tracing-gum", ] diff --git a/Cargo.toml b/Cargo.toml index 8408679fd190..f80ec674df8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,13 +29,15 @@ version = "0.9.43" [dependencies] color-eyre = { version = "0.6.1", default-features = false } tikv-jemallocator = "0.5.0" -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } -# Crates in our workspace, defined as dependencies so we can pass them feature flags. -polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ] } +# Needed for worker binaries. polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + +# Crates in our workspace, defined as dependencies so we can pass them feature flags. +polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ] } polkadot-overseer = { path = "node/overseer" } [dev-dependencies] diff --git a/cli/src/cli.rs b/cli/src/cli.rs index ef074645b07b..e82d53cab14b 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -142,11 +142,10 @@ pub struct RunCmd { #[arg(long)] pub overseer_channel_capacity_override: Option, - /// Path to directory where auxiliary worker binaries reside. If not specified, the main - /// binary's directory is searched first, then `/usr/libexec` is searched, and then the - /// `$PATH` is considered. - /// If the path points to an executable rather then directory, that executable is used - /// both as preparation and execution worker (supposed to be used for tests only). + /// Path to the directory where auxiliary worker binaries reside. If not specified, the main + /// binary's directory is searched first, then `/usr/libexec` is searched, and then the `$PATH` + /// is considered. If the path points to an executable rather then directory, that executable is + /// used both as preparation and execution worker (supposed to be used for tests only). #[arg(long, value_name = "PATH")] pub workers_path: Option, } diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 53e80eb5bff5..26fe52e35491 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -12,6 +12,18 @@ publish = false name = "malus" path = "src/malus.rs" +[[bin]] +name = "polkadot-execute-worker" +path = "../core/pvf/execute-worker/src/main.rs" +# Prevent rustdoc error. Already documented from top-level Cargo.toml. +doc = false + +[[bin]] +name = "polkadot-prepare-worker" +path = "../core/pvf/prepare-worker/src/main.rs" +# Prevent rustdoc error. Already documented from top-level Cargo.toml. +doc = false + [dependencies] polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } polkadot-node-subsystem = { path = "../subsystem" } @@ -20,6 +32,9 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator" } polkadot-node-core-candidate-validation = { path = "../core/candidate-validation" } polkadot-node-core-backing = { path = "../core/backing" } +polkadot-node-core-pvf-common = { path = "../core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } color-eyre = { version = "0.6.1", default-features = false } @@ -27,6 +42,7 @@ assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } clap = { version = "4.0.9", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 00f534a7fe3f..853636218ea4 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -687,10 +687,11 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { /// regardless of the role the node has. The relay chain selection (longest or disputes-aware) is /// still determined based on the role of the node. Likewise for authority discovery. /// -/// `workers_path` is used to get the PVF worker locations. First, `workers_path` is considered, -/// and if it is passed then that location will be used for both the binaries. If it is not passed, -/// current exe's directory is considered and checked for pinaries presence. If not found, the -/// binaries will be taken from `$PATH`. +/// `workers_path` is used to get the path to the directory where auxiliary worker binaries reside. +/// If not specified, the main binary's directory is searched first, then `/usr/libexec` is +/// searched, and then the `$PATH` is considered. If the path points to an executable rather then +/// directory, that executable is used both as preparation and execution worker (supposed to be used +/// for tests only). #[cfg(feature = "full-node")] pub fn new_full( mut config: Configuration, diff --git a/scripts/ci/dockerfiles/malus_injected.Dockerfile b/scripts/ci/dockerfiles/malus_injected.Dockerfile index e6aae8064509..7ab3d74e3b83 100644 --- a/scripts/ci/dockerfiles/malus_injected.Dockerfile +++ b/scripts/ci/dockerfiles/malus_injected.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && \ # add adder-collator binary to docker image -COPY ./malus ./polkadot-*-worker /usr/local/bin +COPY ./malus ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin USER nonroot diff --git a/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile b/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile index 128b802b7adc..ef3ebd14ae60 100644 --- a/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +++ b/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && \ ln -s /data /polkadot/.local/share/polkadot # add polkadot binary to docker image -COPY ./polkadot ./polkadot-*-worker /usr/local/bin +COPY ./polkadot ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin USER polkadot diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index c5c6b30fa398..a7f942c79a85 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -26,8 +26,8 @@ build-linux-stable: - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - mv ./target/testnet/polkadot ./artifacts/. - - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. 2>/dev/null || true - - mv ./target/testnet/polkadot-execute-worker ./artifacts/. 2>/dev/null || true + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - pushd artifacts - sha256sum polkadot | tee polkadot.sha256 - shasum -c polkadot.sha256 @@ -79,12 +79,12 @@ build-malus: - .compiler-info - .collect-artifacts script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus -p polkadot-node-core-pvf-prepare-worker -p polkadot-node-core-pvf-execute-worker + - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. - - mv ./target/testnet/polkadot-node-core-pvf-prepare-worker ./artifacts/polkadot-prepare-worker - - mv ./target/testnet/polkadot-node-core-pvf-execute-worker ./artifacts/polkadot-execute-worker + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" From 126213e4568540aa8645dcc36387d37559a5631f Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 10 Jul 2023 14:08:48 +0200 Subject: [PATCH 16/48] Fix worker node version flag --- node/core/pvf/common/src/worker/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index 458dd3157e2a..dd7deb2a3da1 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -48,8 +48,8 @@ macro_rules! decl_worker_main { for i in 2..args.len() { match args[i].as_ref() { "--socket-path" => socket_path = args[i + 1].as_str(), - "--node-version" => version = Some(args[i + 1].as_str()), - _ => (), + "--node-impl-version" => version = Some(args[i + 1].as_str()), + arg => panic!("Unexpected argument found: {}", arg), } } From 6640b29e8fddd9a4ab5ee730c463ce79a9648abf Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 10 Jul 2023 17:05:21 +0200 Subject: [PATCH 17/48] Rework getting the worker paths --- Cargo.lock | 1 + cli/src/cli.rs | 17 +-- node/core/pvf/common/src/worker/mod.rs | 59 ++++++--- node/core/pvf/src/testing.rs | 12 +- node/service/Cargo.toml | 3 + node/service/build.rs | 19 +++ node/service/src/lib.rs | 171 ++++++++++++++++++------- 7 files changed, 203 insertions(+), 79 deletions(-) create mode 100644 node/service/build.rs diff --git a/Cargo.lock b/Cargo.lock index c3a656eac481..1a4fa0c9313f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8188,6 +8188,7 @@ dependencies = [ "sp-trie", "sp-version", "sp-weights", + "substrate-build-script-utils", "substrate-prometheus-endpoint", "tempfile", "thiserror", diff --git a/cli/src/cli.rs b/cli/src/cli.rs index e82d53cab14b..e398577c3528 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -68,17 +68,6 @@ pub enum Subcommand { ChainInfo(sc_cli::ChainInfoCmd), } -#[allow(missing_docs)] -#[derive(Debug, Parser)] -pub struct ValidationWorkerCommand { - /// The path to the validation host's socket. - #[arg(long)] - pub socket_path: String, - /// Calling node implementation version - #[arg(long)] - pub node_impl_version: String, -} - #[allow(missing_docs)] #[derive(Debug, Parser)] #[group(skip)] @@ -143,9 +132,9 @@ pub struct RunCmd { pub overseer_channel_capacity_override: Option, /// Path to the directory where auxiliary worker binaries reside. If not specified, the main - /// binary's directory is searched first, then `/usr/libexec` is searched, and then the `$PATH` - /// is considered. If the path points to an executable rather then directory, that executable is - /// used both as preparation and execution worker (supposed to be used for tests only). + /// binary's directory is searched first, then `/usr/libexec` is searched. If the path points to + /// an executable rather then directory, that executable is used both as preparation and + /// execution worker (supposed to be used for tests only). #[arg(long, value_name = "PATH")] pub workers_path: Option, } diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index dd7deb2a3da1..659ae01e8c73 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -34,37 +34,63 @@ use tokio::{io, net::UnixStream, runtime::Runtime}; #[macro_export] macro_rules! decl_worker_main { ($expected_command:expr, $entrypoint:expr) => { + fn print_help(expected_command: &str) { + let worker_version = $crate::worker::worker_version(); + + println!("{} {}", expected_command, worker_version); + println!(); + println!("PVF worker that is called by polkadot."); + } + fn main() { ::sp_tracing::try_init_simple(); let args = std::env::args().collect::>(); - if args.len() < 3 { - panic!("wrong number of arguments"); + if args.len() == 1 { + print_help($expected_command); + return + } + + match args[1].as_ref() { + "--help" => { + print_help($expected_command); + return + }, + "--version" => { + println!("{}", $crate::worker::worker_version()); + return + }, + subcommand => { + // Must be passed for compatibility with the single-binary test workers. + if subcommand != $expected_command { + panic!( + "trying to run {} binary with the {} subcommand", + $expected_command, subcommand + ) + } + }, } - let mut version = None; + let mut node_version = None; let mut socket_path: &str = ""; - for i in 2..args.len() { + for i in (2..args.len()).step_by(2) { match args[i].as_ref() { "--socket-path" => socket_path = args[i + 1].as_str(), - "--node-impl-version" => version = Some(args[i + 1].as_str()), + "--node-impl-version" => node_version = Some(args[i + 1].as_str()), arg => panic!("Unexpected argument found: {}", arg), } } - let subcommand = &args[1]; - if subcommand != $expected_command { - panic!( - "trying to run {} binary with the {} subcommand", - $expected_command, subcommand - ) - } - $entrypoint(&socket_path, version); + $entrypoint(&socket_path, node_version); } }; } +pub fn worker_version() -> &'static str { + env!("SUBSTRATE_CLI_IMPL_VERSION") +} + /// Some allowed overhead that we account for in the "CPU time monitor" thread's sleeps, on the /// child process. pub const JOB_TIMEOUT_OVERHEAD: Duration = Duration::from_millis(50); @@ -88,11 +114,14 @@ pub fn worker_event_loop( gum::debug!(target: LOG_TARGET, %worker_pid, "starting pvf worker ({})", debug_id); // Check for a mismatch between the node and worker versions. - if let Some(version) = node_version { - if version != env!("SUBSTRATE_CLI_IMPL_VERSION") { + if let Some(node_version) = node_version { + let worker_version = env!("SUBSTRATE_CLI_IMPL_VERSION"); + if node_version != worker_version { gum::error!( target: LOG_TARGET, %worker_pid, + %node_version, + %worker_version, "Node and worker version mismatch, node needs restarting, forcing shutdown", ); kill_parent_node_in_emergency(); diff --git a/node/core/pvf/src/testing.rs b/node/core/pvf/src/testing.rs index cc07d7aeef02..fc7807958cfb 100644 --- a/node/core/pvf/src/testing.rs +++ b/node/core/pvf/src/testing.rs @@ -62,14 +62,14 @@ macro_rules! decl_puppet_worker_main { panic!("wrong number of arguments"); } - let mut version = None; + let mut node_version = None; let mut socket_path: &str = ""; - for i in 2..args.len() { + for i in (2..args.len()).step_by(2) { match args[i].as_ref() { "--socket-path" => socket_path = args[i + 1].as_str(), - "--node-version" => version = Some(args[i + 1].as_str()), - _ => (), + "--node-impl-version" => node_version = Some(args[i + 1].as_str()), + arg => panic!("Unexpected argument found: {}", arg), } } @@ -82,10 +82,10 @@ macro_rules! decl_puppet_worker_main { std::thread::sleep(std::time::Duration::from_secs(5)); }, "prepare-worker" => { - $crate::prepare_worker_entrypoint(&socket_path, version); + $crate::prepare_worker_entrypoint(&socket_path, node_version); }, "execute-worker" => { - $crate::execute_worker_entrypoint(&socket_path, version); + $crate::execute_worker_entrypoint(&socket_path, node_version); }, other => panic!("unknown subcommand: {}", other), } diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index d06408a7ec2b..448934ae665f 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -147,6 +147,9 @@ env_logger = "0.9.0" assert_matches = "1.5.0" tempfile = "3.2" +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + [features] default = ["db", "full-node"] diff --git a/node/service/build.rs b/node/service/build.rs new file mode 100644 index 000000000000..40e9f832586e --- /dev/null +++ b/node/service/build.rs @@ -0,0 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); +} diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index ff2f54a883b0..55ebf3c41f34 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -55,6 +55,7 @@ use { sc_client_api::BlockBackend, sp_core::traits::SpawnNamed, sp_trie::PrefixedMemoryDB, + std::process::Command, }; use polkadot_node_subsystem_util::database::Database; @@ -233,6 +234,18 @@ pub enum Error { #[cfg(feature = "full-node")] #[error("Expected at least one of polkadot, kusama, westend or rococo runtime feature")] NoRuntime, + + #[cfg(feature = "full-node")] + #[error("Worker binaries not executable, prepare binary: {0:?}, execute binary: {1:?}")] + InvalidWorkerBinaries(PathBuf, PathBuf), + + #[cfg(feature = "full-node")] + #[error("Worker binaries could not be found at workers path ({0:?}), polkadot binary directory, or /usr/libexec")] + MissingWorkerBinaries(Option), + + #[cfg(feature = "full-node")] + #[error("Version of worker binaries ({0}) is different from node version ({1})")] + WorkerBinaryVersionMismatch(String, String), } /// Identifies the variant of the chain. @@ -655,9 +668,8 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { /// /// `workers_path` is used to get the path to the directory where auxiliary worker binaries reside. /// If not specified, the main binary's directory is searched first, then `/usr/libexec` is -/// searched, and then the `$PATH` is considered. If the path points to an executable rather then -/// directory, that executable is used both as preparation and execution worker (supposed to be used -/// for tests only). +/// searched. If the path points to an executable rather then directory, that executable is used +/// both as preparation and execution worker (supposed to be used for tests only). #[cfg(feature = "full-node")] pub fn new_full( mut config: Configuration, @@ -858,47 +870,47 @@ where slot_duration_millis: slot_duration.as_millis() as u64, }; - let (prep_worker_path, exec_worker_path) = if let Some(path) = workers_path { - log::trace!("Using explicitly provided workers path {:?}", path); - if path.is_executable() { - (path.clone(), path) - } else { - let mut prep_worker = path.clone(); - let mut exec_worker = path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - (prep_worker, exec_worker) - } - } else { - let libexec = PathBuf::from("/usr/libexec"); - let mut prep_worker = libexec.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = libexec.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - - if prep_worker.exists() && exec_worker.exists() { - log::trace!("Using /usr/libexec as workers path"); - (prep_worker, exec_worker) - } else { - let mut exe_path = std::env::current_exe()?; - let _ = exe_path.pop(); - let mut prep_worker = exe_path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = exe_path.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - - if prep_worker.exists() && exec_worker.exists() { - log::trace!("Using current exe path as workers path: {:?}", exe_path); - (prep_worker, exec_worker) - } else { - log::trace!("Workers path not found, considering `$PATH`"); - ( - PathBuf::from(polkadot_node_core_pvf::EXECUTE_BINARY_NAME), - PathBuf::from(polkadot_node_core_pvf::PREPARE_BINARY_NAME), - ) - } - } - }; + // TODO: Do some manual tests for this. + // 1. Get the binaries from the workers path if it is passed in, or consider all possible + // locations on the filesystem in order and get all sets of paths at which the binaries exist. + // + // 2. If no paths exist, error out. We can't proceed without workers. + // + // 3. Log a warning if more than one set of paths exists. + // + // 4. Check if the returned paths are executable. If not it's evidence of a borked installation + // so error out. + // + // 5. Do the version check, if mismatch error out. + // + // 6. At this point the paths should be good to use. + let mut workers_paths = get_workers_paths(workers_path.clone())?; + if workers_paths.is_empty() { + return Err(Error::MissingWorkerBinaries(workers_path)) + } else if workers_paths.len() > 1 { + log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); + } + let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); + if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { + return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) + } + // Do the version check. + let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); + let prep_worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; + let prep_worker_version = + String::from_utf8(prep_worker_version).expect("version is printed as a string; qed"); + if prep_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) + } + let exec_worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; + let exec_worker_version = + String::from_utf8(exec_worker_version).expect("version is printed as a string; qed"); + if exec_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) + } + // Paths are good to use. + log::info!("using prepare-worker binary at: {:?}", prep_worker_path); + log::info!("using execute-worker binary at: {:?}", exec_worker_path); let candidate_validation_config = CandidateValidationConfig { artifacts_cache_path: config @@ -1256,6 +1268,77 @@ where }) } +/// Get the workers path by considering the passed-in `workers_path` option, or possible locations +/// on the filesystem. See `new_full`. +fn get_workers_paths( + given_workers_path: Option, +) -> Result, Error> { + if let Some(path) = given_workers_path { + log::trace!("Using explicitly provided workers path {:?}", path); + + if path.is_executable() { + return Ok(vec![(path.clone(), path)]) + } + + let mut prep_worker = path.clone(); + let mut exec_worker = path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Check if both workers exist. Otherwise return an empty vector which results in an error. + return if prep_worker.exists() && exec_worker.exists() { + Ok(vec![(prep_worker, exec_worker)]) + } else { + Ok(vec![]) + } + } + + // Workers path not provided, check all possible valid locations. + + let mut workers_paths = vec![]; + + { + let mut exe_path = std::env::current_exe()?; + let _ = exe_path.pop(); // executable file will always have a parent directory. + let mut prep_worker = exe_path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + let mut exec_worker = exe_path.clone(); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Add to set if both workers exist. Warn on partial installs. + let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); + if prep_worker_exists && exec_worker_exists { + log::trace!("Worker binaries found at current exe path: {:?}", exe_path); + workers_paths.push((prep_worker, exec_worker)); + } else if prep_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); + } else if exec_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + } + } + + { + let libexec = PathBuf::from("/usr/libexec"); + let mut prep_worker = libexec.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + let mut exec_worker = libexec.clone(); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Add to set if both workers exist. Warn on partial installs. + let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); + if prep_worker_exists && exec_worker_exists { + log::trace!("Worker binaries found at /usr/libexec"); + workers_paths.push((prep_worker, exec_worker)); + } else if prep_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); + } else if exec_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + } + } + + Ok(workers_paths) +} + #[cfg(feature = "full-node")] macro_rules! chain_ops { ($config:expr, $jaeger_agent:expr, $telemetry_worker_handle:expr) => {{ From 633edbc30a53f660c72e27db904b4f629bb2d711 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 11 Jul 2023 09:33:14 +0200 Subject: [PATCH 18/48] Address a couple of review comments --- Cargo.toml | 4 +- cli/src/cli.rs | 6 +-- cli/src/command.rs | 26 ++++++----- node/service/src/lib.rs | 96 ++++++++++++++++++++--------------------- 4 files changed, 66 insertions(+), 66 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 159516afd133..f63288499b80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -239,8 +239,8 @@ license-file = ["LICENSE", "0"] maintainer-scripts = "scripts/packaging/deb-maintainer-scripts" assets = [ ["target/release/polkadot", "/usr/bin/", "755"], - ["target/release/polkadot-prepare-worker", "/usr/libexec/", "755"], - ["target/release/polkadot-execute-worker", "/usr/libexec/", "755"], + ["target/release/polkadot-prepare-worker", "/usr/lib/polkadot/", "755"], + ["target/release/polkadot-execute-worker", "/usr/lib/polkadot/", "755"], ["scripts/packaging/polkadot.service", "/lib/systemd/system/", "644"] ] conf-files = [ diff --git a/cli/src/cli.rs b/cli/src/cli.rs index e398577c3528..78f50d38f60f 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -132,9 +132,9 @@ pub struct RunCmd { pub overseer_channel_capacity_override: Option, /// Path to the directory where auxiliary worker binaries reside. If not specified, the main - /// binary's directory is searched first, then `/usr/libexec` is searched. If the path points to - /// an executable rather then directory, that executable is used both as preparation and - /// execution worker (supposed to be used for tests only). + /// binary's directory is searched first, then `/usr/lib/polkadot` is searched. If the path + /// points to an executable rather then directory, that executable is used both as preparation + /// and execution worker (supposed to be used for tests only). #[arg(long, value_name = "PATH")] pub workers_path: Option, } diff --git a/cli/src/command.rs b/cli/src/command.rs index eb7b8d2666aa..65d3f4ed4743 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -285,17 +285,21 @@ where let database_source = config.database.clone(); let task_manager = service::build_full( config, - service::IsCollator::No, - grandpa_pause, - enable_beefy, - jaeger_agent, - cli.run.workers_path, - None, - false, - overseer_gen, - cli.run.overseer_channel_capacity_override, - maybe_malus_finality_delay, - hwbench, + service::NewFullParams { + is_collator: service::IsCollator::No, + grandpa_pause, + enable_beefy, + jaeger_agent, + telemetry_worker_handle: None, + workers_path: cli.run.workers_path, + overseer_enable_anyways: false, + overseer_gen, + overseer_message_channel_capacity_override: cli + .run + .overseer_channel_capacity_override, + malus_finality_delay: maybe_malus_finality_delay, + hwbench, + }, ) .map(|full| full.task_manager)?; diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 55ebf3c41f34..634e5a9b72af 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -240,7 +240,7 @@ pub enum Error { InvalidWorkerBinaries(PathBuf, PathBuf), #[cfg(feature = "full-node")] - #[error("Worker binaries could not be found at workers path ({0:?}), polkadot binary directory, or /usr/libexec")] + #[error("Worker binaries could not be found at workers path ({0:?}), polkadot binary directory, or /usr/lib/polkadot")] MissingWorkerBinaries(Option), #[cfg(feature = "full-node")] @@ -611,6 +611,22 @@ where }) } +#[cfg(feature = "full-node")] +pub struct NewFullParams { + pub is_collator: IsCollator, + pub grandpa_pause: Option<(u32, u32)>, + pub enable_beefy: bool, + pub jaeger_agent: Option, + pub telemetry_worker_handle: Option, + pub workers_path: Option, + pub overseer_enable_anyways: bool, + pub overseer_gen: OverseerGenerator, + pub overseer_message_channel_capacity_override: Option, + #[allow(dead_code)] + pub malus_finality_delay: Option, + pub hwbench: Option, +} + #[cfg(feature = "full-node")] pub struct NewFull { pub task_manager: TaskManager, @@ -656,7 +672,6 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { col_meta: parachains_db::REAL_COLUMNS.col_availability_meta, }; -// TODO: This function seems to be public? How to not break backwards compatibility? /// Create a new full node of arbitrary runtime and executor. /// /// This is an advanced feature and not recommended for general use. Generally, `build_full` is @@ -667,30 +682,31 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { /// still determined based on the role of the node. Likewise for authority discovery. /// /// `workers_path` is used to get the path to the directory where auxiliary worker binaries reside. -/// If not specified, the main binary's directory is searched first, then `/usr/libexec` is +/// If not specified, the main binary's directory is searched first, then `/usr/lib/polkadot` is /// searched. If the path points to an executable rather then directory, that executable is used /// both as preparation and execution worker (supposed to be used for tests only). #[cfg(feature = "full-node")] -pub fn new_full( +pub fn new_full( mut config: Configuration, - is_collator: IsCollator, - grandpa_pause: Option<(u32, u32)>, - enable_beefy: bool, - jaeger_agent: Option, - telemetry_worker_handle: Option, - workers_path: Option, - overseer_enable_anyways: bool, - overseer_gen: OverseerGenerator, - overseer_message_channel_capacity_override: Option, - _malus_finality_delay: Option, - hwbench: Option, -) -> Result -where - OverseerGenerator: OverseerGen, -{ + params: NewFullParams, +) -> Result { use polkadot_node_network_protocol::request_response::IncomingRequest; use sc_network_common::sync::warp::WarpSyncParams; + let NewFullParams { + is_collator, + grandpa_pause, + enable_beefy, + jaeger_agent, + telemetry_worker_handle, + workers_path, + overseer_enable_anyways, + overseer_gen, + overseer_message_channel_capacity_override, + malus_finality_delay: _, + hwbench, + } = params; + let is_offchain_indexing_enabled = config.offchain_worker.indexing_enabled; let role = config.role.clone(); let force_authoring = config.force_authoring; @@ -1318,16 +1334,16 @@ fn get_workers_paths( } { - let libexec = PathBuf::from("/usr/libexec"); - let mut prep_worker = libexec.clone(); + let lib_path = PathBuf::from("/usr/lib/polkadot"); + let mut prep_worker = lib_path.clone(); prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = libexec.clone(); + let mut exec_worker = lib_path.clone(); exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); // Add to set if both workers exist. Warn on partial installs. let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); if prep_worker_exists && exec_worker_exists { - log::trace!("Worker binaries found at /usr/libexec"); + log::trace!("Worker binaries found at /usr/lib/polkadot"); workers_paths.push((prep_worker, exec_worker)); } else if prep_worker_exists { log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); @@ -1396,41 +1412,21 @@ pub fn new_chain_ops( /// regardless of the role the node has. The relay chain selection (longest or disputes-aware) is /// still determined based on the role of the node. Likewise for authority discovery. #[cfg(feature = "full-node")] -pub fn build_full( +pub fn build_full( config: Configuration, - is_collator: IsCollator, - grandpa_pause: Option<(u32, u32)>, - enable_beefy: bool, - jaeger_agent: Option, - workers_path: Option, - telemetry_worker_handle: Option, - overseer_enable_anyways: bool, - overseer_gen: impl OverseerGen, - overseer_message_channel_override: Option, - malus_finality_delay: Option, - hwbench: Option, + mut params: NewFullParams, ) -> Result { let is_polkadot = config.chain_spec.is_polkadot(); - new_full( - config, - is_collator, - grandpa_pause, - enable_beefy, - jaeger_agent, - telemetry_worker_handle, - workers_path, - overseer_enable_anyways, - overseer_gen, - overseer_message_channel_override.map(move |capacity| { + params.overseer_message_channel_capacity_override = + params.overseer_message_channel_capacity_override.map(move |capacity| { if is_polkadot { gum::warn!("Channel capacity should _never_ be tampered with on polkadot!"); } capacity - }), - malus_finality_delay, - hwbench, - ) + }); + + new_full(config, params) } /// Reverts the node state down to at most the last finalized block. From 41b0a49204108bfee0ec1aa2a2fefec0f9305885 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 11 Jul 2023 16:08:05 +0200 Subject: [PATCH 19/48] Minor restructuring --- node/service/src/lib.rs | 113 ++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 52 deletions(-) diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 634e5a9b72af..349d7890a631 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -688,12 +688,7 @@ pub const AVAILABILITY_CONFIG: AvailabilityConfig = AvailabilityConfig { #[cfg(feature = "full-node")] pub fn new_full( mut config: Configuration, - params: NewFullParams, -) -> Result { - use polkadot_node_network_protocol::request_response::IncomingRequest; - use sc_network_common::sync::warp::WarpSyncParams; - - let NewFullParams { + NewFullParams { is_collator, grandpa_pause, enable_beefy, @@ -703,9 +698,12 @@ pub fn new_full( overseer_enable_anyways, overseer_gen, overseer_message_channel_capacity_override, - malus_finality_delay: _, + malus_finality_delay: _malus_finality_delay, hwbench, - } = params; + }: NewFullParams, +) -> Result { + use polkadot_node_network_protocol::request_response::IncomingRequest; + use sc_network_common::sync::warp::WarpSyncParams; let is_offchain_indexing_enabled = config.offchain_worker.indexing_enabled; let role = config.role.clone(); @@ -886,47 +884,7 @@ pub fn new_full( slot_duration_millis: slot_duration.as_millis() as u64, }; - // TODO: Do some manual tests for this. - // 1. Get the binaries from the workers path if it is passed in, or consider all possible - // locations on the filesystem in order and get all sets of paths at which the binaries exist. - // - // 2. If no paths exist, error out. We can't proceed without workers. - // - // 3. Log a warning if more than one set of paths exists. - // - // 4. Check if the returned paths are executable. If not it's evidence of a borked installation - // so error out. - // - // 5. Do the version check, if mismatch error out. - // - // 6. At this point the paths should be good to use. - let mut workers_paths = get_workers_paths(workers_path.clone())?; - if workers_paths.is_empty() { - return Err(Error::MissingWorkerBinaries(workers_path)) - } else if workers_paths.len() > 1 { - log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); - } - let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); - if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { - return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) - } - // Do the version check. - let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); - let prep_worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; - let prep_worker_version = - String::from_utf8(prep_worker_version).expect("version is printed as a string; qed"); - if prep_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) - } - let exec_worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; - let exec_worker_version = - String::from_utf8(exec_worker_version).expect("version is printed as a string; qed"); - if exec_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) - } - // Paths are good to use. - log::info!("using prepare-worker binary at: {:?}", prep_worker_path); - log::info!("using execute-worker binary at: {:?}", exec_worker_path); + let (prep_worker_path, exec_worker_path) = determine_workers_paths(workers_path)?; let candidate_validation_config = CandidateValidationConfig { artifacts_cache_path: config @@ -1284,9 +1242,60 @@ pub fn new_full( }) } -/// Get the workers path by considering the passed-in `workers_path` option, or possible locations -/// on the filesystem. See `new_full`. -fn get_workers_paths( +// TODO: Do some manual tests for this. +/// 1. Get the binaries from the workers path if it is passed in, or consider all possible +/// locations on the filesystem in order and get all sets of paths at which the binaries exist. +/// +/// 2. If no paths exist, error out. We can't proceed without workers. +/// +/// 3. Log a warning if more than one set of paths exists. +/// +/// 4. Check if the returned paths are executable. If not it's evidence of a borked installation +/// so error out. +/// +/// 5. Do the version check, if mismatch error out. +/// +/// 6. At this point the paths should be good to use. +fn determine_workers_paths( + given_workers_path: Option, +) -> Result<(std::path::PathBuf, std::path::PathBuf), Error> { + let mut workers_paths = list_workers_paths(given_workers_path.clone())?; + if workers_paths.is_empty() { + return Err(Error::MissingWorkerBinaries(given_workers_path)) + } else if workers_paths.len() > 1 { + log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); + } + + let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); + if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { + return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) + } + + // Do the version check. + let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); + let prep_worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; + let prep_worker_version = + String::from_utf8(prep_worker_version).expect("version is printed as a string; qed"); + if prep_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) + } + let exec_worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; + let exec_worker_version = + String::from_utf8(exec_worker_version).expect("version is printed as a string; qed"); + if exec_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) + } + + // Paths are good to use. + log::info!("using prepare-worker binary at: {:?}", prep_worker_path); + log::info!("using execute-worker binary at: {:?}", exec_worker_path); + + Ok((prep_worker_path, exec_worker_path)) +} + +/// Get list of workers paths by considering the passed-in `given_workers_path` option, or possible +/// locations on the filesystem. See `new_full`. +fn list_workers_paths( given_workers_path: Option, ) -> Result, Error> { if let Some(path) = given_workers_path { From 49d86968cfb037759f7bcd3716f29eac5e17e4be Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 11 Jul 2023 16:28:31 +0200 Subject: [PATCH 20/48] Fix CI error --- node/test/service/src/lib.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 08d09f8fe69a..776fb7a15993 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -72,21 +72,23 @@ pub use polkadot_service::{FullBackend, GetLastTimestamp}; pub fn new_full( config: Configuration, is_collator: IsCollator, - worker_program_path: Option, + workers_path: Option, ) -> Result { polkadot_service::new_full( config, - is_collator, - None, - true, - None, - None, - worker_program_path, - false, - polkadot_service::RealOverseerGen, - None, - None, - None, + polkadot_service::NewFullParams { + is_collator, + grandpa_pause: None, + enable_beefy: true, + jaeger_agent: None, + telemetry_worker_handle: None, + workers_path, + overseer_enable_anyways: false, + overseer_gen: polkadot_service::RealOverseerGen, + overseer_message_channel_capacity_override: None, + malus_finality_delay: None, + hwbench: None, + }, ) } From dc5d6a0ed0cff77926e265f744cf4a7fcc39925b Mon Sep 17 00:00:00 2001 From: Marcin S Date: Fri, 14 Jul 2023 13:24:51 +0200 Subject: [PATCH 21/48] Add tests for worker binaries detection --- node/core/pvf/common/src/worker/mod.rs | 4 +- node/service/src/lib.rs | 527 ++++++++++++++++++++----- 2 files changed, 426 insertions(+), 105 deletions(-) diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index 659ae01e8c73..89c354fd41a7 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -52,11 +52,11 @@ macro_rules! decl_worker_main { } match args[1].as_ref() { - "--help" => { + "--help" | "-h" => { print_help($expected_command); return }, - "--version" => { + "--version" | "-v" => { println!("{}", $crate::worker::worker_version()); return }, diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 349d7890a631..8d72eecbb0f3 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -38,7 +38,6 @@ mod tests; use { grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider}, gum::info, - is_executable::IsExecutable, polkadot_node_core_approval_voting::{ self as approval_voting_subsystem, Config as ApprovalVotingConfig, }, @@ -55,7 +54,6 @@ use { sc_client_api::BlockBackend, sp_core::traits::SpawnNamed, sp_trie::PrefixedMemoryDB, - std::process::Command, }; use polkadot_node_subsystem_util::database::Database; @@ -884,7 +882,7 @@ pub fn new_full( slot_duration_millis: slot_duration.as_millis() as u64, }; - let (prep_worker_path, exec_worker_path) = determine_workers_paths(workers_path)?; + let (prep_worker_path, exec_worker_path) = workers::determine_workers_paths(workers_path)?; let candidate_validation_config = CandidateValidationConfig { artifacts_cache_path: config @@ -1242,126 +1240,449 @@ pub fn new_full( }) } -// TODO: Do some manual tests for this. -/// 1. Get the binaries from the workers path if it is passed in, or consider all possible -/// locations on the filesystem in order and get all sets of paths at which the binaries exist. -/// -/// 2. If no paths exist, error out. We can't proceed without workers. -/// -/// 3. Log a warning if more than one set of paths exists. -/// -/// 4. Check if the returned paths are executable. If not it's evidence of a borked installation -/// so error out. -/// -/// 5. Do the version check, if mismatch error out. -/// -/// 6. At this point the paths should be good to use. -fn determine_workers_paths( - given_workers_path: Option, -) -> Result<(std::path::PathBuf, std::path::PathBuf), Error> { - let mut workers_paths = list_workers_paths(given_workers_path.clone())?; - if workers_paths.is_empty() { - return Err(Error::MissingWorkerBinaries(given_workers_path)) - } else if workers_paths.len() > 1 { - log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); +#[cfg(feature = "full-node")] +mod workers { + use super::Error; + use is_executable::IsExecutable; + use std::{path::PathBuf, process::Command}; + + #[cfg(test)] + use std::sync::{Mutex, OnceLock}; + + /// Override the workers polkadot binary directory path, used for testing. + #[cfg(test)] + fn workers_exe_path_override() -> &'static Mutex> { + static OVERRIDE: OnceLock>> = OnceLock::new(); + OVERRIDE.get_or_init(|| Mutex::new(None)) } - - let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); - if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { - return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) + /// Override the workers lib directory path, used for testing. + #[cfg(test)] + fn workers_lib_path_override() -> &'static Mutex> { + static OVERRIDE: OnceLock>> = OnceLock::new(); + OVERRIDE.get_or_init(|| Mutex::new(None)) } - // Do the version check. - let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); - let prep_worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; - let prep_worker_version = - String::from_utf8(prep_worker_version).expect("version is printed as a string; qed"); - if prep_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) - } - let exec_worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; - let exec_worker_version = - String::from_utf8(exec_worker_version).expect("version is printed as a string; qed"); - if exec_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) + /// 1. Get the binaries from the workers path if it is passed in, or consider all possible + /// locations on the filesystem in order and get all sets of paths at which the binaries exist. + /// + /// 2. If no paths exist, error out. We can't proceed without workers. + /// + /// 3. Log a warning if more than one set of paths exists. Continue with the first set of paths. + /// + /// 4. Check if the returned paths are executable. If not it's evidence of a borked installation + /// so error out. + /// + /// 5. Do the version check, if mismatch error out. + /// + /// 6. At this point the final set of paths should be good to use. + pub fn determine_workers_paths( + given_workers_path: Option, + ) -> Result<(PathBuf, PathBuf), Error> { + let mut workers_paths = list_workers_paths(given_workers_path.clone())?; + if workers_paths.is_empty() { + return Err(Error::MissingWorkerBinaries(given_workers_path)) + } else if workers_paths.len() > 1 { + log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); + } + + let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); + if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { + return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) + } + + // Do the version check. + let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); + let prep_worker_version = + Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; + let prep_worker_version = std::str::from_utf8(&prep_worker_version) + .expect("version is printed as a string; qed") + .trim() + .to_string(); + if prep_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) + } + let exec_worker_version = + Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; + let exec_worker_version = std::str::from_utf8(&exec_worker_version) + .expect("version is printed as a string; qed") + .trim() + .to_string(); + if exec_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) + } + + // Paths are good to use. + log::info!("using prepare-worker binary at: {:?}", prep_worker_path); + log::info!("using execute-worker binary at: {:?}", exec_worker_path); + + Ok((prep_worker_path, exec_worker_path)) } - // Paths are good to use. - log::info!("using prepare-worker binary at: {:?}", prep_worker_path); - log::info!("using execute-worker binary at: {:?}", exec_worker_path); + /// Get list of workers paths by considering the passed-in `given_workers_path` option, or possible + /// locations on the filesystem. See `new_full`. + fn list_workers_paths( + given_workers_path: Option, + ) -> Result, Error> { + if let Some(path) = given_workers_path { + log::trace!("Using explicitly provided workers path {:?}", path); - Ok((prep_worker_path, exec_worker_path)) -} + if path.is_executable() { + return Ok(vec![(path.clone(), path)]) + } -/// Get list of workers paths by considering the passed-in `given_workers_path` option, or possible -/// locations on the filesystem. See `new_full`. -fn list_workers_paths( - given_workers_path: Option, -) -> Result, Error> { - if let Some(path) = given_workers_path { - log::trace!("Using explicitly provided workers path {:?}", path); + let mut prep_worker = path.clone(); + let mut exec_worker = path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - if path.is_executable() { - return Ok(vec![(path.clone(), path)]) + // Check if both workers exist. Otherwise return an empty vector which results in an error. + return if prep_worker.exists() && exec_worker.exists() { + Ok(vec![(prep_worker, exec_worker)]) + } else { + Ok(vec![]) + } } - let mut prep_worker = path.clone(); - let mut exec_worker = path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + // Workers path not provided, check all possible valid locations. - // Check if both workers exist. Otherwise return an empty vector which results in an error. - return if prep_worker.exists() && exec_worker.exists() { - Ok(vec![(prep_worker, exec_worker)]) - } else { - Ok(vec![]) + let mut workers_paths = vec![]; + + { + let mut exe_path = std::env::current_exe()?; + let _ = exe_path.pop(); // executable file will always have a parent directory. + #[cfg(test)] + if let Some(ref path_override) = *workers_exe_path_override().lock().unwrap() { + exe_path = path_override.clone(); + } + + let mut prep_worker = exe_path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + let mut exec_worker = exe_path.clone(); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Add to set if both workers exist. Warn on partial installs. + let (prep_worker_exists, exec_worker_exists) = + (prep_worker.exists(), exec_worker.exists()); + if prep_worker_exists && exec_worker_exists { + log::trace!("Worker binaries found at current exe path: {:?}", exe_path); + workers_paths.push((prep_worker, exec_worker)); + } else if prep_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); + } else if exec_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + } } + + { + #[allow(unused_mut)] + let mut lib_path = PathBuf::from("/usr/lib/polkadot"); + #[cfg(test)] + if let Some(ref path_override) = *workers_lib_path_override().lock().unwrap() { + lib_path = path_override.clone(); + } + + let mut prep_worker = lib_path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + let mut exec_worker = lib_path.clone(); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Add to set if both workers exist. Warn on partial installs. + let (prep_worker_exists, exec_worker_exists) = + (prep_worker.exists(), exec_worker.exists()); + if prep_worker_exists && exec_worker_exists { + log::trace!("Worker binaries found at /usr/lib/polkadot"); + workers_paths.push((prep_worker, exec_worker)); + } else if prep_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); + } else if exec_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + } + } + + Ok(workers_paths) } - // Workers path not provided, check all possible valid locations. + // Tests that set up a temporary directory tree according to what scenario we want to test and + // run worker detection. + #[cfg(test)] + mod tests { + use super::*; - let mut workers_paths = vec![]; + use assert_matches::assert_matches; + use std::{env::temp_dir, fs, os::unix::fs::PermissionsExt, path::Path}; - { - let mut exe_path = std::env::current_exe()?; - let _ = exe_path.pop(); // executable file will always have a parent directory. - let mut prep_worker = exe_path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = exe_path.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - - // Add to set if both workers exist. Warn on partial installs. - let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); - if prep_worker_exists && exec_worker_exists { - log::trace!("Worker binaries found at current exe path: {:?}", exe_path); - workers_paths.push((prep_worker, exec_worker)); - } else if prep_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); - } else if exec_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); + + /// Write a dummy executable to the path which satisfies the version check. + fn write_worker_exe(path: impl AsRef) -> Result<(), Box> { + let program = get_program(NODE_VERSION); + fs::write(&path, program)?; + Ok(fs::set_permissions(&path, fs::Permissions::from_mode(0o744))?) } - } - { - let lib_path = PathBuf::from("/usr/lib/polkadot"); - let mut prep_worker = lib_path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = lib_path.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - - // Add to set if both workers exist. Warn on partial installs. - let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); - if prep_worker_exists && exec_worker_exists { - log::trace!("Worker binaries found at /usr/lib/polkadot"); - workers_paths.push((prep_worker, exec_worker)); - } else if prep_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); - } else if exec_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + fn write_worker_exe_invalid_version( + path: impl AsRef, + version: &str, + ) -> Result<(), Box> { + let program = get_program(version); + fs::write(&path, program)?; + Ok(fs::set_permissions(&path, fs::Permissions::from_mode(0o744))?) } - } - Ok(workers_paths) + fn get_program(version: &str) -> String { + format!( + "#!/bin/bash + +if [[ $# -ne 1 ]] ; then + echo \"unexpected number of arguments: $#\" + exit 1 +fi + +if [[ \"$1\" != \"--version\" ]] ; then + echo \"unexpected argument: $1\" + exit 1 +fi + +echo {} +", + version + ) + } + + fn with_temp_dir_structure( + f: impl FnOnce(PathBuf) -> Result<(), Box>, + ) -> Result<(), Box> { + // Ensure tests run one at a time so they don't interfere with each other. + static OVERRIDES_LOCK: Mutex<()> = Mutex::new(()); + let _lock = OVERRIDES_LOCK.lock()?; + + // Set up /usr/lib/polkadot and /usr/bin, both empty. + + let tempdir = temp_dir(); + let lib_path = tempdir.join("usr/lib/polkadot"); + let _ = fs::remove_dir_all(&lib_path); + fs::create_dir_all(&lib_path)?; + *workers_lib_path_override().lock()? = Some(lib_path); + + let exe_path = tempdir.join("usr/bin"); + let _ = fs::remove_dir_all(&exe_path); + fs::create_dir_all(&exe_path)?; + *workers_exe_path_override().lock()? = Some(exe_path); + + // Set up custom path at /usr/local/bin. + let custom_path = tempdir.join("usr/local/bin"); + let _ = fs::remove_dir_all(&custom_path); + fs::create_dir_all(&custom_path)?; + + f(tempdir) + } + + #[test] + fn test_given_worker_path() { + with_temp_dir_structure(|tempdir| { + let given_workers_path = tempdir.join("usr/local/bin"); + + // Try with provided workers path that has missing binaries. + assert_matches!( + determine_workers_paths(Some(given_workers_path.clone())), + Err(Error::MissingWorkerBinaries(Some(p))) if p == given_workers_path + ); + + // Try with provided workers path that has not executable binaries. + let prepare_worker_path = given_workers_path.join("polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_path)?; + fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o644))?; + let execute_worker_path = given_workers_path.join("polkadot-execute-worker"); + write_worker_exe(&execute_worker_path)?; + fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o644))?; + assert_matches!( + determine_workers_paths(Some(given_workers_path.clone())), + Err(Error::InvalidWorkerBinaries(p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path + ); + + // Try with valid workers directory path that has executable binaries. + fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o744))?; + fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o744))?; + assert_matches!( + determine_workers_paths(Some(given_workers_path)), + Ok((p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path + ); + + // Try with valid provided workers path that is a binary file. + let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); + write_worker_exe(&given_workers_path)?; + assert_matches!( + determine_workers_paths(Some(given_workers_path.clone())), + Ok((p1, p2)) if p1 == given_workers_path && p2 == given_workers_path + ); + + Ok(()) + }) + .unwrap(); + } + + #[test] + fn missing_workers_paths_throws_error() { + with_temp_dir_structure(|tempdir| { + // Try with both binaries missing. + assert_matches!( + determine_workers_paths(None), + Err(Error::MissingWorkerBinaries(None)) + ); + + // Try with only prep worker (at bin location). + let prepare_worker_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_path)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::MissingWorkerBinaries(None)) + ); + + // Try with only exec worker (at bin location). + fs::remove_file(&prepare_worker_path)?; + let execute_worker_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe(&execute_worker_path)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::MissingWorkerBinaries(None)) + ); + + // Try with only prep worker (at lib location). + fs::remove_file(&execute_worker_path)?; + let prepare_worker_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_path)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::MissingWorkerBinaries(None)) + ); + + // Try with only exec worker (at lib location). + fs::remove_file(&prepare_worker_path)?; + let execute_worker_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); + write_worker_exe(execute_worker_path)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::MissingWorkerBinaries(None)) + ); + + Ok(()) + }) + .unwrap() + } + + #[test] + fn should_find_workers_at_all_locations() { + with_temp_dir_structure(|tempdir| { + let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_bin_path)?; + + let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe(&execute_worker_bin_path)?; + + let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_lib_path)?; + + let execute_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); + write_worker_exe(&execute_worker_lib_path)?; + + assert_matches!( + list_workers_paths(None), + Ok(v) if v == vec![(prepare_worker_bin_path, execute_worker_bin_path), (prepare_worker_lib_path, execute_worker_lib_path)] + ); + + Ok(()) + }) + .unwrap(); + } + + #[test] + fn workers_version_mismatch_throws_error() { + let bad_version = "v9.9.9.9"; + + with_temp_dir_structure(|tempdir| { + // Workers at bin location return bad version. + let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe_invalid_version(&prepare_worker_bin_path, bad_version)?; + write_worker_exe(&execute_worker_bin_path)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + // Workers at lib location return bad version. + let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot-prepare-worker"); + let execute_worker_lib_path = tempdir.join("usr/lib/polkadot-execute-worker"); + write_worker_exe(&prepare_worker_lib_path)?; + write_worker_exe_invalid_version(&execute_worker_lib_path, bad_version)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + // Workers at provided workers location return bad version. + let given_workers_path = tempdir.join("usr/local/bin"); + let prepare_worker_path = given_workers_path.join("polkadot-prepare-worker"); + let execute_worker_path = given_workers_path.join("polkadot-execute-worker"); + write_worker_exe_invalid_version(&prepare_worker_path, bad_version)?; + write_worker_exe_invalid_version(&execute_worker_path, bad_version)?; + assert_matches!( + determine_workers_paths(Some(given_workers_path)), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + // Given worker binary returns bad version. + let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); + write_worker_exe_invalid_version(&given_workers_path, bad_version)?; + assert_matches!( + determine_workers_paths(Some(given_workers_path)), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + Ok(()) + }) + .unwrap(); + } + + #[test] + fn should_find_valid_workers() { + // Test bin location. + with_temp_dir_structure(|tempdir| { + let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_bin_path)?; + + let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe(&execute_worker_bin_path)?; + + assert_matches!( + determine_workers_paths(None), + Ok((p1, p2)) if p1 == prepare_worker_bin_path && p2 == execute_worker_bin_path + ); + + Ok(()) + }) + .unwrap(); + + // Test lib location. + with_temp_dir_structure(|tempdir| { + let prepare_worker_lib_path = + tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_lib_path)?; + + let execute_worker_lib_path = + tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); + write_worker_exe(&execute_worker_lib_path)?; + + assert_matches!( + determine_workers_paths(None), + Ok((p1, p2)) if p1 == prepare_worker_lib_path && p2 == execute_worker_lib_path + ); + + Ok(()) + }) + .unwrap(); + } + } } #[cfg(feature = "full-node")] From 6ef14fed789c7cd7cf3f01cbfdfbd07acd631109 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Fri, 14 Jul 2023 14:27:42 +0200 Subject: [PATCH 22/48] Improve tests; try to fix CI --- Cargo.lock | 39 +++++++++++++++++++ node/service/Cargo.toml | 1 + node/service/src/lib.rs | 10 +++-- .../adder/collator/src/main.rs | 24 ++++++------ .../undying/collator/src/main.rs | 24 ++++++------ 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a4fa0c9313f..615614a7591e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1660,6 +1660,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown 0.12.3", + "lock_api", + "once_cell", + "parking_lot_core 0.9.6", +] + [[package]] name = "data-encoding" version = "2.3.2" @@ -8164,6 +8177,7 @@ dependencies = [ "sc-transaction-pool", "serde", "serde_json", + "serial_test", "sp-api", "sp-authority-discovery", "sp-block-builder", @@ -10920,6 +10934,31 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +dependencies = [ + "dashmap", + "futures", + "lazy_static", + "log", + "parking_lot 0.12.1", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + [[package]] name = "sha-1" version = "0.8.2" diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index 448934ae665f..99e605dee39f 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -145,6 +145,7 @@ polkadot-test-client = { path = "../test/client" } polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } env_logger = "0.9.0" assert_matches = "1.5.0" +serial_test = "2.0.0" tempfile = "3.2" [build-dependencies] diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 8d72eecbb0f3..c513ac0f6667 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -1409,6 +1409,7 @@ mod workers { use super::*; use assert_matches::assert_matches; + use serial_test::serial; use std::{env::temp_dir, fs, os::unix::fs::PermissionsExt, path::Path}; const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); @@ -1452,10 +1453,6 @@ echo {} fn with_temp_dir_structure( f: impl FnOnce(PathBuf) -> Result<(), Box>, ) -> Result<(), Box> { - // Ensure tests run one at a time so they don't interfere with each other. - static OVERRIDES_LOCK: Mutex<()> = Mutex::new(()); - let _lock = OVERRIDES_LOCK.lock()?; - // Set up /usr/lib/polkadot and /usr/bin, both empty. let tempdir = temp_dir(); @@ -1478,6 +1475,7 @@ echo {} } #[test] + #[serial] fn test_given_worker_path() { with_temp_dir_structure(|tempdir| { let given_workers_path = tempdir.join("usr/local/bin"); @@ -1522,6 +1520,7 @@ echo {} } #[test] + #[serial] fn missing_workers_paths_throws_error() { with_temp_dir_structure(|tempdir| { // Try with both binaries missing. @@ -1571,6 +1570,7 @@ echo {} } #[test] + #[serial] fn should_find_workers_at_all_locations() { with_temp_dir_structure(|tempdir| { let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); @@ -1596,6 +1596,7 @@ echo {} } #[test] + #[serial] fn workers_version_mismatch_throws_error() { let bad_version = "v9.9.9.9"; @@ -1645,6 +1646,7 @@ echo {} } #[test] + #[serial] fn should_find_valid_workers() { // Test bin location. with_temp_dir_structure(|tempdir| { diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index cc12e51f714b..f671acea384f 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -58,17 +58,19 @@ fn main() -> Result<()> { let full_node = polkadot_service::build_full( config, - polkadot_service::IsCollator::Yes(collator.collator_key()), - None, - false, - None, - None, - None, - false, - polkadot_service::RealOverseerGen, - None, - None, - None, + polkadot_service::NewFullParams { + is_collator: polkadot_service::IsCollator::Yes(collator.collator_key()), + grandpa_pause: None, + enable_beefy: false, + jaeger_agent: None, + telemetry_worker_handle: None, + workers_path: None, + overseer_enable_anyways: false, + overseer_gen: polkadot_service::RealOverseerGen, + overseer_message_channel_capacity_override: None, + malus_finality_delay: None, + hwbench: None, + }, ) .map_err(|e| e.to_string())?; let mut overseer_handle = full_node diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index a301fef79f8d..9e24f0bb7aca 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -58,17 +58,19 @@ fn main() -> Result<()> { let full_node = polkadot_service::build_full( config, - polkadot_service::IsCollator::Yes(collator.collator_key()), - None, - false, - None, - None, - None, - false, - polkadot_service::RealOverseerGen, - None, - None, - None, + polkadot_service::NewFullParams { + is_collator: polkadot_service::IsCollator::Yes(collator.collator_key()), + grandpa_pause: None, + enable_beefy: false, + jaeger_agent: None, + telemetry_worker_handle: None, + workers_path: None, + overseer_enable_anyways: false, + overseer_gen: polkadot_service::RealOverseerGen, + overseer_message_channel_capacity_override: None, + malus_finality_delay: None, + hwbench: None, + }, ) .map_err(|e| e.to_string())?; let mut overseer_handle = full_node From d2bef1f66ec56c752c947cd7455355e769d4bf1b Mon Sep 17 00:00:00 2001 From: Marcin S Date: Sun, 16 Jul 2023 14:18:44 +0200 Subject: [PATCH 23/48] Move workers module into separate file --- node/service/src/lib.rs | 449 +----------------------------------- node/service/src/workers.rs | 440 +++++++++++++++++++++++++++++++++++ 2 files changed, 442 insertions(+), 447 deletions(-) create mode 100644 node/service/src/workers.rs diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index c513ac0f6667..49165ec3cb89 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -27,6 +27,8 @@ mod relay_chain_selection; #[cfg(feature = "full-node")] pub mod overseer; +#[cfg(feature = "full-node")] +pub mod workers; #[cfg(feature = "full-node")] pub use self::overseer::{OverseerGen, OverseerGenArgs, RealOverseerGen}; @@ -1240,453 +1242,6 @@ pub fn new_full( }) } -#[cfg(feature = "full-node")] -mod workers { - use super::Error; - use is_executable::IsExecutable; - use std::{path::PathBuf, process::Command}; - - #[cfg(test)] - use std::sync::{Mutex, OnceLock}; - - /// Override the workers polkadot binary directory path, used for testing. - #[cfg(test)] - fn workers_exe_path_override() -> &'static Mutex> { - static OVERRIDE: OnceLock>> = OnceLock::new(); - OVERRIDE.get_or_init(|| Mutex::new(None)) - } - /// Override the workers lib directory path, used for testing. - #[cfg(test)] - fn workers_lib_path_override() -> &'static Mutex> { - static OVERRIDE: OnceLock>> = OnceLock::new(); - OVERRIDE.get_or_init(|| Mutex::new(None)) - } - - /// 1. Get the binaries from the workers path if it is passed in, or consider all possible - /// locations on the filesystem in order and get all sets of paths at which the binaries exist. - /// - /// 2. If no paths exist, error out. We can't proceed without workers. - /// - /// 3. Log a warning if more than one set of paths exists. Continue with the first set of paths. - /// - /// 4. Check if the returned paths are executable. If not it's evidence of a borked installation - /// so error out. - /// - /// 5. Do the version check, if mismatch error out. - /// - /// 6. At this point the final set of paths should be good to use. - pub fn determine_workers_paths( - given_workers_path: Option, - ) -> Result<(PathBuf, PathBuf), Error> { - let mut workers_paths = list_workers_paths(given_workers_path.clone())?; - if workers_paths.is_empty() { - return Err(Error::MissingWorkerBinaries(given_workers_path)) - } else if workers_paths.len() > 1 { - log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); - } - - let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); - if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { - return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) - } - - // Do the version check. - let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); - let prep_worker_version = - Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; - let prep_worker_version = std::str::from_utf8(&prep_worker_version) - .expect("version is printed as a string; qed") - .trim() - .to_string(); - if prep_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) - } - let exec_worker_version = - Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; - let exec_worker_version = std::str::from_utf8(&exec_worker_version) - .expect("version is printed as a string; qed") - .trim() - .to_string(); - if exec_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) - } - - // Paths are good to use. - log::info!("using prepare-worker binary at: {:?}", prep_worker_path); - log::info!("using execute-worker binary at: {:?}", exec_worker_path); - - Ok((prep_worker_path, exec_worker_path)) - } - - /// Get list of workers paths by considering the passed-in `given_workers_path` option, or possible - /// locations on the filesystem. See `new_full`. - fn list_workers_paths( - given_workers_path: Option, - ) -> Result, Error> { - if let Some(path) = given_workers_path { - log::trace!("Using explicitly provided workers path {:?}", path); - - if path.is_executable() { - return Ok(vec![(path.clone(), path)]) - } - - let mut prep_worker = path.clone(); - let mut exec_worker = path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - - // Check if both workers exist. Otherwise return an empty vector which results in an error. - return if prep_worker.exists() && exec_worker.exists() { - Ok(vec![(prep_worker, exec_worker)]) - } else { - Ok(vec![]) - } - } - - // Workers path not provided, check all possible valid locations. - - let mut workers_paths = vec![]; - - { - let mut exe_path = std::env::current_exe()?; - let _ = exe_path.pop(); // executable file will always have a parent directory. - #[cfg(test)] - if let Some(ref path_override) = *workers_exe_path_override().lock().unwrap() { - exe_path = path_override.clone(); - } - - let mut prep_worker = exe_path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = exe_path.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - - // Add to set if both workers exist. Warn on partial installs. - let (prep_worker_exists, exec_worker_exists) = - (prep_worker.exists(), exec_worker.exists()); - if prep_worker_exists && exec_worker_exists { - log::trace!("Worker binaries found at current exe path: {:?}", exe_path); - workers_paths.push((prep_worker, exec_worker)); - } else if prep_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); - } else if exec_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); - } - } - - { - #[allow(unused_mut)] - let mut lib_path = PathBuf::from("/usr/lib/polkadot"); - #[cfg(test)] - if let Some(ref path_override) = *workers_lib_path_override().lock().unwrap() { - lib_path = path_override.clone(); - } - - let mut prep_worker = lib_path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = lib_path.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); - - // Add to set if both workers exist. Warn on partial installs. - let (prep_worker_exists, exec_worker_exists) = - (prep_worker.exists(), exec_worker.exists()); - if prep_worker_exists && exec_worker_exists { - log::trace!("Worker binaries found at /usr/lib/polkadot"); - workers_paths.push((prep_worker, exec_worker)); - } else if prep_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); - } else if exec_worker_exists { - log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); - } - } - - Ok(workers_paths) - } - - // Tests that set up a temporary directory tree according to what scenario we want to test and - // run worker detection. - #[cfg(test)] - mod tests { - use super::*; - - use assert_matches::assert_matches; - use serial_test::serial; - use std::{env::temp_dir, fs, os::unix::fs::PermissionsExt, path::Path}; - - const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); - - /// Write a dummy executable to the path which satisfies the version check. - fn write_worker_exe(path: impl AsRef) -> Result<(), Box> { - let program = get_program(NODE_VERSION); - fs::write(&path, program)?; - Ok(fs::set_permissions(&path, fs::Permissions::from_mode(0o744))?) - } - - fn write_worker_exe_invalid_version( - path: impl AsRef, - version: &str, - ) -> Result<(), Box> { - let program = get_program(version); - fs::write(&path, program)?; - Ok(fs::set_permissions(&path, fs::Permissions::from_mode(0o744))?) - } - - fn get_program(version: &str) -> String { - format!( - "#!/bin/bash - -if [[ $# -ne 1 ]] ; then - echo \"unexpected number of arguments: $#\" - exit 1 -fi - -if [[ \"$1\" != \"--version\" ]] ; then - echo \"unexpected argument: $1\" - exit 1 -fi - -echo {} -", - version - ) - } - - fn with_temp_dir_structure( - f: impl FnOnce(PathBuf) -> Result<(), Box>, - ) -> Result<(), Box> { - // Set up /usr/lib/polkadot and /usr/bin, both empty. - - let tempdir = temp_dir(); - let lib_path = tempdir.join("usr/lib/polkadot"); - let _ = fs::remove_dir_all(&lib_path); - fs::create_dir_all(&lib_path)?; - *workers_lib_path_override().lock()? = Some(lib_path); - - let exe_path = tempdir.join("usr/bin"); - let _ = fs::remove_dir_all(&exe_path); - fs::create_dir_all(&exe_path)?; - *workers_exe_path_override().lock()? = Some(exe_path); - - // Set up custom path at /usr/local/bin. - let custom_path = tempdir.join("usr/local/bin"); - let _ = fs::remove_dir_all(&custom_path); - fs::create_dir_all(&custom_path)?; - - f(tempdir) - } - - #[test] - #[serial] - fn test_given_worker_path() { - with_temp_dir_structure(|tempdir| { - let given_workers_path = tempdir.join("usr/local/bin"); - - // Try with provided workers path that has missing binaries. - assert_matches!( - determine_workers_paths(Some(given_workers_path.clone())), - Err(Error::MissingWorkerBinaries(Some(p))) if p == given_workers_path - ); - - // Try with provided workers path that has not executable binaries. - let prepare_worker_path = given_workers_path.join("polkadot-prepare-worker"); - write_worker_exe(&prepare_worker_path)?; - fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o644))?; - let execute_worker_path = given_workers_path.join("polkadot-execute-worker"); - write_worker_exe(&execute_worker_path)?; - fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o644))?; - assert_matches!( - determine_workers_paths(Some(given_workers_path.clone())), - Err(Error::InvalidWorkerBinaries(p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path - ); - - // Try with valid workers directory path that has executable binaries. - fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o744))?; - fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o744))?; - assert_matches!( - determine_workers_paths(Some(given_workers_path)), - Ok((p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path - ); - - // Try with valid provided workers path that is a binary file. - let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); - write_worker_exe(&given_workers_path)?; - assert_matches!( - determine_workers_paths(Some(given_workers_path.clone())), - Ok((p1, p2)) if p1 == given_workers_path && p2 == given_workers_path - ); - - Ok(()) - }) - .unwrap(); - } - - #[test] - #[serial] - fn missing_workers_paths_throws_error() { - with_temp_dir_structure(|tempdir| { - // Try with both binaries missing. - assert_matches!( - determine_workers_paths(None), - Err(Error::MissingWorkerBinaries(None)) - ); - - // Try with only prep worker (at bin location). - let prepare_worker_path = tempdir.join("usr/bin/polkadot-prepare-worker"); - write_worker_exe(&prepare_worker_path)?; - assert_matches!( - determine_workers_paths(None), - Err(Error::MissingWorkerBinaries(None)) - ); - - // Try with only exec worker (at bin location). - fs::remove_file(&prepare_worker_path)?; - let execute_worker_path = tempdir.join("usr/bin/polkadot-execute-worker"); - write_worker_exe(&execute_worker_path)?; - assert_matches!( - determine_workers_paths(None), - Err(Error::MissingWorkerBinaries(None)) - ); - - // Try with only prep worker (at lib location). - fs::remove_file(&execute_worker_path)?; - let prepare_worker_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); - write_worker_exe(&prepare_worker_path)?; - assert_matches!( - determine_workers_paths(None), - Err(Error::MissingWorkerBinaries(None)) - ); - - // Try with only exec worker (at lib location). - fs::remove_file(&prepare_worker_path)?; - let execute_worker_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); - write_worker_exe(execute_worker_path)?; - assert_matches!( - determine_workers_paths(None), - Err(Error::MissingWorkerBinaries(None)) - ); - - Ok(()) - }) - .unwrap() - } - - #[test] - #[serial] - fn should_find_workers_at_all_locations() { - with_temp_dir_structure(|tempdir| { - let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); - write_worker_exe(&prepare_worker_bin_path)?; - - let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); - write_worker_exe(&execute_worker_bin_path)?; - - let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); - write_worker_exe(&prepare_worker_lib_path)?; - - let execute_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); - write_worker_exe(&execute_worker_lib_path)?; - - assert_matches!( - list_workers_paths(None), - Ok(v) if v == vec![(prepare_worker_bin_path, execute_worker_bin_path), (prepare_worker_lib_path, execute_worker_lib_path)] - ); - - Ok(()) - }) - .unwrap(); - } - - #[test] - #[serial] - fn workers_version_mismatch_throws_error() { - let bad_version = "v9.9.9.9"; - - with_temp_dir_structure(|tempdir| { - // Workers at bin location return bad version. - let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); - let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); - write_worker_exe_invalid_version(&prepare_worker_bin_path, bad_version)?; - write_worker_exe(&execute_worker_bin_path)?; - assert_matches!( - determine_workers_paths(None), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION - ); - - // Workers at lib location return bad version. - let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot-prepare-worker"); - let execute_worker_lib_path = tempdir.join("usr/lib/polkadot-execute-worker"); - write_worker_exe(&prepare_worker_lib_path)?; - write_worker_exe_invalid_version(&execute_worker_lib_path, bad_version)?; - assert_matches!( - determine_workers_paths(None), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION - ); - - // Workers at provided workers location return bad version. - let given_workers_path = tempdir.join("usr/local/bin"); - let prepare_worker_path = given_workers_path.join("polkadot-prepare-worker"); - let execute_worker_path = given_workers_path.join("polkadot-execute-worker"); - write_worker_exe_invalid_version(&prepare_worker_path, bad_version)?; - write_worker_exe_invalid_version(&execute_worker_path, bad_version)?; - assert_matches!( - determine_workers_paths(Some(given_workers_path)), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION - ); - - // Given worker binary returns bad version. - let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); - write_worker_exe_invalid_version(&given_workers_path, bad_version)?; - assert_matches!( - determine_workers_paths(Some(given_workers_path)), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION - ); - - Ok(()) - }) - .unwrap(); - } - - #[test] - #[serial] - fn should_find_valid_workers() { - // Test bin location. - with_temp_dir_structure(|tempdir| { - let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); - write_worker_exe(&prepare_worker_bin_path)?; - - let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); - write_worker_exe(&execute_worker_bin_path)?; - - assert_matches!( - determine_workers_paths(None), - Ok((p1, p2)) if p1 == prepare_worker_bin_path && p2 == execute_worker_bin_path - ); - - Ok(()) - }) - .unwrap(); - - // Test lib location. - with_temp_dir_structure(|tempdir| { - let prepare_worker_lib_path = - tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); - write_worker_exe(&prepare_worker_lib_path)?; - - let execute_worker_lib_path = - tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); - write_worker_exe(&execute_worker_lib_path)?; - - assert_matches!( - determine_workers_paths(None), - Ok((p1, p2)) if p1 == prepare_worker_lib_path && p2 == execute_worker_lib_path - ); - - Ok(()) - }) - .unwrap(); - } - } -} - #[cfg(feature = "full-node")] macro_rules! chain_ops { ($config:expr, $jaeger_agent:expr, $telemetry_worker_handle:expr) => {{ diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs new file mode 100644 index 000000000000..fb230a787118 --- /dev/null +++ b/node/service/src/workers.rs @@ -0,0 +1,440 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Utilities and tests for locating the PVF worker binaries. + +use super::Error; +use is_executable::IsExecutable; +use std::{path::PathBuf, process::Command}; + +#[cfg(test)] +use std::sync::{Mutex, OnceLock}; + +/// Override the workers polkadot binary directory path, used for testing. +#[cfg(test)] +fn workers_exe_path_override() -> &'static Mutex> { + static OVERRIDE: OnceLock>> = OnceLock::new(); + OVERRIDE.get_or_init(|| Mutex::new(None)) +} +/// Override the workers lib directory path, used for testing. +#[cfg(test)] +fn workers_lib_path_override() -> &'static Mutex> { + static OVERRIDE: OnceLock>> = OnceLock::new(); + OVERRIDE.get_or_init(|| Mutex::new(None)) +} + +/// 1. Get the binaries from the workers path if it is passed in, or consider all possible +/// locations on the filesystem in order and get all sets of paths at which the binaries exist. +/// +/// 2. If no paths exist, error out. We can't proceed without workers. +/// +/// 3. Log a warning if more than one set of paths exists. Continue with the first set of paths. +/// +/// 4. Check if the returned paths are executable. If not it's evidence of a borked installation +/// so error out. +/// +/// 5. Do the version check, if mismatch error out. +/// +/// 6. At this point the final set of paths should be good to use. +pub fn determine_workers_paths( + given_workers_path: Option, +) -> Result<(PathBuf, PathBuf), Error> { + let mut workers_paths = list_workers_paths(given_workers_path.clone())?; + if workers_paths.is_empty() { + return Err(Error::MissingWorkerBinaries(given_workers_path)) + } else if workers_paths.len() > 1 { + log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); + } + + let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); + if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { + return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) + } + + // Do the version check. + let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); + let prep_worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; + let prep_worker_version = std::str::from_utf8(&prep_worker_version) + .expect("version is printed as a string; qed") + .trim() + .to_string(); + if prep_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) + } + let exec_worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; + let exec_worker_version = std::str::from_utf8(&exec_worker_version) + .expect("version is printed as a string; qed") + .trim() + .to_string(); + if exec_worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) + } + + // Paths are good to use. + log::info!("using prepare-worker binary at: {:?}", prep_worker_path); + log::info!("using execute-worker binary at: {:?}", exec_worker_path); + + Ok((prep_worker_path, exec_worker_path)) +} + +/// Get list of workers paths by considering the passed-in `given_workers_path` option, or possible +/// locations on the filesystem. See `new_full`. +fn list_workers_paths( + given_workers_path: Option, +) -> Result, Error> { + if let Some(path) = given_workers_path { + log::trace!("Using explicitly provided workers path {:?}", path); + + if path.is_executable() { + return Ok(vec![(path.clone(), path)]) + } + + let mut prep_worker = path.clone(); + let mut exec_worker = path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Check if both workers exist. Otherwise return an empty vector which results in an error. + return if prep_worker.exists() && exec_worker.exists() { + Ok(vec![(prep_worker, exec_worker)]) + } else { + Ok(vec![]) + } + } + + // Workers path not provided, check all possible valid locations. + + let mut workers_paths = vec![]; + + { + let mut exe_path = std::env::current_exe()?; + let _ = exe_path.pop(); // executable file will always have a parent directory. + #[cfg(test)] + if let Some(ref path_override) = *workers_exe_path_override().lock().unwrap() { + exe_path = path_override.clone(); + } + + let mut prep_worker = exe_path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + let mut exec_worker = exe_path.clone(); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Add to set if both workers exist. Warn on partial installs. + let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); + if prep_worker_exists && exec_worker_exists { + log::trace!("Worker binaries found at current exe path: {:?}", exe_path); + workers_paths.push((prep_worker, exec_worker)); + } else if prep_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); + } else if exec_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + } + } + + { + #[allow(unused_mut)] + let mut lib_path = PathBuf::from("/usr/lib/polkadot"); + #[cfg(test)] + if let Some(ref path_override) = *workers_lib_path_override().lock().unwrap() { + lib_path = path_override.clone(); + } + + let mut prep_worker = lib_path.clone(); + prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); + let mut exec_worker = lib_path.clone(); + exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + + // Add to set if both workers exist. Warn on partial installs. + let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); + if prep_worker_exists && exec_worker_exists { + log::trace!("Worker binaries found at /usr/lib/polkadot"); + workers_paths.push((prep_worker, exec_worker)); + } else if prep_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", prep_worker, exec_worker); + } else if exec_worker_exists { + log::warn!("Worker binary found at {:?} but not {:?}", exec_worker, prep_worker); + } + } + + Ok(workers_paths) +} + +// Tests that set up a temporary directory tree according to what scenario we want to test and +// run worker detection. +#[cfg(test)] +mod tests { + use super::*; + + use assert_matches::assert_matches; + use serial_test::serial; + use std::{env::temp_dir, fs, os::unix::fs::PermissionsExt, path::Path}; + + const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); + + /// Write a dummy executable to the path which satisfies the version check. + fn write_worker_exe(path: impl AsRef) -> Result<(), Box> { + let program = get_program(NODE_VERSION); + fs::write(&path, program)?; + Ok(fs::set_permissions(&path, fs::Permissions::from_mode(0o744))?) + } + + fn write_worker_exe_invalid_version( + path: impl AsRef, + version: &str, + ) -> Result<(), Box> { + let program = get_program(version); + fs::write(&path, program)?; + Ok(fs::set_permissions(&path, fs::Permissions::from_mode(0o744))?) + } + + fn get_program(version: &str) -> String { + format!( + "#!/bin/bash + +if [[ $# -ne 1 ]] ; then + echo \"unexpected number of arguments: $#\" + exit 1 +fi + +if [[ \"$1\" != \"--version\" ]] ; then + echo \"unexpected argument: $1\" + exit 1 +fi + +echo {} +", + version + ) + } + + fn with_temp_dir_structure( + f: impl FnOnce(PathBuf) -> Result<(), Box>, + ) -> Result<(), Box> { + // Set up /usr/lib/polkadot and /usr/bin, both empty. + + let tempdir = temp_dir(); + let lib_path = tempdir.join("usr/lib/polkadot"); + let _ = fs::remove_dir_all(&lib_path); + fs::create_dir_all(&lib_path)?; + *workers_lib_path_override().lock()? = Some(lib_path); + + let exe_path = tempdir.join("usr/bin"); + let _ = fs::remove_dir_all(&exe_path); + fs::create_dir_all(&exe_path)?; + *workers_exe_path_override().lock()? = Some(exe_path); + + // Set up custom path at /usr/local/bin. + let custom_path = tempdir.join("usr/local/bin"); + let _ = fs::remove_dir_all(&custom_path); + fs::create_dir_all(&custom_path)?; + + f(tempdir) + } + + #[test] + #[serial] + fn test_given_worker_path() { + with_temp_dir_structure(|tempdir| { + let given_workers_path = tempdir.join("usr/local/bin"); + + // Try with provided workers path that has missing binaries. + assert_matches!( + determine_workers_paths(Some(given_workers_path.clone())), + Err(Error::MissingWorkerBinaries(Some(p))) if p == given_workers_path + ); + + // Try with provided workers path that has not executable binaries. + let prepare_worker_path = given_workers_path.join("polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_path)?; + fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o644))?; + let execute_worker_path = given_workers_path.join("polkadot-execute-worker"); + write_worker_exe(&execute_worker_path)?; + fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o644))?; + assert_matches!( + determine_workers_paths(Some(given_workers_path.clone())), + Err(Error::InvalidWorkerBinaries(p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path + ); + + // Try with valid workers directory path that has executable binaries. + fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o744))?; + fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o744))?; + assert_matches!( + determine_workers_paths(Some(given_workers_path)), + Ok((p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path + ); + + // Try with valid provided workers path that is a binary file. + let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); + write_worker_exe(&given_workers_path)?; + assert_matches!( + determine_workers_paths(Some(given_workers_path.clone())), + Ok((p1, p2)) if p1 == given_workers_path && p2 == given_workers_path + ); + + Ok(()) + }) + .unwrap(); + } + + #[test] + #[serial] + fn missing_workers_paths_throws_error() { + with_temp_dir_structure(|tempdir| { + // Try with both binaries missing. + assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + + // Try with only prep worker (at bin location). + let prepare_worker_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_path)?; + assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + + // Try with only exec worker (at bin location). + fs::remove_file(&prepare_worker_path)?; + let execute_worker_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe(&execute_worker_path)?; + assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + + // Try with only prep worker (at lib location). + fs::remove_file(&execute_worker_path)?; + let prepare_worker_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_path)?; + assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + + // Try with only exec worker (at lib location). + fs::remove_file(&prepare_worker_path)?; + let execute_worker_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); + write_worker_exe(execute_worker_path)?; + assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + + Ok(()) + }) + .unwrap() + } + + #[test] + #[serial] + fn should_find_workers_at_all_locations() { + with_temp_dir_structure(|tempdir| { + let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_bin_path)?; + + let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe(&execute_worker_bin_path)?; + + let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_lib_path)?; + + let execute_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); + write_worker_exe(&execute_worker_lib_path)?; + + assert_matches!( + list_workers_paths(None), + Ok(v) if v == vec![(prepare_worker_bin_path, execute_worker_bin_path), (prepare_worker_lib_path, execute_worker_lib_path)] + ); + + Ok(()) + }) + .unwrap(); + } + + #[test] + #[serial] + fn workers_version_mismatch_throws_error() { + let bad_version = "v9.9.9.9"; + + with_temp_dir_structure(|tempdir| { + // Workers at bin location return bad version. + let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe_invalid_version(&prepare_worker_bin_path, bad_version)?; + write_worker_exe(&execute_worker_bin_path)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + // Workers at lib location return bad version. + let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot-prepare-worker"); + let execute_worker_lib_path = tempdir.join("usr/lib/polkadot-execute-worker"); + write_worker_exe(&prepare_worker_lib_path)?; + write_worker_exe_invalid_version(&execute_worker_lib_path, bad_version)?; + assert_matches!( + determine_workers_paths(None), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + // Workers at provided workers location return bad version. + let given_workers_path = tempdir.join("usr/local/bin"); + let prepare_worker_path = given_workers_path.join("polkadot-prepare-worker"); + let execute_worker_path = given_workers_path.join("polkadot-execute-worker"); + write_worker_exe_invalid_version(&prepare_worker_path, bad_version)?; + write_worker_exe_invalid_version(&execute_worker_path, bad_version)?; + assert_matches!( + determine_workers_paths(Some(given_workers_path)), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + // Given worker binary returns bad version. + let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); + write_worker_exe_invalid_version(&given_workers_path, bad_version)?; + assert_matches!( + determine_workers_paths(Some(given_workers_path)), + Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + ); + + Ok(()) + }) + .unwrap(); + } + + #[test] + #[serial] + fn should_find_valid_workers() { + // Test bin location. + with_temp_dir_structure(|tempdir| { + let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_bin_path)?; + + let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); + write_worker_exe(&execute_worker_bin_path)?; + + assert_matches!( + determine_workers_paths(None), + Ok((p1, p2)) if p1 == prepare_worker_bin_path && p2 == execute_worker_bin_path + ); + + Ok(()) + }) + .unwrap(); + + // Test lib location. + with_temp_dir_structure(|tempdir| { + let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); + write_worker_exe(&prepare_worker_lib_path)?; + + let execute_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); + write_worker_exe(&execute_worker_lib_path)?; + + assert_matches!( + determine_workers_paths(None), + Ok((p1, p2)) if p1 == prepare_worker_lib_path && p2 == execute_worker_lib_path + ); + + Ok(()) + }) + .unwrap(); + } +} From 3e09438efb903e5b63ca11e9488aa723dd5d7f03 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Sun, 16 Jul 2023 15:51:53 +0200 Subject: [PATCH 24/48] Try to fix failing test and workers not printing latest version - Tests were not finding the worker binaries - Workers were not being rebuilt when the version changed - Made some errors easier to read --- Cargo.lock | 7 +- node/core/pvf/common/Cargo.toml | 3 - node/core/pvf/common/src/worker/mod.rs | 16 +- node/core/pvf/execute-worker/Cargo.toml | 3 + .../pvf/{common => execute-worker}/build.rs | 1 + node/core/pvf/execute-worker/src/lib.rs | 237 +++++++------ node/core/pvf/execute-worker/src/main.rs | 4 +- node/core/pvf/prepare-worker/Cargo.toml | 3 + node/core/pvf/prepare-worker/build.rs | 20 ++ node/core/pvf/prepare-worker/src/lib.rs | 326 +++++++++--------- node/core/pvf/prepare-worker/src/main.rs | 4 +- node/malus/Cargo.toml | 10 +- node/service/build.rs | 1 + node/service/src/lib.rs | 12 +- node/service/src/workers.rs | 20 +- node/test/service/Cargo.toml | 14 + node/test/service/src/lib.rs | 11 +- 17 files changed, 389 insertions(+), 303 deletions(-) rename node/core/pvf/{common => execute-worker}/build.rs (91%) create mode 100644 node/core/pvf/prepare-worker/build.rs diff --git a/Cargo.lock b/Cargo.lock index 615614a7591e..257bc500428c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7477,7 +7477,6 @@ dependencies = [ "sp-externalities", "sp-io", "sp-tracing", - "substrate-build-script-utils", "tempfile", "tokio", "tracing-gum", @@ -7497,6 +7496,7 @@ dependencies = [ "sp-core", "sp-maybe-compressed-blob", "sp-tracing", + "substrate-build-script-utils", "tikv-jemalloc-ctl", "tokio", "tracing-gum", @@ -7519,6 +7519,7 @@ dependencies = [ "sp-io", "sp-maybe-compressed-blob", "sp-tracing", + "substrate-build-script-utils", "tikv-jemalloc-ctl", "tokio", "tracing-gum", @@ -8386,6 +8387,9 @@ dependencies = [ "pallet-balances", "pallet-staking", "pallet-transaction-payment", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", + "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-overseer", @@ -8420,6 +8424,7 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-state-machine", + "sp-tracing", "substrate-test-client", "substrate-test-utils", "tempfile", diff --git a/node/core/pvf/common/Cargo.toml b/node/core/pvf/common/Cargo.toml index 52a0fb37c569..be119297cbc3 100644 --- a/node/core/pvf/common/Cargo.toml +++ b/node/core/pvf/common/Cargo.toml @@ -31,6 +31,3 @@ landlock = "0.2.0" [dev-dependencies] assert_matches = "1.4.0" tempfile = "3.3.0" - -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index 89c354fd41a7..5dd850eda11b 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -33,11 +33,9 @@ use tokio::{io, net::UnixStream, runtime::Runtime}; /// spawning the desired worker. #[macro_export] macro_rules! decl_worker_main { - ($expected_command:expr, $entrypoint:expr) => { + ($expected_command:expr, $entrypoint:expr, $worker_version:expr) => { fn print_help(expected_command: &str) { - let worker_version = $crate::worker::worker_version(); - - println!("{} {}", expected_command, worker_version); + println!("{} {}", expected_command, $worker_version); println!(); println!("PVF worker that is called by polkadot."); } @@ -57,7 +55,7 @@ macro_rules! decl_worker_main { return }, "--version" | "-v" => { - println!("{}", $crate::worker::worker_version()); + println!("{}", $worker_version); return }, subcommand => { @@ -82,15 +80,11 @@ macro_rules! decl_worker_main { } } - $entrypoint(&socket_path, node_version); + $entrypoint(&socket_path, node_version, $worker_version); } }; } -pub fn worker_version() -> &'static str { - env!("SUBSTRATE_CLI_IMPL_VERSION") -} - /// Some allowed overhead that we account for in the "CPU time monitor" thread's sleeps, on the /// child process. pub const JOB_TIMEOUT_OVERHEAD: Duration = Duration::from_millis(50); @@ -105,6 +99,7 @@ pub fn worker_event_loop( debug_id: &'static str, socket_path: &str, node_version: Option<&str>, + worker_version: &str, mut event_loop: F, ) where F: FnMut(UnixStream) -> Fut, @@ -115,7 +110,6 @@ pub fn worker_event_loop( // Check for a mismatch between the node and worker versions. if let Some(node_version) = node_version { - let worker_version = env!("SUBSTRATE_CLI_IMPL_VERSION"); if node_version != worker_version { gum::error!( target: LOG_TARGET, diff --git a/node/core/pvf/execute-worker/Cargo.toml b/node/core/pvf/execute-worker/Cargo.toml index 167e8b4311a3..86bd9e3d4be7 100644 --- a/node/core/pvf/execute-worker/Cargo.toml +++ b/node/core/pvf/execute-worker/Cargo.toml @@ -25,5 +25,8 @@ sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master [target.'cfg(target_os = "linux")'.dependencies] tikv-jemalloc-ctl = "0.5.0" +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + [features] builder = [] diff --git a/node/core/pvf/common/build.rs b/node/core/pvf/execute-worker/build.rs similarity index 91% rename from node/core/pvf/common/build.rs rename to node/core/pvf/execute-worker/build.rs index 40e9f832586e..abebb1719152 100644 --- a/node/core/pvf/common/build.rs +++ b/node/core/pvf/execute-worker/build.rs @@ -16,4 +16,5 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + println!("cargo:rerun-if-env-changed=SUBSTRATE_CLI_IMPL_VERSION"); } diff --git a/node/core/pvf/execute-worker/src/lib.rs b/node/core/pvf/execute-worker/src/lib.rs index b2714b60a6ee..7cbe225bb11e 100644 --- a/node/core/pvf/execute-worker/src/lib.rs +++ b/node/core/pvf/execute-worker/src/lib.rs @@ -43,6 +43,8 @@ use std::{ }; use tokio::{io, net::UnixStream}; +pub const WORKER_IMPL_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); + // Wasmtime powers the Substrate Executor. It compiles the wasm bytecode into native code. // That native code does not create any stacks and just reuses the stack of the thread that // wasmtime was invoked from. @@ -121,136 +123,149 @@ async fn send_response(stream: &mut UnixStream, response: Response) -> io::Resul /// `node_version`, if `Some`, is checked against the worker version. A mismatch results in /// immediate worker termination. `None` is used for tests and in other situations when version /// check is not necessary. -pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>) { - worker_event_loop("execute", socket_path, node_version, |mut stream| async move { - let worker_pid = std::process::id(); +pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>, worker_version: &str) { + worker_event_loop( + "execute", + socket_path, + node_version, + worker_version, + |mut stream| async move { + let worker_pid = std::process::id(); - let handshake = recv_handshake(&mut stream).await?; - let executor = Executor::new(handshake.executor_params).map_err(|e| { - io::Error::new(io::ErrorKind::Other, format!("cannot create executor: {}", e)) - })?; + let handshake = recv_handshake(&mut stream).await?; + let executor = Executor::new(handshake.executor_params).map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("cannot create executor: {}", e)) + })?; - loop { - let (artifact_path, params, execution_timeout) = recv_request(&mut stream).await?; - gum::debug!( - target: LOG_TARGET, - %worker_pid, - "worker: validating artifact {}", - artifact_path.display(), - ); + loop { + let (artifact_path, params, execution_timeout) = recv_request(&mut stream).await?; + gum::debug!( + target: LOG_TARGET, + %worker_pid, + "worker: validating artifact {}", + artifact_path.display(), + ); - // Get the artifact bytes. - // - // We do this outside the thread so that we can lock down filesystem access there. - let compiled_artifact_blob = match std::fs::read(artifact_path) { - Ok(bytes) => bytes, - Err(err) => { - let response = Response::InternalError( - InternalValidationError::CouldNotOpenFile(err.to_string()), - ); - send_response(&mut stream, response).await?; - continue - }, - }; + // Get the artifact bytes. + // + // We do this outside the thread so that we can lock down filesystem access there. + let compiled_artifact_blob = match std::fs::read(artifact_path) { + Ok(bytes) => bytes, + Err(err) => { + let response = Response::InternalError( + InternalValidationError::CouldNotOpenFile(err.to_string()), + ); + send_response(&mut stream, response).await?; + continue + }, + }; - // Conditional variable to notify us when a thread is done. - let condvar = thread::get_condvar(); + // Conditional variable to notify us when a thread is done. + let condvar = thread::get_condvar(); - let cpu_time_start = ProcessTime::now(); + let cpu_time_start = ProcessTime::now(); - // Spawn a new thread that runs the CPU time monitor. - let (cpu_time_monitor_tx, cpu_time_monitor_rx) = channel::<()>(); - let cpu_time_monitor_thread = thread::spawn_worker_thread( - "cpu time monitor thread", - move || { - cpu_time_monitor_loop(cpu_time_start, execution_timeout, cpu_time_monitor_rx) - }, - Arc::clone(&condvar), - WaitOutcome::TimedOut, - )?; - let executor_2 = executor.clone(); - let execute_thread = thread::spawn_worker_thread_with_stack_size( - "execute thread", - move || { - // Try to enable landlock. - #[cfg(target_os = "linux")] + // Spawn a new thread that runs the CPU time monitor. + let (cpu_time_monitor_tx, cpu_time_monitor_rx) = channel::<()>(); + let cpu_time_monitor_thread = thread::spawn_worker_thread( + "cpu time monitor thread", + move || { + cpu_time_monitor_loop( + cpu_time_start, + execution_timeout, + cpu_time_monitor_rx, + ) + }, + Arc::clone(&condvar), + WaitOutcome::TimedOut, + )?; + let executor_2 = executor.clone(); + let execute_thread = thread::spawn_worker_thread_with_stack_size( + "execute thread", + move || { + // Try to enable landlock. + #[cfg(target_os = "linux")] let landlock_status = polkadot_node_core_pvf_common::worker::security::landlock::try_restrict_thread() .map(LandlockStatus::from_ruleset_status) .map_err(|e| e.to_string()); - #[cfg(not(target_os = "linux"))] - let landlock_status: Result = Ok(LandlockStatus::NotEnforced); - - ( - validate_using_artifact( - &compiled_artifact_blob, - ¶ms, - executor_2, - cpu_time_start, - ), - landlock_status, - ) - }, - Arc::clone(&condvar), - WaitOutcome::Finished, - EXECUTE_THREAD_STACK_SIZE, - )?; - - let outcome = thread::wait_for_threads(condvar); + #[cfg(not(target_os = "linux"))] + let landlock_status: Result = Ok(LandlockStatus::NotEnforced); - let response = match outcome { - WaitOutcome::Finished => { - let _ = cpu_time_monitor_tx.send(()); - let (result, landlock_status) = execute_thread.join().unwrap_or_else(|e| { ( - Response::Panic(stringify_panic_payload(e)), - Ok(LandlockStatus::Unavailable), + validate_using_artifact( + &compiled_artifact_blob, + ¶ms, + executor_2, + cpu_time_start, + ), + landlock_status, ) - }); + }, + Arc::clone(&condvar), + WaitOutcome::Finished, + EXECUTE_THREAD_STACK_SIZE, + )?; - // Log if landlock threw an error. - if let Err(err) = landlock_status { - gum::warn!( - target: LOG_TARGET, - %worker_pid, - "error enabling landlock: {}", - err - ); - } + let outcome = thread::wait_for_threads(condvar); - result - }, - // If the CPU thread is not selected, we signal it to end, the join handle is - // dropped and the thread will finish in the background. - WaitOutcome::TimedOut => { - match cpu_time_monitor_thread.join() { - Ok(Some(cpu_time_elapsed)) => { - // Log if we exceed the timeout and the other thread hasn't finished. + let response = match outcome { + WaitOutcome::Finished => { + let _ = cpu_time_monitor_tx.send(()); + let (result, landlock_status) = execute_thread.join().unwrap_or_else(|e| { + ( + Response::Panic(stringify_panic_payload(e)), + Ok(LandlockStatus::Unavailable), + ) + }); + + // Log if landlock threw an error. + if let Err(err) = landlock_status { gum::warn!( target: LOG_TARGET, %worker_pid, - "execute job took {}ms cpu time, exceeded execute timeout {}ms", - cpu_time_elapsed.as_millis(), - execution_timeout.as_millis(), + "error enabling landlock: {}", + err ); - Response::TimedOut - }, - Ok(None) => - Response::InternalError(InternalValidationError::CpuTimeMonitorThread( - "error communicating over finished channel".into(), - )), - Err(e) => - Response::InternalError(InternalValidationError::CpuTimeMonitorThread( - stringify_panic_payload(e), - )), - } - }, - WaitOutcome::Pending => - unreachable!("we run wait_while until the outcome is no longer pending; qed"), - }; + } + + result + }, + // If the CPU thread is not selected, we signal it to end, the join handle is + // dropped and the thread will finish in the background. + WaitOutcome::TimedOut => { + match cpu_time_monitor_thread.join() { + Ok(Some(cpu_time_elapsed)) => { + // Log if we exceed the timeout and the other thread hasn't finished. + gum::warn!( + target: LOG_TARGET, + %worker_pid, + "execute job took {}ms cpu time, exceeded execute timeout {}ms", + cpu_time_elapsed.as_millis(), + execution_timeout.as_millis(), + ); + Response::TimedOut + }, + Ok(None) => Response::InternalError( + InternalValidationError::CpuTimeMonitorThread( + "error communicating over finished channel".into(), + ), + ), + Err(e) => Response::InternalError( + InternalValidationError::CpuTimeMonitorThread( + stringify_panic_payload(e), + ), + ), + } + }, + WaitOutcome::Pending => unreachable!( + "we run wait_while until the outcome is no longer pending; qed" + ), + }; - send_response(&mut stream, response).await?; - } - }); + send_response(&mut stream, response).await?; + } + }, + ); } fn validate_using_artifact( diff --git a/node/core/pvf/execute-worker/src/main.rs b/node/core/pvf/execute-worker/src/main.rs index 5724b2b9b92a..203929ee71e1 100644 --- a/node/core/pvf/execute-worker/src/main.rs +++ b/node/core/pvf/execute-worker/src/main.rs @@ -18,5 +18,7 @@ polkadot_node_core_pvf_common::decl_worker_main!( "execute-worker", - polkadot_node_core_pvf_execute_worker::worker_entrypoint + polkadot_node_core_pvf_execute_worker::worker_entrypoint, + // Defined in lib.rs because env! doesn't work here. + polkadot_node_core_pvf_execute_worker::WORKER_IMPL_VERSION ); diff --git a/node/core/pvf/prepare-worker/Cargo.toml b/node/core/pvf/prepare-worker/Cargo.toml index 3bd1fd43b673..2331791faf22 100644 --- a/node/core/pvf/prepare-worker/Cargo.toml +++ b/node/core/pvf/prepare-worker/Cargo.toml @@ -28,6 +28,9 @@ sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master [target.'cfg(target_os = "linux")'.dependencies] tikv-jemalloc-ctl = "0.5.0" +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + [features] builder = [] jemalloc-allocator = ["dep:tikv-jemalloc-ctl"] diff --git a/node/core/pvf/prepare-worker/build.rs b/node/core/pvf/prepare-worker/build.rs new file mode 100644 index 000000000000..abebb1719152 --- /dev/null +++ b/node/core/pvf/prepare-worker/build.rs @@ -0,0 +1,20 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); + println!("cargo:rerun-if-env-changed=SUBSTRATE_CLI_IMPL_VERSION"); +} diff --git a/node/core/pvf/prepare-worker/src/lib.rs b/node/core/pvf/prepare-worker/src/lib.rs index 6f3cb18b4280..7d393bd97742 100644 --- a/node/core/pvf/prepare-worker/src/lib.rs +++ b/node/core/pvf/prepare-worker/src/lib.rs @@ -51,6 +51,8 @@ use std::{ }; use tokio::{io, net::UnixStream}; +pub const WORKER_IMPL_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); + /// Contains the bytes for a successfully compiled artifact. pub struct CompiledArtifact(Vec); @@ -116,169 +118,185 @@ async fn send_response(stream: &mut UnixStream, result: PrepareResult) -> io::Re /// /// 7. Send the result of preparation back to the host. If any error occurred in the above steps, we /// send that in the `PrepareResult`. -pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>) { - worker_event_loop("prepare", socket_path, node_version, |mut stream| async move { - let worker_pid = std::process::id(); - - loop { - let (pvf, temp_artifact_dest) = recv_request(&mut stream).await?; - gum::debug!( - target: LOG_TARGET, - %worker_pid, - "worker: preparing artifact", - ); - - let preparation_timeout = pvf.prep_timeout(); - let prepare_job_kind = pvf.prep_kind(); - let executor_params = (*pvf.executor_params()).clone(); - - // Conditional variable to notify us when a thread is done. - let condvar = thread::get_condvar(); - - // Run the memory tracker in a regular, non-worker thread. - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - let condvar_memory = Arc::clone(&condvar); - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - let memory_tracker_thread = std::thread::spawn(|| memory_tracker_loop(condvar_memory)); - - let cpu_time_start = ProcessTime::now(); - - // Spawn a new thread that runs the CPU time monitor. - let (cpu_time_monitor_tx, cpu_time_monitor_rx) = channel::<()>(); - let cpu_time_monitor_thread = thread::spawn_worker_thread( - "cpu time monitor thread", - move || { - cpu_time_monitor_loop(cpu_time_start, preparation_timeout, cpu_time_monitor_rx) - }, - Arc::clone(&condvar), - WaitOutcome::TimedOut, - )?; - // Spawn another thread for preparation. - let prepare_thread = thread::spawn_worker_thread( - "prepare thread", - move || { - // Try to enable landlock. - #[cfg(target_os = "linux")] +pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>, worker_version: &str) { + worker_event_loop( + "prepare", + socket_path, + node_version, + worker_version, + |mut stream| async move { + let worker_pid = std::process::id(); + + loop { + let (pvf, temp_artifact_dest) = recv_request(&mut stream).await?; + gum::debug!( + target: LOG_TARGET, + %worker_pid, + "worker: preparing artifact", + ); + + let preparation_timeout = pvf.prep_timeout(); + let prepare_job_kind = pvf.prep_kind(); + let executor_params = (*pvf.executor_params()).clone(); + + // Conditional variable to notify us when a thread is done. + let condvar = thread::get_condvar(); + + // Run the memory tracker in a regular, non-worker thread. + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + let condvar_memory = Arc::clone(&condvar); + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + let memory_tracker_thread = std::thread::spawn(|| memory_tracker_loop(condvar_memory)); + + let cpu_time_start = ProcessTime::now(); + + // Spawn a new thread that runs the CPU time monitor. + let (cpu_time_monitor_tx, cpu_time_monitor_rx) = channel::<()>(); + let cpu_time_monitor_thread = thread::spawn_worker_thread( + "cpu time monitor thread", + move || { + cpu_time_monitor_loop( + cpu_time_start, + preparation_timeout, + cpu_time_monitor_rx, + ) + }, + Arc::clone(&condvar), + WaitOutcome::TimedOut, + )?; + // Spawn another thread for preparation. + let prepare_thread = thread::spawn_worker_thread( + "prepare thread", + move || { + // Try to enable landlock. + #[cfg(target_os = "linux")] let landlock_status = polkadot_node_core_pvf_common::worker::security::landlock::try_restrict_thread() .map(LandlockStatus::from_ruleset_status) .map_err(|e| e.to_string()); - #[cfg(not(target_os = "linux"))] - let landlock_status: Result = Ok(LandlockStatus::NotEnforced); - - #[allow(unused_mut)] - let mut result = prepare_artifact(pvf, cpu_time_start); - - // Get the `ru_maxrss` stat. If supported, call getrusage for the thread. - #[cfg(target_os = "linux")] - let mut result = result.map(|(artifact, elapsed)| (artifact, elapsed, get_max_rss_thread())); - - // If we are pre-checking, check for runtime construction errors. - // - // As pre-checking is more strict than just preparation in terms of memory and - // time, it is okay to do extra checks here. This takes negligible time anyway. - if let PrepareJobKind::Prechecking = prepare_job_kind { - result = result.and_then(|output| { - runtime_construction_check(output.0.as_ref(), executor_params)?; - Ok(output) - }); - } - - (result, landlock_status) - }, - Arc::clone(&condvar), - WaitOutcome::Finished, - )?; - - let outcome = thread::wait_for_threads(condvar); - - let result = match outcome { - WaitOutcome::Finished => { - let _ = cpu_time_monitor_tx.send(()); - - match prepare_thread.join().unwrap_or_else(|err| { - ( - Err(PrepareError::Panic(stringify_panic_payload(err))), - Ok(LandlockStatus::Unavailable), - ) - }) { - (Err(err), _) => { - // Serialized error will be written into the socket. - Err(err) - }, - (Ok(ok), landlock_status) => { - #[cfg(not(target_os = "linux"))] - let (artifact, cpu_time_elapsed) = ok; - #[cfg(target_os = "linux")] - let (artifact, cpu_time_elapsed, max_rss) = ok; - - // Stop the memory stats worker and get its observed memory stats. - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - let memory_tracker_stats = get_memory_tracker_loop_stats(memory_tracker_thread, worker_pid).await; - let memory_stats = MemoryStats { - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - memory_tracker_stats, + #[cfg(not(target_os = "linux"))] + let landlock_status: Result = Ok(LandlockStatus::NotEnforced); + + #[allow(unused_mut)] + let mut result = prepare_artifact(pvf, cpu_time_start); + + // Get the `ru_maxrss` stat. If supported, call getrusage for the thread. + #[cfg(target_os = "linux")] + let mut result = result + .map(|(artifact, elapsed)| (artifact, elapsed, get_max_rss_thread())); + + // If we are pre-checking, check for runtime construction errors. + // + // As pre-checking is more strict than just preparation in terms of memory and + // time, it is okay to do extra checks here. This takes negligible time anyway. + if let PrepareJobKind::Prechecking = prepare_job_kind { + result = result.and_then(|output| { + runtime_construction_check(output.0.as_ref(), executor_params)?; + Ok(output) + }); + } + + (result, landlock_status) + }, + Arc::clone(&condvar), + WaitOutcome::Finished, + )?; + + let outcome = thread::wait_for_threads(condvar); + + let result = match outcome { + WaitOutcome::Finished => { + let _ = cpu_time_monitor_tx.send(()); + + match prepare_thread.join().unwrap_or_else(|err| { + ( + Err(PrepareError::Panic(stringify_panic_payload(err))), + Ok(LandlockStatus::Unavailable), + ) + }) { + (Err(err), _) => { + // Serialized error will be written into the socket. + Err(err) + }, + (Ok(ok), landlock_status) => { + #[cfg(not(target_os = "linux"))] + let (artifact, cpu_time_elapsed) = ok; #[cfg(target_os = "linux")] - max_rss: extract_max_rss_stat(max_rss, worker_pid), - }; + let (artifact, cpu_time_elapsed, max_rss) = ok; - // Log if landlock threw an error. - if let Err(err) = landlock_status { + // Stop the memory stats worker and get its observed memory stats. + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + let memory_tracker_stats = get_memory_tracker_loop_stats(memory_tracker_thread, worker_pid) + .await; + let memory_stats = MemoryStats { + #[cfg(any( + target_os = "linux", + feature = "jemalloc-allocator" + ))] + memory_tracker_stats, + #[cfg(target_os = "linux")] + max_rss: extract_max_rss_stat(max_rss, worker_pid), + }; + + // Log if landlock threw an error. + if let Err(err) = landlock_status { + gum::warn!( + target: LOG_TARGET, + %worker_pid, + "error enabling landlock: {}", + err + ); + } + + // Write the serialized artifact into a temp file. + // + // PVF host only keeps artifacts statuses in its memory, successfully + // compiled code gets stored on the disk (and consequently deserialized + // by execute-workers). The prepare worker is only required to send `Ok` + // to the pool to indicate the success. + + gum::debug!( + target: LOG_TARGET, + %worker_pid, + "worker: writing artifact to {}", + temp_artifact_dest.display(), + ); + tokio::fs::write(&temp_artifact_dest, &artifact).await?; + + Ok(PrepareStats { cpu_time_elapsed, memory_stats }) + }, + } + }, + // If the CPU thread is not selected, we signal it to end, the join handle is + // dropped and the thread will finish in the background. + WaitOutcome::TimedOut => { + match cpu_time_monitor_thread.join() { + Ok(Some(cpu_time_elapsed)) => { + // Log if we exceed the timeout and the other thread hasn't finished. gum::warn!( target: LOG_TARGET, %worker_pid, - "error enabling landlock: {}", - err + "prepare job took {}ms cpu time, exceeded prepare timeout {}ms", + cpu_time_elapsed.as_millis(), + preparation_timeout.as_millis(), ); - } - - // Write the serialized artifact into a temp file. - // - // PVF host only keeps artifacts statuses in its memory, successfully - // compiled code gets stored on the disk (and consequently deserialized - // by execute-workers). The prepare worker is only required to send `Ok` - // to the pool to indicate the success. - - gum::debug!( - target: LOG_TARGET, - %worker_pid, - "worker: writing artifact to {}", - temp_artifact_dest.display(), - ); - tokio::fs::write(&temp_artifact_dest, &artifact).await?; - - Ok(PrepareStats { cpu_time_elapsed, memory_stats }) - }, - } - }, - // If the CPU thread is not selected, we signal it to end, the join handle is - // dropped and the thread will finish in the background. - WaitOutcome::TimedOut => { - match cpu_time_monitor_thread.join() { - Ok(Some(cpu_time_elapsed)) => { - // Log if we exceed the timeout and the other thread hasn't finished. - gum::warn!( - target: LOG_TARGET, - %worker_pid, - "prepare job took {}ms cpu time, exceeded prepare timeout {}ms", - cpu_time_elapsed.as_millis(), - preparation_timeout.as_millis(), - ); - Err(PrepareError::TimedOut) - }, - Ok(None) => Err(PrepareError::IoErr( - "error communicating over closed channel".into(), - )), - // Errors in this thread are independent of the PVF. - Err(err) => Err(PrepareError::IoErr(stringify_panic_payload(err))), - } - }, - WaitOutcome::Pending => - unreachable!("we run wait_while until the outcome is no longer pending; qed"), - }; - - send_response(&mut stream, result).await?; - } - }); + Err(PrepareError::TimedOut) + }, + Ok(None) => Err(PrepareError::IoErr( + "error communicating over closed channel".into(), + )), + // Errors in this thread are independent of the PVF. + Err(err) => Err(PrepareError::IoErr(stringify_panic_payload(err))), + } + }, + WaitOutcome::Pending => unreachable!( + "we run wait_while until the outcome is no longer pending; qed" + ), + }; + + send_response(&mut stream, result).await?; + } + }, + ); } fn prepare_artifact( diff --git a/node/core/pvf/prepare-worker/src/main.rs b/node/core/pvf/prepare-worker/src/main.rs index 1757dbde1893..4fd41d5c7283 100644 --- a/node/core/pvf/prepare-worker/src/main.rs +++ b/node/core/pvf/prepare-worker/src/main.rs @@ -20,5 +20,7 @@ pub use polkadot_node_core_pvf_common::decl_worker_main; polkadot_node_core_pvf_common::decl_worker_main!( "prepare-worker", - polkadot_node_core_pvf_prepare_worker::worker_entrypoint + polkadot_node_core_pvf_prepare_worker::worker_entrypoint, + // Defined in lib.rs because env! doesn't work here. + polkadot_node_core_pvf_prepare_worker::WORKER_IMPL_VERSION ); diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 26fe52e35491..fede72966a32 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -32,9 +32,6 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator" } polkadot-node-core-candidate-validation = { path = "../core/candidate-validation" } polkadot-node-core-backing = { path = "../core/backing" } -polkadot-node-core-pvf-common = { path = "../core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } color-eyre = { version = "0.6.1", default-features = false } @@ -42,7 +39,6 @@ assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } clap = { version = "4.0.9", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" @@ -50,6 +46,12 @@ gum = { package = "tracing-gum", path = "../gum/" } erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } rand = "0.8.5" +# Required for binary workers to build. +polkadot-node-core-pvf-common = { path = "../core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [features] default = [] fast-runtime = ["polkadot-cli/fast-runtime"] diff --git a/node/service/build.rs b/node/service/build.rs index 40e9f832586e..abebb1719152 100644 --- a/node/service/build.rs +++ b/node/service/build.rs @@ -16,4 +16,5 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + println!("cargo:rerun-if-env-changed=SUBSTRATE_CLI_IMPL_VERSION"); } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 49165ec3cb89..86b79ac12902 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -236,16 +236,16 @@ pub enum Error { NoRuntime, #[cfg(feature = "full-node")] - #[error("Worker binaries not executable, prepare binary: {0:?}, execute binary: {1:?}")] - InvalidWorkerBinaries(PathBuf, PathBuf), + #[error("Worker binaries not executable, prepare binary: {prep_worker_path:?}, execute binary: {exec_worker_path:?}")] + InvalidWorkerBinaries { prep_worker_path: PathBuf, exec_worker_path: PathBuf }, #[cfg(feature = "full-node")] - #[error("Worker binaries could not be found at workers path ({0:?}), polkadot binary directory, or /usr/lib/polkadot")] - MissingWorkerBinaries(Option), + #[error("Worker binaries could not be found at workers path ({given_workers_path:?}), polkadot binary directory, or /usr/lib/polkadot")] + MissingWorkerBinaries { given_workers_path: Option }, #[cfg(feature = "full-node")] - #[error("Version of worker binaries ({0}) is different from node version ({1})")] - WorkerBinaryVersionMismatch(String, String), + #[error("Version of worker binaries ({worker_version}) is different from node version ({node_version})")] + WorkerBinaryVersionMismatch { worker_version: String, node_version: String }, } /// Identifies the variant of the chain. diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index fb230a787118..4c4ace5f1300 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -54,33 +54,33 @@ pub fn determine_workers_paths( ) -> Result<(PathBuf, PathBuf), Error> { let mut workers_paths = list_workers_paths(given_workers_path.clone())?; if workers_paths.is_empty() { - return Err(Error::MissingWorkerBinaries(given_workers_path)) + return Err(Error::MissingWorkerBinaries { given_workers_path }) } else if workers_paths.len() > 1 { log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); } let (prep_worker_path, exec_worker_path) = workers_paths.swap_remove(0); if !prep_worker_path.is_executable() || !exec_worker_path.is_executable() { - return Err(Error::InvalidWorkerBinaries(prep_worker_path, exec_worker_path)) + return Err(Error::InvalidWorkerBinaries { prep_worker_path, exec_worker_path }) } // Do the version check. let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); - let prep_worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; - let prep_worker_version = std::str::from_utf8(&prep_worker_version) + let worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; + let worker_version = std::str::from_utf8(&worker_version) .expect("version is printed as a string; qed") .trim() .to_string(); - if prep_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(prep_worker_version, node_version)) + if worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch { worker_version, node_version }) } - let exec_worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; - let exec_worker_version = std::str::from_utf8(&exec_worker_version) + let worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; + let worker_version = std::str::from_utf8(&worker_version) .expect("version is printed as a string; qed") .trim() .to_string(); - if exec_worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch(exec_worker_version, node_version)) + if worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch { worker_version, node_version }) } // Paths are good to use. diff --git a/node/test/service/Cargo.toml b/node/test/service/Cargo.toml index 504ee5beca74..d2c16cbb3e96 100644 --- a/node/test/service/Cargo.toml +++ b/node/test/service/Cargo.toml @@ -4,6 +4,14 @@ version.workspace = true authors.workspace = true edition.workspace = true +[[bin]] +name = "polkadot-execute-worker" +path = "../../core/pvf/execute-worker/src/main.rs" + +[[bin]] +name = "polkadot-prepare-worker" +path = "../../core/pvf/prepare-worker/src/main.rs" + [dependencies] futures = "0.3.21" hex = "0.4.3" @@ -54,6 +62,12 @@ sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" } substrate-test-client = { git = "https://github.com/paritytech/substrate", branch = "master" } +# Required for binary workers to build. +polkadot-node-core-pvf-common = { path = "../../core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../../core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../../core/pvf/prepare-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [dev-dependencies] pallet-balances = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } serde_json = "1.0.96" diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 776fb7a15993..9138472d9e5e 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -197,9 +197,18 @@ pub fn node_config( /// Run a test validator node that uses the test runtime and specified `config`. pub fn run_validator_node( config: Configuration, - worker_program_path: Option, + mut worker_program_path: Option, ) -> PolkadotTestNode { let multiaddr = config.network.listen_addresses[0].clone(); + if let None = worker_program_path { + // If no explicit worker path is passed in, we need to specify it ourselves as test binaries + // are in the "deps/" directory, one level below where the worker binaries are generated. + let mut exe_path = std::env::current_exe() + .expect("for test purposes it's reasonable to expect that this will not fail"); + let _ = exe_path.pop(); + let _ = exe_path.pop(); + worker_program_path = Some(exe_path); + } let NewFull { task_manager, client, network, rpc_handlers, overseer_handle, .. } = new_full(config, IsCollator::No, worker_program_path) .expect("could not create Polkadot test service"); From 55365d865e284c2a40ef16efb44461f1dfc3027f Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 17 Jul 2023 20:43:33 +0200 Subject: [PATCH 25/48] Make a bunch of fixes --- Cargo.lock | 2 + cli/src/command.rs | 1 + node/core/pvf/bin/puppet_worker.rs | 2 +- node/core/pvf/common/src/worker/mod.rs | 2 + node/core/pvf/execute-worker/build.rs | 2 +- node/core/pvf/prepare-worker/build.rs | 2 +- node/core/pvf/src/testing.rs | 6 +- node/malus/Cargo.toml | 3 +- node/service/build.rs | 4 +- node/service/src/lib.rs | 9 +- node/service/src/workers.rs | 128 +++++++++++++----- node/test/service/Cargo.toml | 7 +- node/test/service/src/lib.rs | 4 + .../test-parachains/adder/collator/Cargo.toml | 3 + .../adder/collator/bin/puppet_worker.rs | 2 +- .../test-parachains/adder/collator/build.rs | 19 +++ .../adder/collator/src/main.rs | 1 + .../undying/collator/Cargo.toml | 3 + .../undying/collator/bin/puppet_worker.rs | 2 +- .../test-parachains/undying/collator/build.rs | 19 +++ .../undying/collator/src/main.rs | 1 + 21 files changed, 172 insertions(+), 50 deletions(-) create mode 100644 parachain/test-parachains/adder/collator/build.rs create mode 100644 parachain/test-parachains/undying/collator/build.rs diff --git a/Cargo.lock b/Cargo.lock index 257bc500428c..16e687455b04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12519,6 +12519,7 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", + "substrate-build-script-utils", "substrate-test-utils", "test-parachain-adder", "tokio", @@ -12567,6 +12568,7 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", + "substrate-build-script-utils", "substrate-test-utils", "test-parachain-undying", "tokio", diff --git a/cli/src/command.rs b/cli/src/command.rs index 65d3f4ed4743..a42d4694fd31 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -292,6 +292,7 @@ where jaeger_agent, telemetry_worker_handle: None, workers_path: cli.run.workers_path, + workers_names: None, overseer_enable_anyways: false, overseer_gen, overseer_message_channel_capacity_override: cli diff --git a/node/core/pvf/bin/puppet_worker.rs b/node/core/pvf/bin/puppet_worker.rs index 7f93519d8454..9ad99d6c9c66 100644 --- a/node/core/pvf/bin/puppet_worker.rs +++ b/node/core/pvf/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(); +polkadot_node_core_pvf::decl_puppet_worker_main!(env!("SUBSTRATE_CLI_IMPL_VERSION")); diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index 5dd850eda11b..464a11df9979 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -95,6 +95,8 @@ pub fn bytes_to_path(bytes: &[u8]) -> Option { std::str::from_utf8(bytes).ok().map(PathBuf::from) } +// The worker version must be passed in so that we accurately get the version of the worker, and not +// the version that this crate was compiled with. pub fn worker_event_loop( debug_id: &'static str, socket_path: &str, diff --git a/node/core/pvf/execute-worker/build.rs b/node/core/pvf/execute-worker/build.rs index abebb1719152..998da2eeb7de 100644 --- a/node/core/pvf/execute-worker/build.rs +++ b/node/core/pvf/execute-worker/build.rs @@ -16,5 +16,5 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - println!("cargo:rerun-if-env-changed=SUBSTRATE_CLI_IMPL_VERSION"); + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/prepare-worker/build.rs b/node/core/pvf/prepare-worker/build.rs index abebb1719152..998da2eeb7de 100644 --- a/node/core/pvf/prepare-worker/build.rs +++ b/node/core/pvf/prepare-worker/build.rs @@ -16,5 +16,5 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - println!("cargo:rerun-if-env-changed=SUBSTRATE_CLI_IMPL_VERSION"); + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/src/testing.rs b/node/core/pvf/src/testing.rs index fc7807958cfb..554c2b10ee31 100644 --- a/node/core/pvf/src/testing.rs +++ b/node/core/pvf/src/testing.rs @@ -53,7 +53,7 @@ pub fn validate_candidate( /// the appropriate worker, making the executable that can be used for spawning workers. #[macro_export] macro_rules! decl_puppet_worker_main { - () => { + ($worker_version:expr) => { fn main() { $crate::sp_tracing::try_init_simple(); @@ -82,10 +82,10 @@ macro_rules! decl_puppet_worker_main { std::thread::sleep(std::time::Duration::from_secs(5)); }, "prepare-worker" => { - $crate::prepare_worker_entrypoint(&socket_path, node_version); + $crate::prepare_worker_entrypoint(&socket_path, node_version, $worker_version); }, "execute-worker" => { - $crate::execute_worker_entrypoint(&socket_path, node_version); + $crate::execute_worker_entrypoint(&socket_path, node_version, $worker_version); }, other => panic!("unknown subcommand: {}", other), } diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index fede72966a32..1f4292488512 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -12,12 +12,13 @@ publish = false name = "malus" path = "src/malus.rs" +# Use artifact dependencies once stable. +# See https://github.com/rust-lang/cargo/issues/9096. [[bin]] name = "polkadot-execute-worker" path = "../core/pvf/execute-worker/src/main.rs" # Prevent rustdoc error. Already documented from top-level Cargo.toml. doc = false - [[bin]] name = "polkadot-prepare-worker" path = "../core/pvf/prepare-worker/src/main.rs" diff --git a/node/service/build.rs b/node/service/build.rs index abebb1719152..64403e6fa4d9 100644 --- a/node/service/build.rs +++ b/node/service/build.rs @@ -16,5 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - println!("cargo:rerun-if-env-changed=SUBSTRATE_CLI_IMPL_VERSION"); + // We do the node/worker version check in this crate. Make sure we always rebuild when the + // version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 86b79ac12902..a6659bf06215 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -240,7 +240,7 @@ pub enum Error { InvalidWorkerBinaries { prep_worker_path: PathBuf, exec_worker_path: PathBuf }, #[cfg(feature = "full-node")] - #[error("Worker binaries could not be found at workers path ({given_workers_path:?}), polkadot binary directory, or /usr/lib/polkadot")] + #[error("Worker binaries could not be found at given workers path ({given_workers_path:?}), polkadot binary directory, or /usr/lib/polkadot")] MissingWorkerBinaries { given_workers_path: Option }, #[cfg(feature = "full-node")] @@ -618,7 +618,10 @@ pub struct NewFullParams { pub enable_beefy: bool, pub jaeger_agent: Option, pub telemetry_worker_handle: Option, + /// An optional path to a directory containing the workers. pub workers_path: Option, + /// Optional custom names for the prepare and execute workers (mainly for tests). + pub workers_names: Option<(String, String)>, pub overseer_enable_anyways: bool, pub overseer_gen: OverseerGenerator, pub overseer_message_channel_capacity_override: Option, @@ -695,6 +698,7 @@ pub fn new_full( jaeger_agent, telemetry_worker_handle, workers_path, + workers_names, overseer_enable_anyways, overseer_gen, overseer_message_channel_capacity_override, @@ -884,7 +888,8 @@ pub fn new_full( slot_duration_millis: slot_duration.as_millis() as u64, }; - let (prep_worker_path, exec_worker_path) = workers::determine_workers_paths(workers_path)?; + let (prep_worker_path, exec_worker_path) = + workers::determine_workers_paths(workers_path, workers_names)?; let candidate_validation_config = CandidateValidationConfig { artifacts_cache_path: config diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index 4c4ace5f1300..bd8d5ebc21e3 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -36,6 +36,8 @@ fn workers_lib_path_override() -> &'static Mutex> { OVERRIDE.get_or_init(|| Mutex::new(None)) } +/// Determines the final set of paths to use for the PVF workers. +/// /// 1. Get the binaries from the workers path if it is passed in, or consider all possible /// locations on the filesystem in order and get all sets of paths at which the binaries exist. /// @@ -51,8 +53,9 @@ fn workers_lib_path_override() -> &'static Mutex> { /// 6. At this point the final set of paths should be good to use. pub fn determine_workers_paths( given_workers_path: Option, + workers_names: Option<(String, String)>, ) -> Result<(PathBuf, PathBuf), Error> { - let mut workers_paths = list_workers_paths(given_workers_path.clone())?; + let mut workers_paths = list_workers_paths(given_workers_path.clone(), workers_names)?; if workers_paths.is_empty() { return Err(Error::MissingWorkerBinaries { given_workers_path }) } else if workers_paths.len() > 1 { @@ -94,6 +97,7 @@ pub fn determine_workers_paths( /// locations on the filesystem. See `new_full`. fn list_workers_paths( given_workers_path: Option, + workers_names: Option<(String, String)>, ) -> Result, Error> { if let Some(path) = given_workers_path { log::trace!("Using explicitly provided workers path {:?}", path); @@ -102,10 +106,7 @@ fn list_workers_paths( return Ok(vec![(path.clone(), path)]) } - let mut prep_worker = path.clone(); - let mut exec_worker = path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + let (prep_worker, exec_worker) = build_worker_paths(path, workers_names); // Check if both workers exist. Otherwise return an empty vector which results in an error. return if prep_worker.exists() && exec_worker.exists() { @@ -119,6 +120,7 @@ fn list_workers_paths( let mut workers_paths = vec![]; + // Consider the polkadot binary directory. { let mut exe_path = std::env::current_exe()?; let _ = exe_path.pop(); // executable file will always have a parent directory. @@ -127,10 +129,8 @@ fn list_workers_paths( exe_path = path_override.clone(); } - let mut prep_worker = exe_path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = exe_path.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + let (prep_worker, exec_worker) = + build_worker_paths(exe_path.clone(), workers_names.clone()); // Add to set if both workers exist. Warn on partial installs. let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); @@ -144,6 +144,7 @@ fn list_workers_paths( } } + // Consider the /usr/lib/polkadot/ directory. { #[allow(unused_mut)] let mut lib_path = PathBuf::from("/usr/lib/polkadot"); @@ -152,10 +153,7 @@ fn list_workers_paths( lib_path = path_override.clone(); } - let mut prep_worker = lib_path.clone(); - prep_worker.push(polkadot_node_core_pvf::PREPARE_BINARY_NAME); - let mut exec_worker = lib_path.clone(); - exec_worker.push(polkadot_node_core_pvf::EXECUTE_BINARY_NAME); + let (prep_worker, exec_worker) = build_worker_paths(lib_path, workers_names); // Add to set if both workers exist. Warn on partial installs. let (prep_worker_exists, exec_worker_exists) = (prep_worker.exists(), exec_worker.exists()); @@ -172,6 +170,23 @@ fn list_workers_paths( Ok(workers_paths) } +fn build_worker_paths( + worker_dir: PathBuf, + workers_names: Option<(String, String)>, +) -> (PathBuf, PathBuf) { + let (prep_worker_name, exec_worker_name) = workers_names.unwrap_or(( + polkadot_node_core_pvf::PREPARE_BINARY_NAME.to_string(), + polkadot_node_core_pvf::EXECUTE_BINARY_NAME.to_string(), + )); + + let mut prep_worker = worker_dir.clone(); + prep_worker.push(prep_worker_name); + let mut exec_worker = worker_dir; + exec_worker.push(exec_worker_name); + + (prep_worker, exec_worker) +} + // Tests that set up a temporary directory tree according to what scenario we want to test and // run worker detection. #[cfg(test)] @@ -252,8 +267,8 @@ echo {} // Try with provided workers path that has missing binaries. assert_matches!( - determine_workers_paths(Some(given_workers_path.clone())), - Err(Error::MissingWorkerBinaries(Some(p))) if p == given_workers_path + determine_workers_paths(Some(given_workers_path.clone()), None), + Err(Error::MissingWorkerBinaries { given_workers_path: Some(p) }) if p == given_workers_path ); // Try with provided workers path that has not executable binaries. @@ -264,15 +279,15 @@ echo {} write_worker_exe(&execute_worker_path)?; fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o644))?; assert_matches!( - determine_workers_paths(Some(given_workers_path.clone())), - Err(Error::InvalidWorkerBinaries(p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path + determine_workers_paths(Some(given_workers_path.clone()), None), + Err(Error::InvalidWorkerBinaries { prep_worker_path: p1, exec_worker_path: p2 }) if p1 == prepare_worker_path && p2 == execute_worker_path ); // Try with valid workers directory path that has executable binaries. fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o744))?; fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o744))?; assert_matches!( - determine_workers_paths(Some(given_workers_path)), + determine_workers_paths(Some(given_workers_path), None), Ok((p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path ); @@ -280,7 +295,7 @@ echo {} let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); write_worker_exe(&given_workers_path)?; assert_matches!( - determine_workers_paths(Some(given_workers_path.clone())), + determine_workers_paths(Some(given_workers_path.clone()), None), Ok((p1, p2)) if p1 == given_workers_path && p2 == given_workers_path ); @@ -294,30 +309,45 @@ echo {} fn missing_workers_paths_throws_error() { with_temp_dir_structure(|tempdir| { // Try with both binaries missing. - assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + assert_matches!( + determine_workers_paths(None, None), + Err(Error::MissingWorkerBinaries { given_workers_path: None }) + ); // Try with only prep worker (at bin location). let prepare_worker_path = tempdir.join("usr/bin/polkadot-prepare-worker"); write_worker_exe(&prepare_worker_path)?; - assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + assert_matches!( + determine_workers_paths(None, None), + Err(Error::MissingWorkerBinaries { given_workers_path: None }) + ); // Try with only exec worker (at bin location). fs::remove_file(&prepare_worker_path)?; let execute_worker_path = tempdir.join("usr/bin/polkadot-execute-worker"); write_worker_exe(&execute_worker_path)?; - assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + assert_matches!( + determine_workers_paths(None, None), + Err(Error::MissingWorkerBinaries { given_workers_path: None }) + ); // Try with only prep worker (at lib location). fs::remove_file(&execute_worker_path)?; let prepare_worker_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); write_worker_exe(&prepare_worker_path)?; - assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + assert_matches!( + determine_workers_paths(None, None), + Err(Error::MissingWorkerBinaries { given_workers_path: None }) + ); // Try with only exec worker (at lib location). fs::remove_file(&prepare_worker_path)?; let execute_worker_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); write_worker_exe(execute_worker_path)?; - assert_matches!(determine_workers_paths(None), Err(Error::MissingWorkerBinaries(None))); + assert_matches!( + determine_workers_paths(None, None), + Err(Error::MissingWorkerBinaries { given_workers_path: None }) + ); Ok(()) }) @@ -341,7 +371,7 @@ echo {} write_worker_exe(&execute_worker_lib_path)?; assert_matches!( - list_workers_paths(None), + list_workers_paths(None, None), Ok(v) if v == vec![(prepare_worker_bin_path, execute_worker_bin_path), (prepare_worker_lib_path, execute_worker_lib_path)] ); @@ -350,6 +380,34 @@ echo {} .unwrap(); } + #[test] + #[serial] + fn should_find_workers_with_custom_names_at_all_locations() { + with_temp_dir_structure(|tempdir| { + let (prep_worker_name, exec_worker_name) = ("test-prepare", "test-execute"); + + let prepare_worker_bin_path = tempdir.join("usr/bin").join(prep_worker_name); + write_worker_exe(&prepare_worker_bin_path)?; + + let execute_worker_bin_path = tempdir.join("usr/bin").join(exec_worker_name); + write_worker_exe(&execute_worker_bin_path)?; + + let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot").join(prep_worker_name); + write_worker_exe(&prepare_worker_lib_path)?; + + let execute_worker_lib_path = tempdir.join("usr/lib/polkadot").join(exec_worker_name); + write_worker_exe(&execute_worker_lib_path)?; + + assert_matches!( + list_workers_paths(None, Some((prep_worker_name.into(), exec_worker_name.into()))), + Ok(v) if v == vec![(prepare_worker_bin_path, execute_worker_bin_path), (prepare_worker_lib_path, execute_worker_lib_path)] + ); + + Ok(()) + }) + .unwrap(); + } + #[test] #[serial] fn workers_version_mismatch_throws_error() { @@ -362,8 +420,8 @@ echo {} write_worker_exe_invalid_version(&prepare_worker_bin_path, bad_version)?; write_worker_exe(&execute_worker_bin_path)?; assert_matches!( - determine_workers_paths(None), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + determine_workers_paths(None, None), + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION ); // Workers at lib location return bad version. @@ -372,8 +430,8 @@ echo {} write_worker_exe(&prepare_worker_lib_path)?; write_worker_exe_invalid_version(&execute_worker_lib_path, bad_version)?; assert_matches!( - determine_workers_paths(None), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + determine_workers_paths(None, None), + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION ); // Workers at provided workers location return bad version. @@ -383,16 +441,16 @@ echo {} write_worker_exe_invalid_version(&prepare_worker_path, bad_version)?; write_worker_exe_invalid_version(&execute_worker_path, bad_version)?; assert_matches!( - determine_workers_paths(Some(given_workers_path)), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + determine_workers_paths(Some(given_workers_path), None), + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION ); // Given worker binary returns bad version. let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); write_worker_exe_invalid_version(&given_workers_path, bad_version)?; assert_matches!( - determine_workers_paths(Some(given_workers_path)), - Err(Error::WorkerBinaryVersionMismatch(v1, v2)) if v1 == bad_version && v2 == NODE_VERSION + determine_workers_paths(Some(given_workers_path), None), + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION ); Ok(()) @@ -412,7 +470,7 @@ echo {} write_worker_exe(&execute_worker_bin_path)?; assert_matches!( - determine_workers_paths(None), + determine_workers_paths(None, None), Ok((p1, p2)) if p1 == prepare_worker_bin_path && p2 == execute_worker_bin_path ); @@ -429,7 +487,7 @@ echo {} write_worker_exe(&execute_worker_lib_path)?; assert_matches!( - determine_workers_paths(None), + determine_workers_paths(None, None), Ok((p1, p2)) if p1 == prepare_worker_lib_path && p2 == execute_worker_lib_path ); diff --git a/node/test/service/Cargo.toml b/node/test/service/Cargo.toml index d2c16cbb3e96..bc92e8ad6028 100644 --- a/node/test/service/Cargo.toml +++ b/node/test/service/Cargo.toml @@ -4,12 +4,13 @@ version.workspace = true authors.workspace = true edition.workspace = true +# Use artifact dependencies once stable. +# See https://github.com/rust-lang/cargo/issues/9096. [[bin]] -name = "polkadot-execute-worker" +name = "polkadot-execute-worker-test-service" path = "../../core/pvf/execute-worker/src/main.rs" - [[bin]] -name = "polkadot-prepare-worker" +name = "polkadot-prepare-worker-test-service" path = "../../core/pvf/prepare-worker/src/main.rs" [dependencies] diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 9138472d9e5e..1684ee9f7ea3 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -83,6 +83,10 @@ pub fn new_full( jaeger_agent: None, telemetry_worker_handle: None, workers_path, + workers_names: Some(( + "polkadot-prepare-worker-test-service".into(), + "polkadot-execute-worker-test-service".into(), + )), overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/adder/collator/Cargo.toml b/parachain/test-parachains/adder/collator/Cargo.toml index 29a10069e3e0..79fff4566496 100644 --- a/parachain/test-parachains/adder/collator/Cargo.toml +++ b/parachain/test-parachains/adder/collator/Cargo.toml @@ -45,3 +45,6 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/parachain/test-parachains/adder/collator/bin/puppet_worker.rs b/parachain/test-parachains/adder/collator/bin/puppet_worker.rs index 7f93519d8454..9ad99d6c9c66 100644 --- a/parachain/test-parachains/adder/collator/bin/puppet_worker.rs +++ b/parachain/test-parachains/adder/collator/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(); +polkadot_node_core_pvf::decl_puppet_worker_main!(env!("SUBSTRATE_CLI_IMPL_VERSION")); diff --git a/parachain/test-parachains/adder/collator/build.rs b/parachain/test-parachains/adder/collator/build.rs new file mode 100644 index 000000000000..40e9f832586e --- /dev/null +++ b/parachain/test-parachains/adder/collator/build.rs @@ -0,0 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); +} diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index f671acea384f..3afc0e9af063 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -65,6 +65,7 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, workers_path: None, + workers_names: None, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/undying/collator/Cargo.toml b/parachain/test-parachains/undying/collator/Cargo.toml index f63757a20958..6669f0f47ed2 100644 --- a/parachain/test-parachains/undying/collator/Cargo.toml +++ b/parachain/test-parachains/undying/collator/Cargo.toml @@ -45,3 +45,6 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/parachain/test-parachains/undying/collator/bin/puppet_worker.rs b/parachain/test-parachains/undying/collator/bin/puppet_worker.rs index 7f93519d8454..9ad99d6c9c66 100644 --- a/parachain/test-parachains/undying/collator/bin/puppet_worker.rs +++ b/parachain/test-parachains/undying/collator/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(); +polkadot_node_core_pvf::decl_puppet_worker_main!(env!("SUBSTRATE_CLI_IMPL_VERSION")); diff --git a/parachain/test-parachains/undying/collator/build.rs b/parachain/test-parachains/undying/collator/build.rs new file mode 100644 index 000000000000..40e9f832586e --- /dev/null +++ b/parachain/test-parachains/undying/collator/build.rs @@ -0,0 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); +} diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 9e24f0bb7aca..363466477b02 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -65,6 +65,7 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, workers_path: None, + workers_names: None, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, From 751dd046614f29f2c57424fd0a4d5465d3dd02cf Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 17 Jul 2023 21:13:47 +0200 Subject: [PATCH 26/48] Rebuild nodes on version change --- Cargo.lock | 1 + cli/build.rs | 3 +++ node/malus/Cargo.toml | 3 +++ node/malus/build.rs | 21 +++++++++++++++++++++ node/service/build.rs | 3 --- 5 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 node/malus/build.rs diff --git a/Cargo.lock b/Cargo.lock index 16e687455b04..5708654360e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8309,6 +8309,7 @@ dependencies = [ "sp-core", "sp-keystore", "sp-tracing", + "substrate-build-script-utils", "tracing-gum", ] diff --git a/cli/build.rs b/cli/build.rs index 018ea752a009..483cc04163fc 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -19,4 +19,7 @@ fn main() { println!("cargo:rustc-cfg=build_type=\"{}\"", profile); } substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the node when the version + // changes. + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 1f4292488512..6588113213f5 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -61,3 +61,6 @@ fast-runtime = ["polkadot-cli/fast-runtime"] polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = { version = "0.3.21", features = ["thread-pool"] } + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/malus/build.rs b/node/malus/build.rs new file mode 100644 index 000000000000..e53dc08120f0 --- /dev/null +++ b/node/malus/build.rs @@ -0,0 +1,21 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + // For the node/worker version check, make sure we always rebuild the node when the version + // changes. + substrate_build_script_utils::rerun_if_git_head_changed(); +} diff --git a/node/service/build.rs b/node/service/build.rs index 64403e6fa4d9..40e9f832586e 100644 --- a/node/service/build.rs +++ b/node/service/build.rs @@ -16,7 +16,4 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - // We do the node/worker version check in this crate. Make sure we always rebuild when the - // version changes. - substrate_build_script_utils::rerun_if_git_head_changed(); } From 76c16f3acfde0b5543b5f1ff080c13c45e8cb94a Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 18 Jul 2023 18:32:42 +0200 Subject: [PATCH 27/48] Fix more issues --- Cargo.lock | 9 +++++ node/core/pvf/build.rs | 3 ++ node/core/pvf/common/src/worker/mod.rs | 2 + node/core/pvf/execute-worker/build.rs | 2 + node/core/pvf/prepare-worker/build.rs | 2 + node/core/pvf/src/testing.rs | 36 +++++++++-------- node/service/src/lib.rs | 8 +++- node/service/src/workers.rs | 28 ++++++++----- node/test/service/Cargo.toml | 3 ++ node/test/service/build.rs | 21 ++++++++++ node/test/service/src/lib.rs | 39 +++++++++++-------- .../test-parachains/adder/collator/Cargo.toml | 15 +++++++ .../test-parachains/adder/collator/build.rs | 3 ++ .../adder/collator/src/main.rs | 5 ++- .../adder/collator/tests/integration.rs | 9 ++++- .../undying/collator/Cargo.toml | 15 +++++++ .../test-parachains/undying/collator/build.rs | 3 ++ .../undying/collator/src/main.rs | 5 ++- .../undying/collator/tests/integration.rs | 9 ++++- 19 files changed, 167 insertions(+), 50 deletions(-) create mode 100644 node/test/service/build.rs diff --git a/Cargo.lock b/Cargo.lock index 5708654360e5..8e1bbf171824 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8426,6 +8426,7 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-tracing", + "substrate-build-script-utils", "substrate-test-client", "substrate-test-utils", "tempfile", @@ -12510,6 +12511,9 @@ dependencies = [ "parity-scale-codec", "polkadot-cli", "polkadot-node-core-pvf", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", + "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-parachain", @@ -12520,6 +12524,7 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", + "sp-tracing", "substrate-build-script-utils", "substrate-test-utils", "test-parachain-adder", @@ -12559,6 +12564,9 @@ dependencies = [ "parity-scale-codec", "polkadot-cli", "polkadot-node-core-pvf", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", + "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-parachain", @@ -12569,6 +12577,7 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", + "sp-tracing", "substrate-build-script-utils", "substrate-test-utils", "test-parachain-undying", diff --git a/node/core/pvf/build.rs b/node/core/pvf/build.rs index 40e9f832586e..05df00e537f0 100644 --- a/node/core/pvf/build.rs +++ b/node/core/pvf/build.rs @@ -16,4 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the test worker when the + // version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index 464a11df9979..8082231142db 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -41,6 +41,8 @@ macro_rules! decl_worker_main { } fn main() { + // NOTE: We don't want sp_tracing as a dependency in this crate to reduce this size. + // This means callers of this macro must depend on it. ::sp_tracing::try_init_simple(); let args = std::env::args().collect::>(); diff --git a/node/core/pvf/execute-worker/build.rs b/node/core/pvf/execute-worker/build.rs index 998da2eeb7de..7c249f50ccc4 100644 --- a/node/core/pvf/execute-worker/build.rs +++ b/node/core/pvf/execute-worker/build.rs @@ -16,5 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the worker when the version + // changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/prepare-worker/build.rs b/node/core/pvf/prepare-worker/build.rs index 998da2eeb7de..7c249f50ccc4 100644 --- a/node/core/pvf/prepare-worker/build.rs +++ b/node/core/pvf/prepare-worker/build.rs @@ -16,5 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the worker when the version + // changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/src/testing.rs b/node/core/pvf/src/testing.rs index 554c2b10ee31..00b51537f570 100644 --- a/node/core/pvf/src/testing.rs +++ b/node/core/pvf/src/testing.rs @@ -58,10 +58,27 @@ macro_rules! decl_puppet_worker_main { $crate::sp_tracing::try_init_simple(); let args = std::env::args().collect::>(); - if args.len() < 3 { + if args.len() == 1 { panic!("wrong number of arguments"); } + let entrypoint = match args[1].as_ref() { + "--version" | "-v" => { + println!("{}", $worker_version); + return + }, + "exit" => { + std::process::exit(1); + }, + "sleep" => { + std::thread::sleep(std::time::Duration::from_secs(5)); + return + }, + "prepare-worker" => $crate::prepare_worker_entrypoint, + "execute-worker" => $crate::execute_worker_entrypoint, + other => panic!("unknown subcommand: {}", other), + }; + let mut node_version = None; let mut socket_path: &str = ""; @@ -73,22 +90,7 @@ macro_rules! decl_puppet_worker_main { } } - let subcommand = &args[1]; - match subcommand.as_ref() { - "exit" => { - std::process::exit(1); - }, - "sleep" => { - std::thread::sleep(std::time::Duration::from_secs(5)); - }, - "prepare-worker" => { - $crate::prepare_worker_entrypoint(&socket_path, node_version, $worker_version); - }, - "execute-worker" => { - $crate::execute_worker_entrypoint(&socket_path, node_version, $worker_version); - }, - other => panic!("unknown subcommand: {}", other), - } + entrypoint(&socket_path, node_version, $worker_version); } }; } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index a6659bf06215..d50007f9bd22 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -244,8 +244,12 @@ pub enum Error { MissingWorkerBinaries { given_workers_path: Option }, #[cfg(feature = "full-node")] - #[error("Version of worker binaries ({worker_version}) is different from node version ({node_version})")] - WorkerBinaryVersionMismatch { worker_version: String, node_version: String }, + #[error("Version of worker binary ({worker_version}) is different from node version ({node_version}), worker_path: {worker_path}")] + WorkerBinaryVersionMismatch { + worker_version: String, + node_version: String, + worker_path: PathBuf, + }, } /// Identifies the variant of the chain. diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index bd8d5ebc21e3..af1da5e758ec 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -75,7 +75,11 @@ pub fn determine_workers_paths( .trim() .to_string(); if worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch { worker_version, node_version }) + return Err(Error::WorkerBinaryVersionMismatch { + worker_version, + node_version, + worker_path: prep_worker_path, + }) } let worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; let worker_version = std::str::from_utf8(&worker_version) @@ -83,7 +87,11 @@ pub fn determine_workers_paths( .trim() .to_string(); if worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch { worker_version, node_version }) + return Err(Error::WorkerBinaryVersionMismatch { + worker_version, + node_version, + worker_path: exec_worker_path, + }) } // Paths are good to use. @@ -421,17 +429,19 @@ echo {} write_worker_exe(&execute_worker_bin_path)?; assert_matches!( determine_workers_paths(None, None), - Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == prepare_worker_bin_path ); // Workers at lib location return bad version. - let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot-prepare-worker"); - let execute_worker_lib_path = tempdir.join("usr/lib/polkadot-execute-worker"); + fs::remove_file(prepare_worker_bin_path)?; + fs::remove_file(execute_worker_bin_path)?; + let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); + let execute_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); write_worker_exe(&prepare_worker_lib_path)?; write_worker_exe_invalid_version(&execute_worker_lib_path, bad_version)?; assert_matches!( determine_workers_paths(None, None), - Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == execute_worker_lib_path ); // Workers at provided workers location return bad version. @@ -442,15 +452,15 @@ echo {} write_worker_exe_invalid_version(&execute_worker_path, bad_version)?; assert_matches!( determine_workers_paths(Some(given_workers_path), None), - Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == prepare_worker_path ); // Given worker binary returns bad version. let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); write_worker_exe_invalid_version(&given_workers_path, bad_version)?; assert_matches!( - determine_workers_paths(Some(given_workers_path), None), - Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2 }) if v1 == bad_version && v2 == NODE_VERSION + determine_workers_paths(Some(given_workers_path.clone()), None), + Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == given_workers_path ); Ok(()) diff --git a/node/test/service/Cargo.toml b/node/test/service/Cargo.toml index bc92e8ad6028..c2a3d33501cc 100644 --- a/node/test/service/Cargo.toml +++ b/node/test/service/Cargo.toml @@ -75,6 +75,9 @@ serde_json = "1.0.96" substrate-test-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + [features] runtime-metrics=["polkadot-test-runtime/runtime-metrics"] runtime-benchmarks=[ diff --git a/node/test/service/build.rs b/node/test/service/build.rs new file mode 100644 index 000000000000..02fd01a6ee28 --- /dev/null +++ b/node/test/service/build.rs @@ -0,0 +1,21 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + // For the node/worker version check, make sure we always rebuild the test workers when the + // version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); +} diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 1684ee9f7ea3..eab05299fde0 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -73,7 +73,14 @@ pub fn new_full( config: Configuration, is_collator: IsCollator, workers_path: Option, + workers_names: Option<(String, String)>, ) -> Result { + let workers_path = Some(workers_path.unwrap_or_else(get_relative_workers_path_for_test)); + let workers_names = Some(workers_names.unwrap_or(( + "polkadot-prepare-worker-test-service".into(), + "polkadot-execute-worker-test-service".into(), + ))); + polkadot_service::new_full( config, polkadot_service::NewFullParams { @@ -83,10 +90,7 @@ pub fn new_full( jaeger_agent: None, telemetry_worker_handle: None, workers_path, - workers_names: Some(( - "polkadot-prepare-worker-test-service".into(), - "polkadot-execute-worker-test-service".into(), - )), + workers_names, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, @@ -201,20 +205,12 @@ pub fn node_config( /// Run a test validator node that uses the test runtime and specified `config`. pub fn run_validator_node( config: Configuration, - mut worker_program_path: Option, + worker_program_path: Option, + workers_names: Option<(String, String)>, ) -> PolkadotTestNode { let multiaddr = config.network.listen_addresses[0].clone(); - if let None = worker_program_path { - // If no explicit worker path is passed in, we need to specify it ourselves as test binaries - // are in the "deps/" directory, one level below where the worker binaries are generated. - let mut exe_path = std::env::current_exe() - .expect("for test purposes it's reasonable to expect that this will not fail"); - let _ = exe_path.pop(); - let _ = exe_path.pop(); - worker_program_path = Some(exe_path); - } let NewFull { task_manager, client, network, rpc_handlers, overseer_handle, .. } = - new_full(config, IsCollator::No, worker_program_path) + new_full(config, IsCollator::No, worker_program_path, workers_names) .expect("could not create Polkadot test service"); let overseer_handle = overseer_handle.expect("test node must have an overseer handle"); @@ -242,11 +238,12 @@ pub fn run_collator_node( storage_update_func: impl Fn(), boot_nodes: Vec, collator_pair: CollatorPair, + workers_names: Option<(String, String)>, ) -> PolkadotTestNode { let config = node_config(storage_update_func, tokio_handle, key, boot_nodes, false); let multiaddr = config.network.listen_addresses[0].clone(); let NewFull { task_manager, client, network, rpc_handlers, overseer_handle, .. } = - new_full(config, IsCollator::Yes(collator_pair), None) + new_full(config, IsCollator::Yes(collator_pair), None, workers_names) .expect("could not create Polkadot test service"); let overseer_handle = overseer_handle.expect("test node must have an overseer handle"); @@ -256,6 +253,16 @@ pub fn run_collator_node( PolkadotTestNode { task_manager, client, overseer_handle, addr, rpc_handlers } } +fn get_relative_workers_path_for_test() -> PathBuf { + // If no explicit worker path is passed in, we need to specify it ourselves as test binaries + // are in the "deps/" directory, one level below where the worker binaries are generated. + let mut exe_path = std::env::current_exe() + .expect("for test purposes it's reasonable to expect that this will not fail"); + let _ = exe_path.pop(); + let _ = exe_path.pop(); + exe_path +} + /// A Polkadot test node instance used for testing. pub struct PolkadotTestNode { /// `TaskManager`'s instance. diff --git a/parachain/test-parachains/adder/collator/Cargo.toml b/parachain/test-parachains/adder/collator/Cargo.toml index 79fff4566496..f0fd8c9b9aaa 100644 --- a/parachain/test-parachains/adder/collator/Cargo.toml +++ b/parachain/test-parachains/adder/collator/Cargo.toml @@ -13,6 +13,15 @@ path = "src/main.rs" name = "adder_collator_puppet_worker" path = "bin/puppet_worker.rs" +# Use artifact dependencies once stable. +# See https://github.com/rust-lang/cargo/issues/9096. +[[bin]] +name = "polkadot-execute-worker-adder-collator" +path = "../../../../node/core/pvf/execute-worker/src/main.rs" +[[bin]] +name = "polkadot-prepare-worker-adder-collator" +path = "../../../../node/core/pvf/prepare-worker/src/main.rs" + [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } clap = { version = "4.0.9", features = ["derive"] } @@ -36,6 +45,12 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master # a big problem since it is used transitively anyway. polkadot-node-core-pvf = { path = "../../../../node/core/pvf" } +# Required for binary workers to build. +polkadot-node-core-pvf-common = { path = "../../../../node/core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../../../../node/core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../../../../node/core/pvf/prepare-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [dev-dependencies] polkadot-parachain = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } diff --git a/parachain/test-parachains/adder/collator/build.rs b/parachain/test-parachains/adder/collator/build.rs index 40e9f832586e..05df00e537f0 100644 --- a/parachain/test-parachains/adder/collator/build.rs +++ b/parachain/test-parachains/adder/collator/build.rs @@ -16,4 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the test worker when the + // version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index 3afc0e9af063..d03c64fbf5cc 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -65,7 +65,10 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, workers_path: None, - workers_names: None, + workers_names: Some(( + "polkadot-prepare-worker-adder-collator".into(), + "polkadot-execute-worker-adder-collator".into(), + )), overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/adder/collator/tests/integration.rs b/parachain/test-parachains/adder/collator/tests/integration.rs index 9ab1c0c337a6..13e57bd82505 100644 --- a/parachain/test-parachains/adder/collator/tests/integration.rs +++ b/parachain/test-parachains/adder/collator/tests/integration.rs @@ -19,6 +19,9 @@ const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_adder_collator_puppet_worker"); +const WORKERS_NAMES: (&str, &str) = + ("polkadot-prepare-worker-adder-collator", "polkadot-execute-worker-adder-collator"); + // If this test is failing, make sure to run all tests with the `real-overseer` feature being enabled. #[substrate_test_utils::test(flavor = "multi_thread")] @@ -41,7 +44,8 @@ async fn collating_using_adder_collator() { ); // start alice - let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); + let alice = + polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into()), None); let bob_config = polkadot_test_service::node_config( || {}, @@ -52,7 +56,7 @@ async fn collating_using_adder_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into()), None); let collator = test_parachain_adder_collator::Collator::new(); @@ -69,6 +73,7 @@ async fn collating_using_adder_collator() { || {}, vec![alice.addr.clone(), bob.addr.clone()], collator.collator_key(), + Some((WORKERS_NAMES.0.into(), WORKERS_NAMES.1.into())), ); charlie diff --git a/parachain/test-parachains/undying/collator/Cargo.toml b/parachain/test-parachains/undying/collator/Cargo.toml index 6669f0f47ed2..4cf944ae6270 100644 --- a/parachain/test-parachains/undying/collator/Cargo.toml +++ b/parachain/test-parachains/undying/collator/Cargo.toml @@ -13,6 +13,15 @@ path = "src/main.rs" name = "undying_collator_puppet_worker" path = "bin/puppet_worker.rs" +# Use artifact dependencies once stable. +# See https://github.com/rust-lang/cargo/issues/9096. +[[bin]] +name = "polkadot-execute-worker-undying-collator" +path = "../../../../node/core/pvf/execute-worker/src/main.rs" +[[bin]] +name = "polkadot-prepare-worker-undying-collator" +path = "../../../../node/core/pvf/prepare-worker/src/main.rs" + [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } clap = { version = "4.0.9", features = ["derive"] } @@ -36,6 +45,12 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master # a big problem since it is used transitively anyway. polkadot-node-core-pvf = { path = "../../../../node/core/pvf" } +# Required for binary workers to build. +polkadot-node-core-pvf-common = { path = "../../../../node/core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../../../../node/core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../../../../node/core/pvf/prepare-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [dev-dependencies] polkadot-parachain = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } diff --git a/parachain/test-parachains/undying/collator/build.rs b/parachain/test-parachains/undying/collator/build.rs index 40e9f832586e..05df00e537f0 100644 --- a/parachain/test-parachains/undying/collator/build.rs +++ b/parachain/test-parachains/undying/collator/build.rs @@ -16,4 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the test worker when the + // version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 363466477b02..431b868c5285 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -65,7 +65,10 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, workers_path: None, - workers_names: None, + workers_names: Some(( + "polkadot-prepare-worker-undying-collator".into(), + "polkadot-execute-worker-undying-collator".into(), + )), overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/undying/collator/tests/integration.rs b/parachain/test-parachains/undying/collator/tests/integration.rs index 8ca6eec9aa62..54c82c904b83 100644 --- a/parachain/test-parachains/undying/collator/tests/integration.rs +++ b/parachain/test-parachains/undying/collator/tests/integration.rs @@ -19,6 +19,9 @@ const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_undying_collator_puppet_worker"); +const WORKERS_NAMES: (&str, &str) = + ("polkadot-prepare-worker-undying-collator", "polkadot-execute-worker-undying-collator"); + // If this test is failing, make sure to run all tests with the `real-overseer` feature being enabled. #[substrate_test_utils::test(flavor = "multi_thread")] async fn collating_using_undying_collator() { @@ -40,7 +43,8 @@ async fn collating_using_undying_collator() { ); // start alice - let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); + let alice = + polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into()), None); let bob_config = polkadot_test_service::node_config( || {}, @@ -51,7 +55,7 @@ async fn collating_using_undying_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into()), None); let collator = test_parachain_undying_collator::Collator::new(1_000, 1); @@ -68,6 +72,7 @@ async fn collating_using_undying_collator() { || {}, vec![alice.addr.clone(), bob.addr.clone()], collator.collator_key(), + Some((WORKERS_NAMES.0.into(), WORKERS_NAMES.1.into())), ); charlie From a1e21dcf0b94733d536d2b3121e5e28a8f93fd6d Mon Sep 17 00:00:00 2001 From: Marcin S Date: Wed, 19 Jul 2023 10:36:26 +0200 Subject: [PATCH 28/48] Fix tests --- Cargo.lock | 17 ++----- cli/src/cli.rs | 19 ++++++++ cli/src/command.rs | 47 +++++++++++++++++++ node/core/pvf/build.rs | 2 +- node/malus/Cargo.toml | 2 +- node/malus/build.rs | 4 +- node/metrics/Cargo.toml | 6 +++ node/service/src/lib.rs | 23 +++++++-- node/service/src/workers.rs | 20 ++++---- node/test/service/Cargo.toml | 18 ------- node/test/service/build.rs | 21 --------- node/test/service/src/lib.rs | 26 ++-------- .../test-parachains/adder/collator/Cargo.toml | 15 ------ .../test-parachains/adder/collator/build.rs | 2 +- .../adder/collator/src/main.rs | 6 +-- .../adder/collator/tests/integration.rs | 9 +--- .../undying/collator/Cargo.toml | 15 ------ .../test-parachains/undying/collator/build.rs | 2 +- .../undying/collator/src/main.rs | 6 +-- .../undying/collator/tests/integration.rs | 9 +--- 20 files changed, 121 insertions(+), 148 deletions(-) delete mode 100644 node/test/service/build.rs diff --git a/Cargo.lock b/Cargo.lock index 8e1bbf171824..61dd83c6ef80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7574,6 +7574,9 @@ dependencies = [ "hyper", "log", "parity-scale-codec", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", + "polkadot-node-core-pvf-prepare-worker", "polkadot-primitives", "polkadot-test-service", "prioritized-metered-channel", @@ -7583,6 +7586,7 @@ dependencies = [ "sc-service", "sc-tracing", "sp-keyring", + "sp-tracing", "substrate-prometheus-endpoint", "substrate-test-utils", "tempfile", @@ -8388,9 +8392,6 @@ dependencies = [ "pallet-balances", "pallet-staking", "pallet-transaction-payment", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-overseer", @@ -8425,8 +8426,6 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-state-machine", - "sp-tracing", - "substrate-build-script-utils", "substrate-test-client", "substrate-test-utils", "tempfile", @@ -12511,9 +12510,6 @@ dependencies = [ "parity-scale-codec", "polkadot-cli", "polkadot-node-core-pvf", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-parachain", @@ -12524,7 +12520,6 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", - "sp-tracing", "substrate-build-script-utils", "substrate-test-utils", "test-parachain-adder", @@ -12564,9 +12559,6 @@ dependencies = [ "parity-scale-codec", "polkadot-cli", "polkadot-node-core-pvf", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-parachain", @@ -12577,7 +12569,6 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", - "sp-tracing", "substrate-build-script-utils", "substrate-test-utils", "test-parachain-undying", diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 78f50d38f60f..0a1b8d003d86 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -43,6 +43,14 @@ pub enum Subcommand { /// Revert the chain to a previous state. Revert(sc_cli::RevertCmd), + #[allow(missing_docs)] + #[command(name = "prepare-worker", hide = true)] + PvfPrepareWorker(ValidationWorkerCommand), + + #[allow(missing_docs)] + #[command(name = "execute-worker", hide = true)] + PvfExecuteWorker(ValidationWorkerCommand), + /// Sub-commands concerned with benchmarking. /// The pallet benchmarking moved to the `pallet` sub-command. #[command(subcommand)] @@ -68,6 +76,17 @@ pub enum Subcommand { ChainInfo(sc_cli::ChainInfoCmd), } +#[allow(missing_docs)] +#[derive(Debug, Parser)] +pub struct ValidationWorkerCommand { + /// The path to the validation host's socket. + #[arg(long)] + pub socket_path: String, + /// Calling node implementation version + #[arg(long)] + pub node_impl_version: String, +} + #[allow(missing_docs)] #[derive(Debug, Parser)] #[group(skip)] diff --git a/cli/src/command.rs b/cli/src/command.rs index a42d4694fd31..da93cc190e0a 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -293,6 +293,7 @@ where telemetry_worker_handle: None, workers_path: cli.run.workers_path, workers_names: None, + dont_use_external_workers: false, overseer_enable_anyways: false, overseer_gen, overseer_message_channel_capacity_override: cli @@ -427,6 +428,52 @@ pub fn run() -> Result<()> { )) })?) }, + Some(Subcommand::PvfPrepareWorker(cmd)) => { + let mut builder = sc_cli::LoggerBuilder::new(""); + builder.with_colors(false); + let _ = builder.init(); + + #[cfg(target_os = "android")] + { + return Err(sc_cli::Error::Input( + "PVF preparation workers are not supported under this platform".into(), + ) + .into()) + } + + #[cfg(not(target_os = "android"))] + { + polkadot_node_core_pvf_prepare_worker::worker_entrypoint( + &cmd.socket_path, + Some(&cmd.node_impl_version), + &cmd.node_impl_version, + ); + Ok(()) + } + }, + Some(Subcommand::PvfExecuteWorker(cmd)) => { + let mut builder = sc_cli::LoggerBuilder::new(""); + builder.with_colors(false); + let _ = builder.init(); + + #[cfg(target_os = "android")] + { + return Err(sc_cli::Error::Input( + "PVF execution workers are not supported under this platform".into(), + ) + .into()) + } + + #[cfg(not(target_os = "android"))] + { + polkadot_node_core_pvf_execute_worker::worker_entrypoint( + &cmd.socket_path, + Some(&cmd.node_impl_version), + &cmd.node_impl_version, + ); + Ok(()) + } + }, Some(Subcommand::Benchmark(cmd)) => { let runner = cli.create_runner(cmd)?; let chain_spec = &runner.config().chain_spec; diff --git a/node/core/pvf/build.rs b/node/core/pvf/build.rs index 05df00e537f0..8aa155080a31 100644 --- a/node/core/pvf/build.rs +++ b/node/core/pvf/build.rs @@ -16,7 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the test worker when the + // For the node/worker version check, make sure we always rebuild the puppet worker when the // version changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 6588113213f5..4aaf47070816 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -47,7 +47,7 @@ gum = { package = "tracing-gum", path = "../gum/" } erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } rand = "0.8.5" -# Required for binary workers to build. +# Required for worker binaries to build. polkadot-node-core-pvf-common = { path = "../core/pvf/common" } polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } diff --git a/node/malus/build.rs b/node/malus/build.rs index e53dc08120f0..545778102b40 100644 --- a/node/malus/build.rs +++ b/node/malus/build.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . fn main() { - // For the node/worker version check, make sure we always rebuild the node when the version - // changes. + // For the node/worker version check, make sure we always rebuild the malus node and binary + // workers when the version changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/metrics/Cargo.toml b/node/metrics/Cargo.toml index 7365d7593d7d..75f68fca0669 100644 --- a/node/metrics/Cargo.toml +++ b/node/metrics/Cargo.toml @@ -23,6 +23,12 @@ primitives = { package = "polkadot-primitives", path = "../../primitives/" } bs58 = { version = "0.4.0", features = ["alloc"] } log = "0.4.17" +# Required for worker binaries to build. +polkadot-node-core-pvf-common = { path = "../core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [dev-dependencies] assert_cmd = "2.0.4" tempfile = "3.2.0" diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index d50007f9bd22..ef1addf6a20e 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -240,8 +240,11 @@ pub enum Error { InvalidWorkerBinaries { prep_worker_path: PathBuf, exec_worker_path: PathBuf }, #[cfg(feature = "full-node")] - #[error("Worker binaries could not be found at given workers path ({given_workers_path:?}), polkadot binary directory, or /usr/lib/polkadot")] - MissingWorkerBinaries { given_workers_path: Option }, + #[error("Worker binaries could not be found at given workers path ({given_workers_path:?}), polkadot binary directory, or /usr/lib/polkadot, workers names: {workers_names:?}")] + MissingWorkerBinaries { + given_workers_path: Option, + workers_names: Option<(String, String)>, + }, #[cfg(feature = "full-node")] #[error("Version of worker binary ({worker_version}) is different from node version ({node_version}), worker_path: {worker_path}")] @@ -624,8 +627,10 @@ pub struct NewFullParams { pub telemetry_worker_handle: Option, /// An optional path to a directory containing the workers. pub workers_path: Option, - /// Optional custom names for the prepare and execute workers (mainly for tests). + /// Optional custom names for the prepare and execute workers. pub workers_names: Option<(String, String)>, + /// Use the current binary instead of relying on external workers. *Only for tests.* + pub dont_use_external_workers: bool, pub overseer_enable_anyways: bool, pub overseer_gen: OverseerGenerator, pub overseer_message_channel_capacity_override: Option, @@ -703,6 +708,7 @@ pub fn new_full( telemetry_worker_handle, workers_path, workers_names, + dont_use_external_workers, overseer_enable_anyways, overseer_gen, overseer_message_channel_capacity_override, @@ -892,8 +898,15 @@ pub fn new_full( slot_duration_millis: slot_duration.as_millis() as u64, }; - let (prep_worker_path, exec_worker_path) = - workers::determine_workers_paths(workers_path, workers_names)?; + let (prep_worker_path, exec_worker_path) = if dont_use_external_workers { + // Use the current binary. + let program_path = std::env::current_exe()?; + (program_path.clone(), program_path) + } else { + workers::determine_workers_paths(workers_path, workers_names)? + }; + log::info!("🚀 Using prepare-worker binary at: {:?}", prep_worker_path); + log::info!("🚀 Using execute-worker binary at: {:?}", exec_worker_path); let candidate_validation_config = CandidateValidationConfig { artifacts_cache_path: config diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index af1da5e758ec..d94721afce98 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -55,9 +55,9 @@ pub fn determine_workers_paths( given_workers_path: Option, workers_names: Option<(String, String)>, ) -> Result<(PathBuf, PathBuf), Error> { - let mut workers_paths = list_workers_paths(given_workers_path.clone(), workers_names)?; + let mut workers_paths = list_workers_paths(given_workers_path.clone(), workers_names.clone())?; if workers_paths.is_empty() { - return Err(Error::MissingWorkerBinaries { given_workers_path }) + return Err(Error::MissingWorkerBinaries { given_workers_path, workers_names }) } else if workers_paths.len() > 1 { log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); } @@ -94,10 +94,6 @@ pub fn determine_workers_paths( }) } - // Paths are good to use. - log::info!("using prepare-worker binary at: {:?}", prep_worker_path); - log::info!("using execute-worker binary at: {:?}", exec_worker_path); - Ok((prep_worker_path, exec_worker_path)) } @@ -276,7 +272,7 @@ echo {} // Try with provided workers path that has missing binaries. assert_matches!( determine_workers_paths(Some(given_workers_path.clone()), None), - Err(Error::MissingWorkerBinaries { given_workers_path: Some(p) }) if p == given_workers_path + Err(Error::MissingWorkerBinaries { given_workers_path: Some(p), workers_names: None }) if p == given_workers_path ); // Try with provided workers path that has not executable binaries. @@ -319,7 +315,7 @@ echo {} // Try with both binaries missing. assert_matches!( determine_workers_paths(None, None), - Err(Error::MissingWorkerBinaries { given_workers_path: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); // Try with only prep worker (at bin location). @@ -327,7 +323,7 @@ echo {} write_worker_exe(&prepare_worker_path)?; assert_matches!( determine_workers_paths(None, None), - Err(Error::MissingWorkerBinaries { given_workers_path: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); // Try with only exec worker (at bin location). @@ -336,7 +332,7 @@ echo {} write_worker_exe(&execute_worker_path)?; assert_matches!( determine_workers_paths(None, None), - Err(Error::MissingWorkerBinaries { given_workers_path: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); // Try with only prep worker (at lib location). @@ -345,7 +341,7 @@ echo {} write_worker_exe(&prepare_worker_path)?; assert_matches!( determine_workers_paths(None, None), - Err(Error::MissingWorkerBinaries { given_workers_path: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); // Try with only exec worker (at lib location). @@ -354,7 +350,7 @@ echo {} write_worker_exe(execute_worker_path)?; assert_matches!( determine_workers_paths(None, None), - Err(Error::MissingWorkerBinaries { given_workers_path: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); Ok(()) diff --git a/node/test/service/Cargo.toml b/node/test/service/Cargo.toml index c2a3d33501cc..504ee5beca74 100644 --- a/node/test/service/Cargo.toml +++ b/node/test/service/Cargo.toml @@ -4,15 +4,6 @@ version.workspace = true authors.workspace = true edition.workspace = true -# Use artifact dependencies once stable. -# See https://github.com/rust-lang/cargo/issues/9096. -[[bin]] -name = "polkadot-execute-worker-test-service" -path = "../../core/pvf/execute-worker/src/main.rs" -[[bin]] -name = "polkadot-prepare-worker-test-service" -path = "../../core/pvf/prepare-worker/src/main.rs" - [dependencies] futures = "0.3.21" hex = "0.4.3" @@ -63,21 +54,12 @@ sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" } substrate-test-client = { git = "https://github.com/paritytech/substrate", branch = "master" } -# Required for binary workers to build. -polkadot-node-core-pvf-common = { path = "../../core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../../core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../../core/pvf/prepare-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [dev-dependencies] pallet-balances = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } serde_json = "1.0.96" substrate-test-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } - [features] runtime-metrics=["polkadot-test-runtime/runtime-metrics"] runtime-benchmarks=[ diff --git a/node/test/service/build.rs b/node/test/service/build.rs deleted file mode 100644 index 02fd01a6ee28..000000000000 --- a/node/test/service/build.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - // For the node/worker version check, make sure we always rebuild the test workers when the - // version changes. - substrate_build_script_utils::rerun_if_git_head_changed(); -} diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index eab05299fde0..93ff2516cf3f 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -73,13 +73,8 @@ pub fn new_full( config: Configuration, is_collator: IsCollator, workers_path: Option, - workers_names: Option<(String, String)>, ) -> Result { - let workers_path = Some(workers_path.unwrap_or_else(get_relative_workers_path_for_test)); - let workers_names = Some(workers_names.unwrap_or(( - "polkadot-prepare-worker-test-service".into(), - "polkadot-execute-worker-test-service".into(), - ))); + let dont_use_external_workers = !workers_path.is_some(); polkadot_service::new_full( config, @@ -90,7 +85,8 @@ pub fn new_full( jaeger_agent: None, telemetry_worker_handle: None, workers_path, - workers_names, + workers_names: None, + dont_use_external_workers, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, @@ -206,11 +202,10 @@ pub fn node_config( pub fn run_validator_node( config: Configuration, worker_program_path: Option, - workers_names: Option<(String, String)>, ) -> PolkadotTestNode { let multiaddr = config.network.listen_addresses[0].clone(); let NewFull { task_manager, client, network, rpc_handlers, overseer_handle, .. } = - new_full(config, IsCollator::No, worker_program_path, workers_names) + new_full(config, IsCollator::No, worker_program_path) .expect("could not create Polkadot test service"); let overseer_handle = overseer_handle.expect("test node must have an overseer handle"); @@ -238,12 +233,11 @@ pub fn run_collator_node( storage_update_func: impl Fn(), boot_nodes: Vec, collator_pair: CollatorPair, - workers_names: Option<(String, String)>, ) -> PolkadotTestNode { let config = node_config(storage_update_func, tokio_handle, key, boot_nodes, false); let multiaddr = config.network.listen_addresses[0].clone(); let NewFull { task_manager, client, network, rpc_handlers, overseer_handle, .. } = - new_full(config, IsCollator::Yes(collator_pair), None, workers_names) + new_full(config, IsCollator::Yes(collator_pair), None) .expect("could not create Polkadot test service"); let overseer_handle = overseer_handle.expect("test node must have an overseer handle"); @@ -253,16 +247,6 @@ pub fn run_collator_node( PolkadotTestNode { task_manager, client, overseer_handle, addr, rpc_handlers } } -fn get_relative_workers_path_for_test() -> PathBuf { - // If no explicit worker path is passed in, we need to specify it ourselves as test binaries - // are in the "deps/" directory, one level below where the worker binaries are generated. - let mut exe_path = std::env::current_exe() - .expect("for test purposes it's reasonable to expect that this will not fail"); - let _ = exe_path.pop(); - let _ = exe_path.pop(); - exe_path -} - /// A Polkadot test node instance used for testing. pub struct PolkadotTestNode { /// `TaskManager`'s instance. diff --git a/parachain/test-parachains/adder/collator/Cargo.toml b/parachain/test-parachains/adder/collator/Cargo.toml index f0fd8c9b9aaa..79fff4566496 100644 --- a/parachain/test-parachains/adder/collator/Cargo.toml +++ b/parachain/test-parachains/adder/collator/Cargo.toml @@ -13,15 +13,6 @@ path = "src/main.rs" name = "adder_collator_puppet_worker" path = "bin/puppet_worker.rs" -# Use artifact dependencies once stable. -# See https://github.com/rust-lang/cargo/issues/9096. -[[bin]] -name = "polkadot-execute-worker-adder-collator" -path = "../../../../node/core/pvf/execute-worker/src/main.rs" -[[bin]] -name = "polkadot-prepare-worker-adder-collator" -path = "../../../../node/core/pvf/prepare-worker/src/main.rs" - [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } clap = { version = "4.0.9", features = ["derive"] } @@ -45,12 +36,6 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master # a big problem since it is used transitively anyway. polkadot-node-core-pvf = { path = "../../../../node/core/pvf" } -# Required for binary workers to build. -polkadot-node-core-pvf-common = { path = "../../../../node/core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../../../../node/core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../../../../node/core/pvf/prepare-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [dev-dependencies] polkadot-parachain = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } diff --git a/parachain/test-parachains/adder/collator/build.rs b/parachain/test-parachains/adder/collator/build.rs index 05df00e537f0..8aa155080a31 100644 --- a/parachain/test-parachains/adder/collator/build.rs +++ b/parachain/test-parachains/adder/collator/build.rs @@ -16,7 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the test worker when the + // For the node/worker version check, make sure we always rebuild the puppet worker when the // version changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index d03c64fbf5cc..66ebfd9488b0 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -65,10 +65,8 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, workers_path: None, - workers_names: Some(( - "polkadot-prepare-worker-adder-collator".into(), - "polkadot-execute-worker-adder-collator".into(), - )), + workers_names: None, + dont_use_external_workers: false, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/adder/collator/tests/integration.rs b/parachain/test-parachains/adder/collator/tests/integration.rs index 13e57bd82505..9ab1c0c337a6 100644 --- a/parachain/test-parachains/adder/collator/tests/integration.rs +++ b/parachain/test-parachains/adder/collator/tests/integration.rs @@ -19,9 +19,6 @@ const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_adder_collator_puppet_worker"); -const WORKERS_NAMES: (&str, &str) = - ("polkadot-prepare-worker-adder-collator", "polkadot-execute-worker-adder-collator"); - // If this test is failing, make sure to run all tests with the `real-overseer` feature being enabled. #[substrate_test_utils::test(flavor = "multi_thread")] @@ -44,8 +41,7 @@ async fn collating_using_adder_collator() { ); // start alice - let alice = - polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into()), None); + let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); let bob_config = polkadot_test_service::node_config( || {}, @@ -56,7 +52,7 @@ async fn collating_using_adder_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into()), None); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); let collator = test_parachain_adder_collator::Collator::new(); @@ -73,7 +69,6 @@ async fn collating_using_adder_collator() { || {}, vec![alice.addr.clone(), bob.addr.clone()], collator.collator_key(), - Some((WORKERS_NAMES.0.into(), WORKERS_NAMES.1.into())), ); charlie diff --git a/parachain/test-parachains/undying/collator/Cargo.toml b/parachain/test-parachains/undying/collator/Cargo.toml index 4cf944ae6270..6669f0f47ed2 100644 --- a/parachain/test-parachains/undying/collator/Cargo.toml +++ b/parachain/test-parachains/undying/collator/Cargo.toml @@ -13,15 +13,6 @@ path = "src/main.rs" name = "undying_collator_puppet_worker" path = "bin/puppet_worker.rs" -# Use artifact dependencies once stable. -# See https://github.com/rust-lang/cargo/issues/9096. -[[bin]] -name = "polkadot-execute-worker-undying-collator" -path = "../../../../node/core/pvf/execute-worker/src/main.rs" -[[bin]] -name = "polkadot-prepare-worker-undying-collator" -path = "../../../../node/core/pvf/prepare-worker/src/main.rs" - [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } clap = { version = "4.0.9", features = ["derive"] } @@ -45,12 +36,6 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master # a big problem since it is used transitively anyway. polkadot-node-core-pvf = { path = "../../../../node/core/pvf" } -# Required for binary workers to build. -polkadot-node-core-pvf-common = { path = "../../../../node/core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../../../../node/core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../../../../node/core/pvf/prepare-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [dev-dependencies] polkadot-parachain = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } diff --git a/parachain/test-parachains/undying/collator/build.rs b/parachain/test-parachains/undying/collator/build.rs index 05df00e537f0..8aa155080a31 100644 --- a/parachain/test-parachains/undying/collator/build.rs +++ b/parachain/test-parachains/undying/collator/build.rs @@ -16,7 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the test worker when the + // For the node/worker version check, make sure we always rebuild the puppet worker when the // version changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 431b868c5285..0ef19aae711e 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -65,10 +65,8 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, workers_path: None, - workers_names: Some(( - "polkadot-prepare-worker-undying-collator".into(), - "polkadot-execute-worker-undying-collator".into(), - )), + workers_names: None, + dont_use_external_workers: false, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/undying/collator/tests/integration.rs b/parachain/test-parachains/undying/collator/tests/integration.rs index 54c82c904b83..8ca6eec9aa62 100644 --- a/parachain/test-parachains/undying/collator/tests/integration.rs +++ b/parachain/test-parachains/undying/collator/tests/integration.rs @@ -19,9 +19,6 @@ const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_undying_collator_puppet_worker"); -const WORKERS_NAMES: (&str, &str) = - ("polkadot-prepare-worker-undying-collator", "polkadot-execute-worker-undying-collator"); - // If this test is failing, make sure to run all tests with the `real-overseer` feature being enabled. #[substrate_test_utils::test(flavor = "multi_thread")] async fn collating_using_undying_collator() { @@ -43,8 +40,7 @@ async fn collating_using_undying_collator() { ); // start alice - let alice = - polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into()), None); + let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); let bob_config = polkadot_test_service::node_config( || {}, @@ -55,7 +51,7 @@ async fn collating_using_undying_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into()), None); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); let collator = test_parachain_undying_collator::Collator::new(1_000, 1); @@ -72,7 +68,6 @@ async fn collating_using_undying_collator() { || {}, vec![alice.addr.clone(), bob.addr.clone()], collator.collator_key(), - Some((WORKERS_NAMES.0.into(), WORKERS_NAMES.1.into())), ); charlie From 11bbf59f8a67a7a0c96bc2f2a6c31438dbd3394e Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 20 Jul 2023 12:01:59 +0200 Subject: [PATCH 29/48] Pass node version from node into dependencies to avoid recompiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [X] get version in CLI - [X] pass it in to service - [X] pass version along to PVF - [X] remove rerun from service - [X] add rerun to CLI - [X] don’t rerun pvf/worker’s (these should be built by nodes which have rerun enabled) --- Cargo.lock | 5 ++- Cargo.toml | 15 +-------- cli/Cargo.toml | 12 ++++++++ cli/src/command.rs | 1 + node/core/candidate-validation/src/lib.rs | 26 ++++++++-------- node/core/pvf/execute-worker/build.rs | 3 -- node/core/pvf/prepare-worker/build.rs | 3 -- node/core/pvf/src/execute/queue.rs | 37 +++++++++++++++++++---- node/core/pvf/src/execute/worker_intf.rs | 3 +- node/core/pvf/src/host.rs | 6 ++++ node/core/pvf/src/prepare/pool.rs | 20 ++++++++++-- node/core/pvf/src/prepare/worker_intf.rs | 3 +- node/core/pvf/src/worker_intf.rs | 8 +++-- node/service/src/lib.rs | 4 +++ 14 files changed, 95 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61dd83c6ef80..65a88abe951a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6847,11 +6847,8 @@ dependencies = [ "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", - "sp-tracing", "substrate-rpc-client", "tempfile", "tikv-jemallocator", @@ -6977,6 +6974,7 @@ dependencies = [ "frame-benchmarking-cli", "futures", "log", + "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-metrics", @@ -6994,6 +6992,7 @@ dependencies = [ "sp-io", "sp-keyring", "sp-maybe-compressed-blob", + "sp-tracing", "substrate-build-script-utils", "thiserror", "try-runtime-cli", diff --git a/Cargo.toml b/Cargo.toml index f63288499b80..80d0acee587d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,14 +2,6 @@ name = "polkadot" path = "src/main.rs" -[[bin]] -name = "polkadot-execute-worker" -path = "node/core/pvf/execute-worker/src/main.rs" - -[[bin]] -name = "polkadot-prepare-worker" -path = "node/core/pvf/prepare-worker/src/main.rs" - [package] name = "polkadot" description = "Implementation of a `https://polkadot.network` node in Rust based on the Substrate framework." @@ -30,15 +22,10 @@ version = "0.9.43" color-eyre = { version = "0.6.1", default-features = false } tikv-jemallocator = "0.5.0" -# Needed for worker binaries. -polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - # Crates in our workspace, defined as dependencies so we can pass them feature flags. polkadot-cli = { path = "cli", features = [ "polkadot-native", "kusama-native", "westend-native", "rococo-native" ] } polkadot-node-core-pvf = { path = "node/core/pvf" } +polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } polkadot-overseer = { path = "node/overseer" } [dev-dependencies] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e7aa562880cc..b42969064190 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -13,6 +13,14 @@ wasm-opt = false [lib] crate-type = ["cdylib", "rlib"] +[[bin]] +name = "polkadot-execute-worker" +path = "../node/core/pvf/execute-worker/src/main.rs" + +[[bin]] +name = "polkadot-prepare-worker" +path = "../node/core/pvf/prepare-worker/src/main.rs" + [dependencies] clap = { version = "4.0.9", features = ["derive"], optional = true } log = "0.4.17" @@ -40,6 +48,10 @@ sc-sysinfo = { git = "https://github.com/paritytech/substrate", branch = "master sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" } sc-storage-monitor = { git = "https://github.com/paritytech/substrate", branch = "master" } +# Needed for worker binaries. +polkadot-node-core-pvf-common = { path = "../node/core/pvf/common" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [build-dependencies] substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/cli/src/command.rs b/cli/src/command.rs index da93cc190e0a..d199529fd68a 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -291,6 +291,7 @@ where enable_beefy, jaeger_agent, telemetry_worker_handle: None, + node_version: env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(), workers_path: cli.run.workers_path, workers_names: None, dont_use_external_workers: false, diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 749bf410596e..4b05c8f67837 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -93,6 +93,8 @@ const DEFAULT_APPROVAL_EXECUTION_TIMEOUT: Duration = Duration::from_secs(12); pub struct Config { /// The path where candidate validation can store compiled artifacts for PVFs. pub artifacts_cache_path: PathBuf, + /// The version of the node. + pub node_version: String, /// Path to the preparation worker binary pub prep_worker_path: PathBuf, /// Path to the execution worker binary @@ -125,16 +127,9 @@ impl CandidateValidationSubsystem { #[overseer::subsystem(CandidateValidation, error=SubsystemError, prefix=self::overseer)] impl CandidateValidationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { - let future = run( - ctx, - self.metrics, - self.pvf_metrics, - self.config.artifacts_cache_path, - self.config.prep_worker_path, - self.config.exec_worker_path, - ) - .map_err(|e| SubsystemError::with_origin("candidate-validation", e)) - .boxed(); + let future = run(ctx, self.metrics, self.pvf_metrics, self.config) + .map_err(|e| SubsystemError::with_origin("candidate-validation", e)) + .boxed(); SpawnedSubsystem { name: "candidate-validation-subsystem", future } } } @@ -144,12 +139,15 @@ async fn run( mut ctx: Context, metrics: Metrics, pvf_metrics: polkadot_node_core_pvf::Metrics, - cache_path: PathBuf, - prep_worker_path: PathBuf, - exec_worker_path: PathBuf, + Config { artifacts_cache_path, node_version, prep_worker_path, exec_worker_path }: Config, ) -> SubsystemResult<()> { let (validation_host, task) = polkadot_node_core_pvf::start( - polkadot_node_core_pvf::Config::new(cache_path, prep_worker_path, exec_worker_path), + polkadot_node_core_pvf::Config::new( + artifacts_cache_path, + node_version, + prep_worker_path, + exec_worker_path, + ), pvf_metrics, ); ctx.spawn_blocking("pvf-validation-host", task.boxed())?; diff --git a/node/core/pvf/execute-worker/build.rs b/node/core/pvf/execute-worker/build.rs index 7c249f50ccc4..40e9f832586e 100644 --- a/node/core/pvf/execute-worker/build.rs +++ b/node/core/pvf/execute-worker/build.rs @@ -16,7 +16,4 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the worker when the version - // changes. - substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/prepare-worker/build.rs b/node/core/pvf/prepare-worker/build.rs index 7c249f50ccc4..40e9f832586e 100644 --- a/node/core/pvf/prepare-worker/build.rs +++ b/node/core/pvf/prepare-worker/build.rs @@ -16,7 +16,4 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the worker when the version - // changes. - substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/src/execute/queue.rs b/node/core/pvf/src/execute/queue.rs index 395697616b36..e2b61b607f26 100644 --- a/node/core/pvf/src/execute/queue.rs +++ b/node/core/pvf/src/execute/queue.rs @@ -137,8 +137,10 @@ struct Queue { /// The receiver that receives messages to the pool. to_queue_rx: mpsc::Receiver, + // Some variables related to the current session. program_path: PathBuf, spawn_timeout: Duration, + node_version: String, /// The queue of jobs that are waiting for a worker to pick up. queue: VecDeque, @@ -152,12 +154,14 @@ impl Queue { program_path: PathBuf, worker_capacity: usize, spawn_timeout: Duration, + node_version: String, to_queue_rx: mpsc::Receiver, ) -> Self { Self { metrics, program_path, spawn_timeout, + node_version, to_queue_rx, queue: VecDeque::new(), mux: Mux::new(), @@ -398,9 +402,15 @@ fn spawn_extra_worker(queue: &mut Queue, job: ExecuteJob) { queue.metrics.execute_worker().on_begin_spawn(); gum::debug!(target: LOG_TARGET, "spawning an extra worker"); - queue - .mux - .push(spawn_worker_task(queue.program_path.clone(), job, queue.spawn_timeout).boxed()); + queue.mux.push( + spawn_worker_task( + queue.program_path.clone(), + job, + queue.spawn_timeout, + queue.node_version.clone(), + ) + .boxed(), + ); queue.workers.spawn_inflight += 1; } @@ -414,12 +424,18 @@ async fn spawn_worker_task( program_path: PathBuf, job: ExecuteJob, spawn_timeout: Duration, + node_version: String, ) -> QueueEvent { use futures_timer::Delay; loop { - match super::worker_intf::spawn(&program_path, job.executor_params.clone(), spawn_timeout) - .await + match super::worker_intf::spawn( + &program_path, + job.executor_params.clone(), + spawn_timeout, + &node_version, + ) + .await { Ok((idle, handle)) => break QueueEvent::Spawn(idle, handle, job), Err(err) => { @@ -481,8 +497,17 @@ pub fn start( program_path: PathBuf, worker_capacity: usize, spawn_timeout: Duration, + node_version: String, ) -> (mpsc::Sender, impl Future) { let (to_queue_tx, to_queue_rx) = mpsc::channel(20); - let run = Queue::new(metrics, program_path, worker_capacity, spawn_timeout, to_queue_rx).run(); + let run = Queue::new( + metrics, + program_path, + worker_capacity, + spawn_timeout, + node_version, + to_queue_rx, + ) + .run(); (to_queue_tx, run) } diff --git a/node/core/pvf/src/execute/worker_intf.rs b/node/core/pvf/src/execute/worker_intf.rs index 6e54e17e515a..076849185326 100644 --- a/node/core/pvf/src/execute/worker_intf.rs +++ b/node/core/pvf/src/execute/worker_intf.rs @@ -45,11 +45,12 @@ pub async fn spawn( program_path: &Path, executor_params: ExecutorParams, spawn_timeout: Duration, + node_version: &str, ) -> Result<(IdleWorker, WorkerHandle), SpawnErr> { let (mut idle_worker, worker_handle) = spawn_with_program_path( "execute", program_path, - &["execute-worker", "--node-impl-version", env!("SUBSTRATE_CLI_IMPL_VERSION")], + &["execute-worker", "--node-impl-version", node_version], spawn_timeout, ) .await?; diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index a22efc08fa5a..4e367cc8e166 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -150,6 +150,8 @@ struct ExecutePvfInputs { pub struct Config { /// The root directory where the prepared artifacts can be stored. pub cache_path: PathBuf, + /// The version of the node. + pub node_version: String, /// The path to the program that can be used to spawn the prepare workers. pub prepare_worker_program_path: PathBuf, /// The time allotted for a prepare worker to spawn and report to the host. @@ -171,11 +173,13 @@ impl Config { /// Create a new instance of the configuration. pub fn new( cache_path: PathBuf, + node_version: String, prepare_worker_program_path: PathBuf, execute_worker_program_path: PathBuf, ) -> Self { Self { cache_path, + node_version, prepare_worker_program_path, prepare_worker_spawn_timeout: Duration::from_secs(3), prepare_workers_soft_max_num: 1, @@ -210,6 +214,7 @@ pub fn start(config: Config, metrics: Metrics) -> (ValidationHost, impl Future (ValidationHost, impl Future, from_pool: mpsc::UnboundedSender, spawned: HopSlotMap, mux: Mux, + metrics: Metrics, } @@ -128,6 +131,7 @@ async fn run( program_path, cache_path, spawn_timeout, + node_version, to_pool, mut from_pool, mut spawned, @@ -155,6 +159,7 @@ async fn run( &program_path, &cache_path, spawn_timeout, + node_version.clone(), &mut spawned, &mut mux, to_pool, @@ -201,6 +206,7 @@ fn handle_to_pool( program_path: &Path, cache_path: &Path, spawn_timeout: Duration, + node_version: String, spawned: &mut HopSlotMap, mux: &mut Mux, to_pool: ToPool, @@ -209,7 +215,9 @@ fn handle_to_pool( ToPool::Spawn => { gum::debug!(target: LOG_TARGET, "spawning a new prepare worker"); metrics.prepare_worker().on_begin_spawn(); - mux.push(spawn_worker_task(program_path.to_owned(), spawn_timeout).boxed()); + mux.push( + spawn_worker_task(program_path.to_owned(), spawn_timeout, node_version).boxed(), + ); }, ToPool::StartWork { worker, pvf, artifact_path } => { if let Some(data) = spawned.get_mut(worker) { @@ -248,11 +256,15 @@ fn handle_to_pool( } } -async fn spawn_worker_task(program_path: PathBuf, spawn_timeout: Duration) -> PoolEvent { +async fn spawn_worker_task( + program_path: PathBuf, + spawn_timeout: Duration, + node_version: String, +) -> PoolEvent { use futures_timer::Delay; loop { - match worker_intf::spawn(&program_path, spawn_timeout).await { + match worker_intf::spawn(&program_path, spawn_timeout, &node_version).await { Ok((idle, handle)) => break PoolEvent::Spawn(idle, handle), Err(err) => { gum::warn!(target: LOG_TARGET, "failed to spawn a prepare worker: {:?}", err); @@ -419,6 +431,7 @@ pub fn start( program_path: PathBuf, cache_path: PathBuf, spawn_timeout: Duration, + node_version: String, ) -> (mpsc::Sender, mpsc::UnboundedReceiver, impl Future) { let (to_pool_tx, to_pool_rx) = mpsc::channel(10); let (from_pool_tx, from_pool_rx) = mpsc::unbounded(); @@ -428,6 +441,7 @@ pub fn start( program_path, cache_path, spawn_timeout, + node_version, to_pool: to_pool_rx, from_pool: from_pool_tx, spawned: HopSlotMap::with_capacity_and_key(20), diff --git a/node/core/pvf/src/prepare/worker_intf.rs b/node/core/pvf/src/prepare/worker_intf.rs index 47522d3f0856..f423e9823c02 100644 --- a/node/core/pvf/src/prepare/worker_intf.rs +++ b/node/core/pvf/src/prepare/worker_intf.rs @@ -45,11 +45,12 @@ use tokio::{io, net::UnixStream}; pub async fn spawn( program_path: &Path, spawn_timeout: Duration, + node_version: &str, ) -> Result<(IdleWorker, WorkerHandle), SpawnErr> { spawn_with_program_path( "prepare", program_path, - &["prepare-worker", "--node-impl-version", env!("SUBSTRATE_CLI_IMPL_VERSION")], + &["prepare-worker", "--node-impl-version", node_version], spawn_timeout, ) .await diff --git a/node/core/pvf/src/worker_intf.rs b/node/core/pvf/src/worker_intf.rs index 33144616601d..ef5733ec0e6d 100644 --- a/node/core/pvf/src/worker_intf.rs +++ b/node/core/pvf/src/worker_intf.rs @@ -43,12 +43,14 @@ pub const JOB_TIMEOUT_WALL_CLOCK_FACTOR: u32 = 4; pub async fn spawn_with_program_path( debug_id: &'static str, program_path: impl Into, - extra_args: &'static [&'static str], + extra_args: &[&str], spawn_timeout: Duration, ) -> Result<(IdleWorker, WorkerHandle), SpawnErr> { let program_path = program_path.into(); with_transient_socket_path(debug_id, |socket_path| { let socket_path = socket_path.to_owned(); + let extra_args: Vec = extra_args.iter().map(|arg| arg.to_string()).collect(); + async move { let listener = UnixListener::bind(&socket_path).map_err(|err| { gum::warn!( @@ -63,7 +65,7 @@ pub async fn spawn_with_program_path( })?; let handle = - WorkerHandle::spawn(&program_path, extra_args, socket_path).map_err(|err| { + WorkerHandle::spawn(&program_path, &extra_args, socket_path).map_err(|err| { gum::warn!( target: LOG_TARGET, %debug_id, @@ -214,7 +216,7 @@ pub struct WorkerHandle { impl WorkerHandle { fn spawn( program: impl AsRef, - extra_args: &[&str], + extra_args: &[String], socket_path: impl AsRef, ) -> io::Result { let mut child = process::Command::new(program.as_ref()) diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index ef1addf6a20e..afcf46e08c36 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -625,6 +625,8 @@ pub struct NewFullParams { pub enable_beefy: bool, pub jaeger_agent: Option, pub telemetry_worker_handle: Option, + /// The version of the node. + pub node_version: String, /// An optional path to a directory containing the workers. pub workers_path: Option, /// Optional custom names for the prepare and execute workers. @@ -706,6 +708,7 @@ pub fn new_full( enable_beefy, jaeger_agent, telemetry_worker_handle, + node_version, workers_path, workers_names, dont_use_external_workers, @@ -914,6 +917,7 @@ pub fn new_full( .path() .ok_or(Error::DatabasePathRequired)? .join("pvf-artifacts"), + node_version, prep_worker_path, exec_worker_path, }; From 6244e3f0572d70cbfc477adaf6b51b5ca8ee2e05 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 20 Jul 2023 17:10:26 +0200 Subject: [PATCH 30/48] Some more improvements for smoother tests - [X] Fix tests - [X] Make puppet workers pass None for version and remove rerun - [X] Make test collators self-contained --- Cargo.lock | 6 +- Cargo.toml | 13 +++ cli/Cargo.toml | 12 --- cli/src/command.rs | 6 +- node/core/candidate-validation/src/lib.rs | 4 +- node/core/pvf/bin/puppet_worker.rs | 2 +- node/core/pvf/build.rs | 3 - node/core/pvf/common/src/worker/mod.rs | 6 +- node/core/pvf/execute-worker/src/lib.rs | 6 +- node/core/pvf/prepare-worker/src/lib.rs | 6 +- node/core/pvf/src/execute/queue.rs | 10 +-- node/core/pvf/src/execute/worker_intf.rs | 15 ++-- node/core/pvf/src/host.rs | 6 +- node/core/pvf/src/prepare/pool.rs | 16 ++-- node/core/pvf/src/prepare/worker_intf.rs | 14 ++-- node/core/pvf/src/testing.rs | 8 +- node/core/pvf/tests/it/main.rs | 2 +- node/service/Cargo.toml | 3 - node/service/build.rs | 19 ----- node/service/src/lib.rs | 6 +- node/service/src/workers.rs | 82 ++++++++++--------- node/test/service/src/lib.rs | 1 + .../adder/collator/bin/puppet_worker.rs | 2 +- .../test-parachains/adder/collator/build.rs | 22 ----- .../adder/collator/src/main.rs | 6 +- .../undying/collator/bin/puppet_worker.rs | 2 +- .../test-parachains/undying/collator/build.rs | 22 ----- .../undying/collator/src/main.rs | 6 +- 28 files changed, 128 insertions(+), 178 deletions(-) delete mode 100644 node/service/build.rs delete mode 100644 parachain/test-parachains/adder/collator/build.rs delete mode 100644 parachain/test-parachains/undying/collator/build.rs diff --git a/Cargo.lock b/Cargo.lock index 65a88abe951a..bb10930fb0b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6847,8 +6847,11 @@ dependencies = [ "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", + "sp-tracing", "substrate-rpc-client", "tempfile", "tikv-jemallocator", @@ -6974,7 +6977,6 @@ dependencies = [ "frame-benchmarking-cli", "futures", "log", - "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-metrics", @@ -6992,7 +6994,6 @@ dependencies = [ "sp-io", "sp-keyring", "sp-maybe-compressed-blob", - "sp-tracing", "substrate-build-script-utils", "thiserror", "try-runtime-cli", @@ -8206,7 +8207,6 @@ dependencies = [ "sp-trie", "sp-version", "sp-weights", - "substrate-build-script-utils", "substrate-prometheus-endpoint", "tempfile", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 80d0acee587d..e8f45748d6a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,14 @@ authors.workspace = true edition.workspace = true version.workspace = true +[[bin]] +name = "polkadot-execute-worker" +path = "node/core/pvf/execute-worker/src/main.rs" + +[[bin]] +name = "polkadot-prepare-worker" +path = "node/core/pvf/prepare-worker/src/main.rs" + [workspace.package] authors = ["Parity Technologies "] edition = "2021" @@ -28,6 +36,11 @@ polkadot-node-core-pvf = { path = "node/core/pvf" } polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } polkadot-overseer = { path = "node/overseer" } +# Needed for worker binaries. +polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [dev-dependencies] assert_cmd = "2.0.4" nix = { version = "0.26.1", features = ["signal"] } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b42969064190..e7aa562880cc 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -13,14 +13,6 @@ wasm-opt = false [lib] crate-type = ["cdylib", "rlib"] -[[bin]] -name = "polkadot-execute-worker" -path = "../node/core/pvf/execute-worker/src/main.rs" - -[[bin]] -name = "polkadot-prepare-worker" -path = "../node/core/pvf/prepare-worker/src/main.rs" - [dependencies] clap = { version = "4.0.9", features = ["derive"], optional = true } log = "0.4.17" @@ -48,10 +40,6 @@ sc-sysinfo = { git = "https://github.com/paritytech/substrate", branch = "master sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" } sc-storage-monitor = { git = "https://github.com/paritytech/substrate", branch = "master" } -# Needed for worker binaries. -polkadot-node-core-pvf-common = { path = "../node/core/pvf/common" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [build-dependencies] substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/cli/src/command.rs b/cli/src/command.rs index d199529fd68a..64a0a21d5671 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -291,7 +291,7 @@ where enable_beefy, jaeger_agent, telemetry_worker_handle: None, - node_version: env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(), + node_version: Some(env!("SUBSTRATE_CLI_IMPL_VERSION").to_string()), workers_path: cli.run.workers_path, workers_names: None, dont_use_external_workers: false, @@ -447,7 +447,7 @@ pub fn run() -> Result<()> { polkadot_node_core_pvf_prepare_worker::worker_entrypoint( &cmd.socket_path, Some(&cmd.node_impl_version), - &cmd.node_impl_version, + Some(&cmd.node_impl_version), ); Ok(()) } @@ -470,7 +470,7 @@ pub fn run() -> Result<()> { polkadot_node_core_pvf_execute_worker::worker_entrypoint( &cmd.socket_path, Some(&cmd.node_impl_version), - &cmd.node_impl_version, + Some(&cmd.node_impl_version), ); Ok(()) } diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 4b05c8f67837..6bb50463c93c 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -93,8 +93,8 @@ const DEFAULT_APPROVAL_EXECUTION_TIMEOUT: Duration = Duration::from_secs(12); pub struct Config { /// The path where candidate validation can store compiled artifacts for PVFs. pub artifacts_cache_path: PathBuf, - /// The version of the node. - pub node_version: String, + /// The version of the node. `None` can be passed to skip the version check (only for tests). + pub node_version: Option, /// Path to the preparation worker binary pub prep_worker_path: PathBuf, /// Path to the execution worker binary diff --git a/node/core/pvf/bin/puppet_worker.rs b/node/core/pvf/bin/puppet_worker.rs index 9ad99d6c9c66..7f93519d8454 100644 --- a/node/core/pvf/bin/puppet_worker.rs +++ b/node/core/pvf/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(env!("SUBSTRATE_CLI_IMPL_VERSION")); +polkadot_node_core_pvf::decl_puppet_worker_main!(); diff --git a/node/core/pvf/build.rs b/node/core/pvf/build.rs index 8aa155080a31..40e9f832586e 100644 --- a/node/core/pvf/build.rs +++ b/node/core/pvf/build.rs @@ -16,7 +16,4 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the puppet worker when the - // version changes. - substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index 8082231142db..a940d951a220 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -82,7 +82,7 @@ macro_rules! decl_worker_main { } } - $entrypoint(&socket_path, node_version, $worker_version); + $entrypoint(&socket_path, node_version, Some($worker_version)); } }; } @@ -103,7 +103,7 @@ pub fn worker_event_loop( debug_id: &'static str, socket_path: &str, node_version: Option<&str>, - worker_version: &str, + worker_version: Option<&str>, mut event_loop: F, ) where F: FnMut(UnixStream) -> Fut, @@ -113,7 +113,7 @@ pub fn worker_event_loop( gum::debug!(target: LOG_TARGET, %worker_pid, "starting pvf worker ({})", debug_id); // Check for a mismatch between the node and worker versions. - if let Some(node_version) = node_version { + if let (Some(node_version), Some(worker_version)) = (node_version, worker_version) { if node_version != worker_version { gum::error!( target: LOG_TARGET, diff --git a/node/core/pvf/execute-worker/src/lib.rs b/node/core/pvf/execute-worker/src/lib.rs index 7cbe225bb11e..e36b34f66c80 100644 --- a/node/core/pvf/execute-worker/src/lib.rs +++ b/node/core/pvf/execute-worker/src/lib.rs @@ -123,7 +123,11 @@ async fn send_response(stream: &mut UnixStream, response: Response) -> io::Resul /// `node_version`, if `Some`, is checked against the worker version. A mismatch results in /// immediate worker termination. `None` is used for tests and in other situations when version /// check is not necessary. -pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>, worker_version: &str) { +pub fn worker_entrypoint( + socket_path: &str, + node_version: Option<&str>, + worker_version: Option<&str>, +) { worker_event_loop( "execute", socket_path, diff --git a/node/core/pvf/prepare-worker/src/lib.rs b/node/core/pvf/prepare-worker/src/lib.rs index 7d393bd97742..bd1413b77597 100644 --- a/node/core/pvf/prepare-worker/src/lib.rs +++ b/node/core/pvf/prepare-worker/src/lib.rs @@ -118,7 +118,11 @@ async fn send_response(stream: &mut UnixStream, result: PrepareResult) -> io::Re /// /// 7. Send the result of preparation back to the host. If any error occurred in the above steps, we /// send that in the `PrepareResult`. -pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>, worker_version: &str) { +pub fn worker_entrypoint( + socket_path: &str, + node_version: Option<&str>, + worker_version: Option<&str>, +) { worker_event_loop( "prepare", socket_path, diff --git a/node/core/pvf/src/execute/queue.rs b/node/core/pvf/src/execute/queue.rs index e2b61b607f26..57c19cd04533 100644 --- a/node/core/pvf/src/execute/queue.rs +++ b/node/core/pvf/src/execute/queue.rs @@ -140,7 +140,7 @@ struct Queue { // Some variables related to the current session. program_path: PathBuf, spawn_timeout: Duration, - node_version: String, + node_version: Option, /// The queue of jobs that are waiting for a worker to pick up. queue: VecDeque, @@ -154,7 +154,7 @@ impl Queue { program_path: PathBuf, worker_capacity: usize, spawn_timeout: Duration, - node_version: String, + node_version: Option, to_queue_rx: mpsc::Receiver, ) -> Self { Self { @@ -424,7 +424,7 @@ async fn spawn_worker_task( program_path: PathBuf, job: ExecuteJob, spawn_timeout: Duration, - node_version: String, + node_version: Option, ) -> QueueEvent { use futures_timer::Delay; @@ -433,7 +433,7 @@ async fn spawn_worker_task( &program_path, job.executor_params.clone(), spawn_timeout, - &node_version, + node_version.as_ref().map(|v| v.as_str()), ) .await { @@ -497,7 +497,7 @@ pub fn start( program_path: PathBuf, worker_capacity: usize, spawn_timeout: Duration, - node_version: String, + node_version: Option, ) -> (mpsc::Sender, impl Future) { let (to_queue_tx, to_queue_rx) = mpsc::channel(20); let run = Queue::new( diff --git a/node/core/pvf/src/execute/worker_intf.rs b/node/core/pvf/src/execute/worker_intf.rs index 076849185326..9d8b61d10447 100644 --- a/node/core/pvf/src/execute/worker_intf.rs +++ b/node/core/pvf/src/execute/worker_intf.rs @@ -45,15 +45,14 @@ pub async fn spawn( program_path: &Path, executor_params: ExecutorParams, spawn_timeout: Duration, - node_version: &str, + node_version: Option<&str>, ) -> Result<(IdleWorker, WorkerHandle), SpawnErr> { - let (mut idle_worker, worker_handle) = spawn_with_program_path( - "execute", - program_path, - &["execute-worker", "--node-impl-version", node_version], - spawn_timeout, - ) - .await?; + let mut extra_args = vec!["execute-worker"]; + if let Some(node_version) = node_version { + extra_args.extend_from_slice(&["--node-impl-version", node_version]); + } + let (mut idle_worker, worker_handle) = + spawn_with_program_path("execute", program_path, &extra_args, spawn_timeout).await?; send_handshake(&mut idle_worker.stream, Handshake { executor_params }) .await .map_err(|error| { diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index 4e367cc8e166..a5772e34e16e 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -150,8 +150,8 @@ struct ExecutePvfInputs { pub struct Config { /// The root directory where the prepared artifacts can be stored. pub cache_path: PathBuf, - /// The version of the node. - pub node_version: String, + /// The version of the node. `None` can be passed to skip the version check (only for tests). + pub node_version: Option, /// The path to the program that can be used to spawn the prepare workers. pub prepare_worker_program_path: PathBuf, /// The time allotted for a prepare worker to spawn and report to the host. @@ -173,7 +173,7 @@ impl Config { /// Create a new instance of the configuration. pub fn new( cache_path: PathBuf, - node_version: String, + node_version: Option, prepare_worker_program_path: PathBuf, execute_worker_program_path: PathBuf, ) -> Self { diff --git a/node/core/pvf/src/prepare/pool.rs b/node/core/pvf/src/prepare/pool.rs index d3af0132cde3..3903409a870b 100644 --- a/node/core/pvf/src/prepare/pool.rs +++ b/node/core/pvf/src/prepare/pool.rs @@ -113,7 +113,7 @@ struct Pool { program_path: PathBuf, cache_path: PathBuf, spawn_timeout: Duration, - node_version: String, + node_version: Option, to_pool: mpsc::Receiver, from_pool: mpsc::UnboundedSender, @@ -206,7 +206,7 @@ fn handle_to_pool( program_path: &Path, cache_path: &Path, spawn_timeout: Duration, - node_version: String, + node_version: Option, spawned: &mut HopSlotMap, mux: &mut Mux, to_pool: ToPool, @@ -259,12 +259,18 @@ fn handle_to_pool( async fn spawn_worker_task( program_path: PathBuf, spawn_timeout: Duration, - node_version: String, + node_version: Option, ) -> PoolEvent { use futures_timer::Delay; loop { - match worker_intf::spawn(&program_path, spawn_timeout, &node_version).await { + match worker_intf::spawn( + &program_path, + spawn_timeout, + node_version.as_ref().map(|v| v.as_str()), + ) + .await + { Ok((idle, handle)) => break PoolEvent::Spawn(idle, handle), Err(err) => { gum::warn!(target: LOG_TARGET, "failed to spawn a prepare worker: {:?}", err); @@ -431,7 +437,7 @@ pub fn start( program_path: PathBuf, cache_path: PathBuf, spawn_timeout: Duration, - node_version: String, + node_version: Option, ) -> (mpsc::Sender, mpsc::UnboundedReceiver, impl Future) { let (to_pool_tx, to_pool_rx) = mpsc::channel(10); let (from_pool_tx, from_pool_rx) = mpsc::unbounded(); diff --git a/node/core/pvf/src/prepare/worker_intf.rs b/node/core/pvf/src/prepare/worker_intf.rs index f423e9823c02..d0d9a026dda7 100644 --- a/node/core/pvf/src/prepare/worker_intf.rs +++ b/node/core/pvf/src/prepare/worker_intf.rs @@ -45,15 +45,13 @@ use tokio::{io, net::UnixStream}; pub async fn spawn( program_path: &Path, spawn_timeout: Duration, - node_version: &str, + node_version: Option<&str>, ) -> Result<(IdleWorker, WorkerHandle), SpawnErr> { - spawn_with_program_path( - "prepare", - program_path, - &["prepare-worker", "--node-impl-version", node_version], - spawn_timeout, - ) - .await + let mut extra_args = vec!["prepare-worker"]; + if let Some(node_version) = node_version { + extra_args.extend_from_slice(&["--node-impl-version", node_version]); + } + spawn_with_program_path("prepare", program_path, &extra_args, spawn_timeout).await } pub enum Outcome { diff --git a/node/core/pvf/src/testing.rs b/node/core/pvf/src/testing.rs index 00b51537f570..3cd1ce304ab8 100644 --- a/node/core/pvf/src/testing.rs +++ b/node/core/pvf/src/testing.rs @@ -53,7 +53,7 @@ pub fn validate_candidate( /// the appropriate worker, making the executable that can be used for spawning workers. #[macro_export] macro_rules! decl_puppet_worker_main { - ($worker_version:expr) => { + () => { fn main() { $crate::sp_tracing::try_init_simple(); @@ -63,10 +63,6 @@ macro_rules! decl_puppet_worker_main { } let entrypoint = match args[1].as_ref() { - "--version" | "-v" => { - println!("{}", $worker_version); - return - }, "exit" => { std::process::exit(1); }, @@ -90,7 +86,7 @@ macro_rules! decl_puppet_worker_main { } } - entrypoint(&socket_path, node_version, $worker_version); + entrypoint(&socket_path, node_version, None); } }; } diff --git a/node/core/pvf/tests/it/main.rs b/node/core/pvf/tests/it/main.rs index b66fd4ba41e6..72c459c2f632 100644 --- a/node/core/pvf/tests/it/main.rs +++ b/node/core/pvf/tests/it/main.rs @@ -54,7 +54,7 @@ impl TestHost { let cache_dir = tempfile::tempdir().unwrap(); let program_path = std::path::PathBuf::from(PUPPET_EXE); let mut config = - Config::new(cache_dir.path().to_owned(), program_path.clone(), program_path); + Config::new(cache_dir.path().to_owned(), None, program_path.clone(), program_path); f(&mut config); let (host, task) = start(config, Metrics::default()); let _ = tokio::task::spawn(task); diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index 99e605dee39f..1086c8bcab67 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -148,9 +148,6 @@ assert_matches = "1.5.0" serial_test = "2.0.0" tempfile = "3.2" -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } - [features] default = ["db", "full-node"] diff --git a/node/service/build.rs b/node/service/build.rs deleted file mode 100644 index 40e9f832586e..000000000000 --- a/node/service/build.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); -} diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index afcf46e08c36..563bd00195ec 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -625,8 +625,8 @@ pub struct NewFullParams { pub enable_beefy: bool, pub jaeger_agent: Option, pub telemetry_worker_handle: Option, - /// The version of the node. - pub node_version: String, + /// The version of the node. `None` can be passed to skip the version check (only for tests). + pub node_version: Option, /// An optional path to a directory containing the workers. pub workers_path: Option, /// Optional custom names for the prepare and execute workers. @@ -906,7 +906,7 @@ pub fn new_full( let program_path = std::env::current_exe()?; (program_path.clone(), program_path) } else { - workers::determine_workers_paths(workers_path, workers_names)? + workers::determine_workers_paths(workers_path, workers_names, node_version.clone())? }; log::info!("🚀 Using prepare-worker binary at: {:?}", prep_worker_path); log::info!("🚀 Using execute-worker binary at: {:?}", exec_worker_path); diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index d94721afce98..ab3d2968208f 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -54,6 +54,7 @@ fn workers_lib_path_override() -> &'static Mutex> { pub fn determine_workers_paths( given_workers_path: Option, workers_names: Option<(String, String)>, + node_version: Option, ) -> Result<(PathBuf, PathBuf), Error> { let mut workers_paths = list_workers_paths(given_workers_path.clone(), workers_names.clone())?; if workers_paths.is_empty() { @@ -68,30 +69,31 @@ pub fn determine_workers_paths( } // Do the version check. - let node_version = env!("SUBSTRATE_CLI_IMPL_VERSION").to_string(); - let worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; - let worker_version = std::str::from_utf8(&worker_version) - .expect("version is printed as a string; qed") - .trim() - .to_string(); - if worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch { - worker_version, - node_version, - worker_path: prep_worker_path, - }) - } - let worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; - let worker_version = std::str::from_utf8(&worker_version) - .expect("version is printed as a string; qed") - .trim() - .to_string(); - if worker_version != node_version { - return Err(Error::WorkerBinaryVersionMismatch { - worker_version, - node_version, - worker_path: exec_worker_path, - }) + if let Some(node_version) = node_version { + let worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout; + let worker_version = std::str::from_utf8(&worker_version) + .expect("version is printed as a string; qed") + .trim() + .to_string(); + if worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch { + worker_version, + node_version, + worker_path: prep_worker_path, + }) + } + let worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout; + let worker_version = std::str::from_utf8(&worker_version) + .expect("version is printed as a string; qed") + .trim() + .to_string(); + if worker_version != node_version { + return Err(Error::WorkerBinaryVersionMismatch { + worker_version, + node_version, + worker_path: exec_worker_path, + }) + } } Ok((prep_worker_path, exec_worker_path)) @@ -201,7 +203,7 @@ mod tests { use serial_test::serial; use std::{env::temp_dir, fs, os::unix::fs::PermissionsExt, path::Path}; - const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); + const NODE_VERSION: &'static str = "v0.1.2"; /// Write a dummy executable to the path which satisfies the version check. fn write_worker_exe(path: impl AsRef) -> Result<(), Box> { @@ -271,7 +273,7 @@ echo {} // Try with provided workers path that has missing binaries. assert_matches!( - determine_workers_paths(Some(given_workers_path.clone()), None), + determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())), Err(Error::MissingWorkerBinaries { given_workers_path: Some(p), workers_names: None }) if p == given_workers_path ); @@ -283,7 +285,7 @@ echo {} write_worker_exe(&execute_worker_path)?; fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o644))?; assert_matches!( - determine_workers_paths(Some(given_workers_path.clone()), None), + determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())), Err(Error::InvalidWorkerBinaries { prep_worker_path: p1, exec_worker_path: p2 }) if p1 == prepare_worker_path && p2 == execute_worker_path ); @@ -291,7 +293,7 @@ echo {} fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o744))?; fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o744))?; assert_matches!( - determine_workers_paths(Some(given_workers_path), None), + determine_workers_paths(Some(given_workers_path), None, Some(NODE_VERSION.into())), Ok((p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path ); @@ -299,7 +301,7 @@ echo {} let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); write_worker_exe(&given_workers_path)?; assert_matches!( - determine_workers_paths(Some(given_workers_path.clone()), None), + determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())), Ok((p1, p2)) if p1 == given_workers_path && p2 == given_workers_path ); @@ -314,7 +316,7 @@ echo {} with_temp_dir_structure(|tempdir| { // Try with both binaries missing. assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); @@ -322,7 +324,7 @@ echo {} let prepare_worker_path = tempdir.join("usr/bin/polkadot-prepare-worker"); write_worker_exe(&prepare_worker_path)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); @@ -331,7 +333,7 @@ echo {} let execute_worker_path = tempdir.join("usr/bin/polkadot-execute-worker"); write_worker_exe(&execute_worker_path)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); @@ -340,7 +342,7 @@ echo {} let prepare_worker_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); write_worker_exe(&prepare_worker_path)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); @@ -349,7 +351,7 @@ echo {} let execute_worker_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker"); write_worker_exe(execute_worker_path)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) ); @@ -424,7 +426,7 @@ echo {} write_worker_exe_invalid_version(&prepare_worker_bin_path, bad_version)?; write_worker_exe(&execute_worker_bin_path)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == prepare_worker_bin_path ); @@ -436,7 +438,7 @@ echo {} write_worker_exe(&prepare_worker_lib_path)?; write_worker_exe_invalid_version(&execute_worker_lib_path, bad_version)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == execute_worker_lib_path ); @@ -447,7 +449,7 @@ echo {} write_worker_exe_invalid_version(&prepare_worker_path, bad_version)?; write_worker_exe_invalid_version(&execute_worker_path, bad_version)?; assert_matches!( - determine_workers_paths(Some(given_workers_path), None), + determine_workers_paths(Some(given_workers_path), None, Some(NODE_VERSION.into())), Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == prepare_worker_path ); @@ -455,7 +457,7 @@ echo {} let given_workers_path = tempdir.join("usr/local/bin/puppet-worker"); write_worker_exe_invalid_version(&given_workers_path, bad_version)?; assert_matches!( - determine_workers_paths(Some(given_workers_path.clone()), None), + determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())), Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == given_workers_path ); @@ -476,7 +478,7 @@ echo {} write_worker_exe(&execute_worker_bin_path)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Ok((p1, p2)) if p1 == prepare_worker_bin_path && p2 == execute_worker_bin_path ); @@ -493,7 +495,7 @@ echo {} write_worker_exe(&execute_worker_lib_path)?; assert_matches!( - determine_workers_paths(None, None), + determine_workers_paths(None, None, Some(NODE_VERSION.into())), Ok((p1, p2)) if p1 == prepare_worker_lib_path && p2 == execute_worker_lib_path ); diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 93ff2516cf3f..4119cb6e36f6 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -84,6 +84,7 @@ pub fn new_full( enable_beefy: true, jaeger_agent: None, telemetry_worker_handle: None, + node_version: None, workers_path, workers_names: None, dont_use_external_workers, diff --git a/parachain/test-parachains/adder/collator/bin/puppet_worker.rs b/parachain/test-parachains/adder/collator/bin/puppet_worker.rs index 9ad99d6c9c66..7f93519d8454 100644 --- a/parachain/test-parachains/adder/collator/bin/puppet_worker.rs +++ b/parachain/test-parachains/adder/collator/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(env!("SUBSTRATE_CLI_IMPL_VERSION")); +polkadot_node_core_pvf::decl_puppet_worker_main!(); diff --git a/parachain/test-parachains/adder/collator/build.rs b/parachain/test-parachains/adder/collator/build.rs deleted file mode 100644 index 8aa155080a31..000000000000 --- a/parachain/test-parachains/adder/collator/build.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the puppet worker when the - // version changes. - substrate_build_script_utils::rerun_if_git_head_changed(); -} diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index 66ebfd9488b0..d6a59411c7f8 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -64,9 +64,13 @@ fn main() -> Result<()> { enable_beefy: false, jaeger_agent: None, telemetry_worker_handle: None, + // Skip the version check since we don't use external binaries. + node_version: None, workers_path: None, workers_names: None, - dont_use_external_workers: false, + // Don't use workers to make this test binary self-contained and easier to + // use. External binaries are for increased security in production. + dont_use_external_workers: true, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/undying/collator/bin/puppet_worker.rs b/parachain/test-parachains/undying/collator/bin/puppet_worker.rs index 9ad99d6c9c66..7f93519d8454 100644 --- a/parachain/test-parachains/undying/collator/bin/puppet_worker.rs +++ b/parachain/test-parachains/undying/collator/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(env!("SUBSTRATE_CLI_IMPL_VERSION")); +polkadot_node_core_pvf::decl_puppet_worker_main!(); diff --git a/parachain/test-parachains/undying/collator/build.rs b/parachain/test-parachains/undying/collator/build.rs deleted file mode 100644 index 8aa155080a31..000000000000 --- a/parachain/test-parachains/undying/collator/build.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the puppet worker when the - // version changes. - substrate_build_script_utils::rerun_if_git_head_changed(); -} diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 0ef19aae711e..0ed72aab421e 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -64,9 +64,13 @@ fn main() -> Result<()> { enable_beefy: false, jaeger_agent: None, telemetry_worker_handle: None, + // Skip the version check since we don't use external binaries. + node_version: None, workers_path: None, workers_names: None, - dont_use_external_workers: false, + // Don't use workers to make this test binary self-contained and easier to + // use. External binaries are for increased security in production. + dont_use_external_workers: true, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, From 019bd9780dc42c7bb104a96ee57b0af5f9c770d8 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 20 Jul 2023 17:41:56 +0200 Subject: [PATCH 31/48] Add back rerun to PVF workers --- node/core/pvf/execute-worker/build.rs | 3 +++ node/core/pvf/prepare-worker/build.rs | 3 +++ node/service/src/lib.rs | 3 ++- node/service/src/workers.rs | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/node/core/pvf/execute-worker/build.rs b/node/core/pvf/execute-worker/build.rs index 40e9f832586e..7c249f50ccc4 100644 --- a/node/core/pvf/execute-worker/build.rs +++ b/node/core/pvf/execute-worker/build.rs @@ -16,4 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the worker when the version + // changes. + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/core/pvf/prepare-worker/build.rs b/node/core/pvf/prepare-worker/build.rs index 40e9f832586e..7c249f50ccc4 100644 --- a/node/core/pvf/prepare-worker/build.rs +++ b/node/core/pvf/prepare-worker/build.rs @@ -16,4 +16,7 @@ fn main() { substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the worker when the version + // changes. + substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index bc23a62257b5..f2150c621e05 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -633,7 +633,8 @@ pub struct NewFullParams { pub enable_beefy: bool, pub jaeger_agent: Option, pub telemetry_worker_handle: Option, - /// The version of the node. `None` can be passed to skip the version check (only for tests). + /// The version of the node. `None` can be passed to skip the node/worker version check (only + /// for tests). pub node_version: Option, /// An optional path to a directory containing the workers. pub workers_path: Option, diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index ab3d2968208f..9f0591d0fb7c 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -241,6 +241,8 @@ echo {} ) } + /// Sets up an empty temp dir structure where the workers can be put by tests. Uses the temp dir + /// to override the standard locations where the node searches for the workers. fn with_temp_dir_structure( f: impl FnOnce(PathBuf) -> Result<(), Box>, ) -> Result<(), Box> { From df1117a905b1582ffd1d788f7eda2657b4d69c38 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Fri, 21 Jul 2023 15:01:46 +0200 Subject: [PATCH 32/48] Move worker binaries into files in cli crate As a final optimization I've separated out each worker binary from its own crate into the CLI crate. Before, the worker bin shared a crate with the worker lib, so when the binaries got recompiled so did the libs and everything transitively depending on the libs. This commit fixes this regression that was causing recompiles after every commit. --- Cargo.lock | 18 +-------- Cargo.toml | 13 ------- cli/Cargo.toml | 12 ++++++ .../src/main.rs => cli/bin/execute-worker.rs | 3 +- .../src/main.rs => cli/bin/prepare-worker.rs | 5 +-- cli/build.rs | 4 +- cli/src/cli.rs | 3 ++ cli/src/command.rs | 4 +- cli/tests/lib.rs | 38 +++++++++++++++++++ node/core/pvf/build.rs | 19 ---------- node/core/pvf/execute-worker/Cargo.toml | 3 -- node/core/pvf/execute-worker/build.rs | 22 ----------- node/core/pvf/execute-worker/src/lib.rs | 2 - node/core/pvf/prepare-worker/Cargo.toml | 3 -- node/core/pvf/prepare-worker/build.rs | 22 ----------- node/core/pvf/prepare-worker/src/lib.rs | 2 - node/malus/Cargo.toml | 23 +---------- node/malus/build.rs | 21 ---------- node/metrics/Cargo.toml | 6 --- .../test-parachains/adder/collator/Cargo.toml | 3 -- .../undying/collator/Cargo.toml | 3 -- 21 files changed, 62 insertions(+), 167 deletions(-) rename node/core/pvf/execute-worker/src/main.rs => cli/bin/execute-worker.rs (88%) rename node/core/pvf/prepare-worker/src/main.rs => cli/bin/prepare-worker.rs (83%) create mode 100644 cli/tests/lib.rs delete mode 100644 node/core/pvf/build.rs delete mode 100644 node/core/pvf/execute-worker/build.rs delete mode 100644 node/core/pvf/prepare-worker/build.rs delete mode 100644 node/malus/build.rs diff --git a/Cargo.lock b/Cargo.lock index d3fdb3dc1f36..75076f3ba75f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6944,11 +6944,8 @@ dependencies = [ "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", - "sp-tracing", "substrate-rpc-client", "tempfile", "tikv-jemallocator", @@ -7074,6 +7071,7 @@ dependencies = [ "frame-benchmarking-cli", "futures", "log", + "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-metrics", @@ -7091,6 +7089,7 @@ dependencies = [ "sp-io", "sp-keyring", "sp-maybe-compressed-blob", + "sp-tracing", "substrate-build-script-utils", "thiserror", "try-runtime-cli", @@ -7593,7 +7592,6 @@ dependencies = [ "sp-core", "sp-maybe-compressed-blob", "sp-tracing", - "substrate-build-script-utils", "tikv-jemalloc-ctl", "tokio", "tracing-gum", @@ -7616,7 +7614,6 @@ dependencies = [ "sp-io", "sp-maybe-compressed-blob", "sp-tracing", - "substrate-build-script-utils", "tikv-jemalloc-ctl", "tokio", "tracing-gum", @@ -7671,9 +7668,6 @@ dependencies = [ "hyper", "log", "parity-scale-codec", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-primitives", "polkadot-test-service", "prioritized-metered-channel", @@ -7682,7 +7676,6 @@ dependencies = [ "sc-service", "sc-tracing", "sp-keyring", - "sp-tracing", "substrate-prometheus-endpoint", "substrate-test-utils", "tempfile", @@ -8398,9 +8391,6 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -8410,8 +8400,6 @@ dependencies = [ "rand 0.8.5", "sp-core", "sp-keystore", - "sp-tracing", - "substrate-build-script-utils", "tracing-gum", ] @@ -12646,7 +12634,6 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", - "substrate-build-script-utils", "substrate-test-utils", "test-parachain-adder", "tokio", @@ -12695,7 +12682,6 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", - "substrate-build-script-utils", "substrate-test-utils", "test-parachain-undying", "tokio", diff --git a/Cargo.toml b/Cargo.toml index e8f45748d6a4..80d0acee587d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,14 +12,6 @@ authors.workspace = true edition.workspace = true version.workspace = true -[[bin]] -name = "polkadot-execute-worker" -path = "node/core/pvf/execute-worker/src/main.rs" - -[[bin]] -name = "polkadot-prepare-worker" -path = "node/core/pvf/prepare-worker/src/main.rs" - [workspace.package] authors = ["Parity Technologies "] edition = "2021" @@ -36,11 +28,6 @@ polkadot-node-core-pvf = { path = "node/core/pvf" } polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } polkadot-overseer = { path = "node/overseer" } -# Needed for worker binaries. -polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [dev-dependencies] assert_cmd = "2.0.4" nix = { version = "0.26.1", features = ["signal"] } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e7aa562880cc..6ca61e8eae33 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -13,6 +13,14 @@ wasm-opt = false [lib] crate-type = ["cdylib", "rlib"] +[[bin]] +name = "polkadot-execute-worker" +path = "bin/execute-worker.rs" + +[[bin]] +name = "polkadot-prepare-worker" +path = "bin/prepare-worker.rs" + [dependencies] clap = { version = "4.0.9", features = ["derive"], optional = true } log = "0.4.17" @@ -40,6 +48,10 @@ sc-sysinfo = { git = "https://github.com/paritytech/substrate", branch = "master sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" } sc-storage-monitor = { git = "https://github.com/paritytech/substrate", branch = "master" } +# Needed for worker binaries. +polkadot-node-core-pvf-common = { path = "../node/core/pvf/common" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [build-dependencies] substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/core/pvf/execute-worker/src/main.rs b/cli/bin/execute-worker.rs similarity index 88% rename from node/core/pvf/execute-worker/src/main.rs rename to cli/bin/execute-worker.rs index 203929ee71e1..72cab799d753 100644 --- a/node/core/pvf/execute-worker/src/main.rs +++ b/cli/bin/execute-worker.rs @@ -19,6 +19,5 @@ polkadot_node_core_pvf_common::decl_worker_main!( "execute-worker", polkadot_node_core_pvf_execute_worker::worker_entrypoint, - // Defined in lib.rs because env! doesn't work here. - polkadot_node_core_pvf_execute_worker::WORKER_IMPL_VERSION + env!("SUBSTRATE_CLI_IMPL_VERSION") ); diff --git a/node/core/pvf/prepare-worker/src/main.rs b/cli/bin/prepare-worker.rs similarity index 83% rename from node/core/pvf/prepare-worker/src/main.rs rename to cli/bin/prepare-worker.rs index 4fd41d5c7283..33fdd3097a59 100644 --- a/node/core/pvf/prepare-worker/src/main.rs +++ b/cli/bin/prepare-worker.rs @@ -16,11 +16,8 @@ //! Prepare worker. -pub use polkadot_node_core_pvf_common::decl_worker_main; - polkadot_node_core_pvf_common::decl_worker_main!( "prepare-worker", polkadot_node_core_pvf_prepare_worker::worker_entrypoint, - // Defined in lib.rs because env! doesn't work here. - polkadot_node_core_pvf_prepare_worker::WORKER_IMPL_VERSION + polkadot_cli::CLI_IMPL_VERSION ); diff --git a/cli/build.rs b/cli/build.rs index 483cc04163fc..bd6765f53e9e 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -19,7 +19,7 @@ fn main() { println!("cargo:rustc-cfg=build_type=\"{}\"", profile); } substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the node when the version - // changes. + // For the node/worker version check, make sure we always rebuild the node and binary workers + // when the version changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 0a1b8d003d86..a19f2cb526cd 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -19,6 +19,9 @@ use clap::Parser; use std::path::PathBuf; +/// The version of the node and workers. +pub const CLI_IMPL_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); + #[allow(missing_docs)] #[derive(Debug, Parser)] pub enum Subcommand { diff --git a/cli/src/command.rs b/cli/src/command.rs index a6c4663cf404..a5771e8188c9 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::cli::{Cli, Subcommand}; +use crate::cli::{Cli, Subcommand, CLI_IMPL_VERSION}; use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; use futures::future::TryFutureExt; use log::info; @@ -289,7 +289,7 @@ where enable_beefy, jaeger_agent, telemetry_worker_handle: None, - node_version: Some(env!("SUBSTRATE_CLI_IMPL_VERSION").to_string()), + node_version: Some(CLI_IMPL_VERSION.to_string()), workers_path: cli.run.workers_path, workers_names: None, dont_use_external_workers: false, diff --git a/cli/tests/lib.rs b/cli/tests/lib.rs new file mode 100644 index 000000000000..d84aaa877047 --- /dev/null +++ b/cli/tests/lib.rs @@ -0,0 +1,38 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_cli::CLI_IMPL_VERSION; +use std::process::Command; + +const PREPARE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-prepare-worker"); +const EXECUTE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-execute-worker"); + +#[test] +fn worker_binaries_are_built_and_have_correct_version() { + let prep_worker_version = + Command::new(&PREPARE_WORKER_EXE).args(["--version"]).output().unwrap().stdout; + let prep_worker_version = std::str::from_utf8(&prep_worker_version) + .expect("version is printed as a string; qed") + .trim(); + assert_eq!(prep_worker_version, CLI_IMPL_VERSION); + + let exec_worker_version = + Command::new(&EXECUTE_WORKER_EXE).args(["--version"]).output().unwrap().stdout; + let exec_worker_version = std::str::from_utf8(&exec_worker_version) + .expect("version is printed as a string; qed") + .trim(); + assert_eq!(exec_worker_version, CLI_IMPL_VERSION); +} diff --git a/node/core/pvf/build.rs b/node/core/pvf/build.rs deleted file mode 100644 index 40e9f832586e..000000000000 --- a/node/core/pvf/build.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); -} diff --git a/node/core/pvf/execute-worker/Cargo.toml b/node/core/pvf/execute-worker/Cargo.toml index 86bd9e3d4be7..167e8b4311a3 100644 --- a/node/core/pvf/execute-worker/Cargo.toml +++ b/node/core/pvf/execute-worker/Cargo.toml @@ -25,8 +25,5 @@ sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master [target.'cfg(target_os = "linux")'.dependencies] tikv-jemalloc-ctl = "0.5.0" -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } - [features] builder = [] diff --git a/node/core/pvf/execute-worker/build.rs b/node/core/pvf/execute-worker/build.rs deleted file mode 100644 index 7c249f50ccc4..000000000000 --- a/node/core/pvf/execute-worker/build.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the worker when the version - // changes. - substrate_build_script_utils::rerun_if_git_head_changed(); -} diff --git a/node/core/pvf/execute-worker/src/lib.rs b/node/core/pvf/execute-worker/src/lib.rs index e36b34f66c80..d90cac2522fd 100644 --- a/node/core/pvf/execute-worker/src/lib.rs +++ b/node/core/pvf/execute-worker/src/lib.rs @@ -43,8 +43,6 @@ use std::{ }; use tokio::{io, net::UnixStream}; -pub const WORKER_IMPL_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); - // Wasmtime powers the Substrate Executor. It compiles the wasm bytecode into native code. // That native code does not create any stacks and just reuses the stack of the thread that // wasmtime was invoked from. diff --git a/node/core/pvf/prepare-worker/Cargo.toml b/node/core/pvf/prepare-worker/Cargo.toml index 2331791faf22..3bd1fd43b673 100644 --- a/node/core/pvf/prepare-worker/Cargo.toml +++ b/node/core/pvf/prepare-worker/Cargo.toml @@ -28,9 +28,6 @@ sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master [target.'cfg(target_os = "linux")'.dependencies] tikv-jemalloc-ctl = "0.5.0" -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } - [features] builder = [] jemalloc-allocator = ["dep:tikv-jemalloc-ctl"] diff --git a/node/core/pvf/prepare-worker/build.rs b/node/core/pvf/prepare-worker/build.rs deleted file mode 100644 index 7c249f50ccc4..000000000000 --- a/node/core/pvf/prepare-worker/build.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the worker when the version - // changes. - substrate_build_script_utils::rerun_if_git_head_changed(); -} diff --git a/node/core/pvf/prepare-worker/src/lib.rs b/node/core/pvf/prepare-worker/src/lib.rs index bd1413b77597..228ad3d4668d 100644 --- a/node/core/pvf/prepare-worker/src/lib.rs +++ b/node/core/pvf/prepare-worker/src/lib.rs @@ -51,8 +51,6 @@ use std::{ }; use tokio::{io, net::UnixStream}; -pub const WORKER_IMPL_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); - /// Contains the bytes for a successfully compiled artifact. pub struct CompiledArtifact(Vec); diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 4aaf47070816..d030fa162c22 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -12,20 +12,8 @@ publish = false name = "malus" path = "src/malus.rs" -# Use artifact dependencies once stable. -# See https://github.com/rust-lang/cargo/issues/9096. -[[bin]] -name = "polkadot-execute-worker" -path = "../core/pvf/execute-worker/src/main.rs" -# Prevent rustdoc error. Already documented from top-level Cargo.toml. -doc = false -[[bin]] -name = "polkadot-prepare-worker" -path = "../core/pvf/prepare-worker/src/main.rs" -# Prevent rustdoc error. Already documented from top-level Cargo.toml. -doc = false - [dependencies] +# Note that malus depends on the worker binaries that are built by polkadot-cli. polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } polkadot-node-subsystem = { path = "../subsystem" } polkadot-node-subsystem-util = { path = "../subsystem-util" } @@ -47,12 +35,6 @@ gum = { package = "tracing-gum", path = "../gum/" } erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } rand = "0.8.5" -# Required for worker binaries to build. -polkadot-node-core-pvf-common = { path = "../core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [features] default = [] fast-runtime = ["polkadot-cli/fast-runtime"] @@ -61,6 +43,3 @@ fast-runtime = ["polkadot-cli/fast-runtime"] polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = { version = "0.3.21", features = ["thread-pool"] } - -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/malus/build.rs b/node/malus/build.rs deleted file mode 100644 index 545778102b40..000000000000 --- a/node/malus/build.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - // For the node/worker version check, make sure we always rebuild the malus node and binary - // workers when the version changes. - substrate_build_script_utils::rerun_if_git_head_changed(); -} diff --git a/node/metrics/Cargo.toml b/node/metrics/Cargo.toml index ab82473e96b1..2e9bc22d1cb3 100644 --- a/node/metrics/Cargo.toml +++ b/node/metrics/Cargo.toml @@ -23,12 +23,6 @@ primitives = { package = "polkadot-primitives", path = "../../primitives/" } bs58 = { version = "0.4.0", features = ["alloc"] } log = "0.4.17" -# Required for worker binaries to build. -polkadot-node-core-pvf-common = { path = "../core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [dev-dependencies] assert_cmd = "2.0.4" tempfile = "3.2.0" diff --git a/parachain/test-parachains/adder/collator/Cargo.toml b/parachain/test-parachains/adder/collator/Cargo.toml index 79fff4566496..29a10069e3e0 100644 --- a/parachain/test-parachains/adder/collator/Cargo.toml +++ b/parachain/test-parachains/adder/collator/Cargo.toml @@ -45,6 +45,3 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } - -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/parachain/test-parachains/undying/collator/Cargo.toml b/parachain/test-parachains/undying/collator/Cargo.toml index 6669f0f47ed2..f63757a20958 100644 --- a/parachain/test-parachains/undying/collator/Cargo.toml +++ b/parachain/test-parachains/undying/collator/Cargo.toml @@ -45,6 +45,3 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } - -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } From cde8dc62d08dffa3df11535cf1b6221497bf4a5b Mon Sep 17 00:00:00 2001 From: Marcin S Date: Sun, 23 Jul 2023 09:08:06 +0200 Subject: [PATCH 33/48] Fix bug (was passing worker version for node version) --- cli/src/cli.rs | 4 ++-- cli/src/command.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index a19f2cb526cd..1c640873cf2e 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -19,8 +19,8 @@ use clap::Parser; use std::path::PathBuf; -/// The version of the node and workers. -pub const CLI_IMPL_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); +/// The version of the node. The passed-in version of the workers should match this. +pub const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); #[allow(missing_docs)] #[derive(Debug, Parser)] diff --git a/cli/src/command.rs b/cli/src/command.rs index a5771e8188c9..0a0c523e9b63 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::cli::{Cli, Subcommand, CLI_IMPL_VERSION}; +use crate::cli::{Cli, Subcommand, NODE_VERSION}; use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; use futures::future::TryFutureExt; use log::info; @@ -55,7 +55,7 @@ impl SubstrateCli for Cli { } fn impl_version() -> String { - env!("SUBSTRATE_CLI_IMPL_VERSION").into() + NODE_VERSION.into() } fn description() -> String { @@ -289,7 +289,7 @@ where enable_beefy, jaeger_agent, telemetry_worker_handle: None, - node_version: Some(CLI_IMPL_VERSION.to_string()), + node_version: Some(NODE_VERSION.to_string()), workers_path: cli.run.workers_path, workers_names: None, dont_use_external_workers: false, @@ -444,7 +444,7 @@ pub fn run() -> Result<()> { { polkadot_node_core_pvf_prepare_worker::worker_entrypoint( &cmd.socket_path, - Some(&cmd.node_impl_version), + Some(NODE_VERSION), Some(&cmd.node_impl_version), ); Ok(()) @@ -467,7 +467,7 @@ pub fn run() -> Result<()> { { polkadot_node_core_pvf_execute_worker::worker_entrypoint( &cmd.socket_path, - Some(&cmd.node_impl_version), + Some(NODE_VERSION), Some(&cmd.node_impl_version), ); Ok(()) From 98323d210e971371e27bc92592674ef626d0c9c7 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 24 Jul 2023 16:05:06 +0200 Subject: [PATCH 34/48] Move workers out of cli into root src/bin/ dir - [X] Pass in node version from top-level (polkadot) - [X] Add build.rs with rerun-git-head to root dir --- Cargo.lock | 8 +++++-- Cargo.toml | 16 ++++++++++++++ build.rs | 22 +++++++++++++++++++ cli/Cargo.toml | 12 ---------- cli/build.rs | 4 ++-- .../test-parachains/adder/collator/Cargo.toml | 3 +++ .../test-parachains/adder/collator/build.rs | 19 ++++++++++++++++ .../adder/collator/src/main.rs | 3 +-- .../undying/collator/Cargo.toml | 3 +++ .../test-parachains/undying/collator/build.rs | 19 ++++++++++++++++ .../undying/collator/src/main.rs | 3 +-- {cli => src}/bin/execute-worker.rs | 0 {cli => src}/bin/prepare-worker.rs | 2 +- cli/tests/lib.rs => tests/workers.rs | 8 +++---- 14 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 build.rs create mode 100644 parachain/test-parachains/adder/collator/build.rs create mode 100644 parachain/test-parachains/undying/collator/build.rs rename {cli => src}/bin/execute-worker.rs (100%) rename {cli => src}/bin/prepare-worker.rs (96%) rename cli/tests/lib.rs => tests/workers.rs (87%) diff --git a/Cargo.lock b/Cargo.lock index 75076f3ba75f..aa4da6783d53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6944,8 +6944,12 @@ dependencies = [ "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", + "sp-tracing", + "substrate-build-script-utils", "substrate-rpc-client", "tempfile", "tikv-jemallocator", @@ -7071,7 +7075,6 @@ dependencies = [ "frame-benchmarking-cli", "futures", "log", - "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-metrics", @@ -7089,7 +7092,6 @@ dependencies = [ "sp-io", "sp-keyring", "sp-maybe-compressed-blob", - "sp-tracing", "substrate-build-script-utils", "thiserror", "try-runtime-cli", @@ -12634,6 +12636,7 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", + "substrate-build-script-utils", "substrate-test-utils", "test-parachain-adder", "tokio", @@ -12682,6 +12685,7 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", + "substrate-build-script-utils", "substrate-test-utils", "test-parachain-undying", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 80d0acee587d..a4ce09489351 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,14 @@ name = "polkadot" path = "src/main.rs" +[[bin]] +name = "polkadot-execute-worker" +path = "src/bin/execute-worker.rs" + +[[bin]] +name = "polkadot-prepare-worker" +path = "src/bin/prepare-worker.rs" + [package] name = "polkadot" description = "Implementation of a `https://polkadot.network` node in Rust based on the Substrate framework." @@ -28,6 +36,11 @@ polkadot-node-core-pvf = { path = "node/core/pvf" } polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } polkadot-overseer = { path = "node/overseer" } +# Needed for worker binaries. +polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + [dev-dependencies] assert_cmd = "2.0.4" nix = { version = "0.26.1", features = ["signal"] } @@ -36,6 +49,9 @@ tokio = "1.24.2" substrate-rpc-client = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-core-primitives = { path = "core-primitives" } +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + [workspace] members = [ "cli", diff --git a/build.rs b/build.rs new file mode 100644 index 000000000000..84fe22e23ed6 --- /dev/null +++ b/build.rs @@ -0,0 +1,22 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the node and binary workers + // when the version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); +} diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 6ca61e8eae33..e7aa562880cc 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -13,14 +13,6 @@ wasm-opt = false [lib] crate-type = ["cdylib", "rlib"] -[[bin]] -name = "polkadot-execute-worker" -path = "bin/execute-worker.rs" - -[[bin]] -name = "polkadot-prepare-worker" -path = "bin/prepare-worker.rs" - [dependencies] clap = { version = "4.0.9", features = ["derive"], optional = true } log = "0.4.17" @@ -48,10 +40,6 @@ sc-sysinfo = { git = "https://github.com/paritytech/substrate", branch = "master sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" } sc-storage-monitor = { git = "https://github.com/paritytech/substrate", branch = "master" } -# Needed for worker binaries. -polkadot-node-core-pvf-common = { path = "../node/core/pvf/common" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } - [build-dependencies] substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/cli/build.rs b/cli/build.rs index bd6765f53e9e..483cc04163fc 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -19,7 +19,7 @@ fn main() { println!("cargo:rustc-cfg=build_type=\"{}\"", profile); } substrate_build_script_utils::generate_cargo_keys(); - // For the node/worker version check, make sure we always rebuild the node and binary workers - // when the version changes. + // For the node/worker version check, make sure we always rebuild the node when the version + // changes. substrate_build_script_utils::rerun_if_git_head_changed(); } diff --git a/parachain/test-parachains/adder/collator/Cargo.toml b/parachain/test-parachains/adder/collator/Cargo.toml index 29a10069e3e0..79fff4566496 100644 --- a/parachain/test-parachains/adder/collator/Cargo.toml +++ b/parachain/test-parachains/adder/collator/Cargo.toml @@ -45,3 +45,6 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/parachain/test-parachains/adder/collator/build.rs b/parachain/test-parachains/adder/collator/build.rs new file mode 100644 index 000000000000..40e9f832586e --- /dev/null +++ b/parachain/test-parachains/adder/collator/build.rs @@ -0,0 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); +} diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index d6a59411c7f8..a6c9d89ffcf7 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -64,8 +64,7 @@ fn main() -> Result<()> { enable_beefy: false, jaeger_agent: None, telemetry_worker_handle: None, - // Skip the version check since we don't use external binaries. - node_version: None, + node_version: Some(env!("SUBSTRATE_CLI_IMPL_VERSION").into()), workers_path: None, workers_names: None, // Don't use workers to make this test binary self-contained and easier to diff --git a/parachain/test-parachains/undying/collator/Cargo.toml b/parachain/test-parachains/undying/collator/Cargo.toml index f63757a20958..6669f0f47ed2 100644 --- a/parachain/test-parachains/undying/collator/Cargo.toml +++ b/parachain/test-parachains/undying/collator/Cargo.toml @@ -45,3 +45,6 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/parachain/test-parachains/undying/collator/build.rs b/parachain/test-parachains/undying/collator/build.rs new file mode 100644 index 000000000000..40e9f832586e --- /dev/null +++ b/parachain/test-parachains/undying/collator/build.rs @@ -0,0 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); +} diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 0ed72aab421e..7c7ad53f7216 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -64,8 +64,7 @@ fn main() -> Result<()> { enable_beefy: false, jaeger_agent: None, telemetry_worker_handle: None, - // Skip the version check since we don't use external binaries. - node_version: None, + node_version: Some(env!("SUBSTRATE_CLI_IMPL_VERSION").into()), workers_path: None, workers_names: None, // Don't use workers to make this test binary self-contained and easier to diff --git a/cli/bin/execute-worker.rs b/src/bin/execute-worker.rs similarity index 100% rename from cli/bin/execute-worker.rs rename to src/bin/execute-worker.rs diff --git a/cli/bin/prepare-worker.rs b/src/bin/prepare-worker.rs similarity index 96% rename from cli/bin/prepare-worker.rs rename to src/bin/prepare-worker.rs index 33fdd3097a59..695f66cc7b7d 100644 --- a/cli/bin/prepare-worker.rs +++ b/src/bin/prepare-worker.rs @@ -19,5 +19,5 @@ polkadot_node_core_pvf_common::decl_worker_main!( "prepare-worker", polkadot_node_core_pvf_prepare_worker::worker_entrypoint, - polkadot_cli::CLI_IMPL_VERSION + env!("SUBSTRATE_CLI_IMPL_VERSION") ); diff --git a/cli/tests/lib.rs b/tests/workers.rs similarity index 87% rename from cli/tests/lib.rs rename to tests/workers.rs index d84aaa877047..2872a1298dcd 100644 --- a/cli/tests/lib.rs +++ b/tests/workers.rs @@ -14,25 +14,25 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_cli::CLI_IMPL_VERSION; +use polkadot_cli::NODE_VERSION; use std::process::Command; const PREPARE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-prepare-worker"); const EXECUTE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-execute-worker"); #[test] -fn worker_binaries_are_built_and_have_correct_version() { +fn worker_binaries_have_same_version_as_node() { let prep_worker_version = Command::new(&PREPARE_WORKER_EXE).args(["--version"]).output().unwrap().stdout; let prep_worker_version = std::str::from_utf8(&prep_worker_version) .expect("version is printed as a string; qed") .trim(); - assert_eq!(prep_worker_version, CLI_IMPL_VERSION); + assert_eq!(prep_worker_version, NODE_VERSION); let exec_worker_version = Command::new(&EXECUTE_WORKER_EXE).args(["--version"]).output().unwrap().stdout; let exec_worker_version = std::str::from_utf8(&exec_worker_version) .expect("version is printed as a string; qed") .trim(); - assert_eq!(exec_worker_version, CLI_IMPL_VERSION); + assert_eq!(exec_worker_version, NODE_VERSION); } From ecf0af88f9ef1c54273c3086366c222b417566a5 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 24 Jul 2023 18:54:48 +0200 Subject: [PATCH 35/48] Add some sanity checks for workers to dockerfiles --- scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile | 2 ++ scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile b/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile index ef3ebd14ae60..aebbbdcf1b7f 100644 --- a/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +++ b/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile @@ -39,6 +39,8 @@ USER polkadot # check if executable works in this container RUN /usr/local/bin/polkadot --version +RUN /usr/local/bin/polkadot-execute-worker --version +RUN /usr/local/bin/polkadot-prepare-worker --version EXPOSE 30333 9933 9944 VOLUME ["/polkadot"] diff --git a/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile b/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile index ba0a79e78187..74b5c7f48f88 100644 --- a/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile +++ b/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile @@ -44,6 +44,8 @@ USER polkadot # check if executable works in this container RUN /usr/bin/polkadot --version +RUN /usr/bin/polkadot-execute-worker --version +RUN /usr/bin/polkadot-prepare-worker --version EXPOSE 30333 9933 9944 VOLUME ["/polkadot"] From 3f929bb04dcdec450082be795c98ba439c348a7f Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 24 Jul 2023 19:05:23 +0200 Subject: [PATCH 36/48] Update malus + [X] Make it self-contained + [X] Undo multiple binary changes --- cli/src/command.rs | 13 +++++++++++-- node/malus/Cargo.toml | 9 ++++----- node/malus/src/malus.rs | 9 ++++++++- scripts/ci/dockerfiles/malus_injected.Dockerfile | 4 ++-- scripts/ci/gitlab/pipeline/build.yml | 2 -- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/cli/src/command.rs b/cli/src/command.rs index 0a0c523e9b63..a098731511d3 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -222,8 +222,15 @@ pub fn run_node( run: Cli, overseer_gen: impl service::OverseerGen, malus_finality_delay: Option, + dont_use_external_workers: bool, ) -> Result<()> { - run_node_inner(run, overseer_gen, malus_finality_delay, |_logger_builder, _config| {}) + run_node_inner( + run, + overseer_gen, + malus_finality_delay, + |_logger_builder, _config| {}, + dont_use_external_workers, + ) } fn run_node_inner( @@ -231,6 +238,7 @@ fn run_node_inner( overseer_gen: impl service::OverseerGen, maybe_malus_finality_delay: Option, logger_hook: F, + dont_use_external_workers: bool, ) -> Result<()> where F: FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration), @@ -292,7 +300,7 @@ where node_version: Some(NODE_VERSION.to_string()), workers_path: cli.run.workers_path, workers_names: None, - dont_use_external_workers: false, + dont_use_external_workers, overseer_enable_anyways: false, overseer_gen, overseer_message_channel_capacity_override: cli @@ -348,6 +356,7 @@ pub fn run() -> Result<()> { service::RealOverseerGen, None, polkadot_node_metrics::logger_hook(), + false, ), Some(Subcommand::BuildSpec(cmd)) => { let runner = cli.create_runner(cmd)?; diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index d030fa162c22..e9ed7406367a 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -13,7 +13,6 @@ name = "malus" path = "src/malus.rs" [dependencies] -# Note that malus depends on the worker binaries that are built by polkadot-cli. polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } polkadot-node-subsystem = { path = "../subsystem" } polkadot-node-subsystem-util = { path = "../subsystem-util" } @@ -35,11 +34,11 @@ gum = { package = "tracing-gum", path = "../gum/" } erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } rand = "0.8.5" -[features] -default = [] -fast-runtime = ["polkadot-cli/fast-runtime"] - [dev-dependencies] polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = { version = "0.3.21", features = ["thread-pool"] } + +[features] +default = [] +fast-runtime = ["polkadot-cli/fast-runtime"] diff --git a/node/malus/src/malus.rs b/node/malus/src/malus.rs index 69dd7c869fc0..791112079425 100644 --- a/node/malus/src/malus.rs +++ b/node/malus/src/malus.rs @@ -55,7 +55,12 @@ impl MalusCli { NemesisVariant::BackGarbageCandidate(opts) => { let BackGarbageCandidateOptions { percentage, cli } = opts; - polkadot_cli::run_node(cli, BackGarbageCandidates { percentage }, finality_delay)? + polkadot_cli::run_node( + cli, + BackGarbageCandidates { percentage }, + finality_delay, + true, + )? }, NemesisVariant::SuggestGarbageCandidate(opts) => { let SuggestGarbageCandidateOptions { percentage, cli } = opts; @@ -64,6 +69,7 @@ impl MalusCli { cli, SuggestGarbageCandidates { percentage }, finality_delay, + true, )? }, NemesisVariant::DisputeAncestor(opts) => { @@ -78,6 +84,7 @@ impl MalusCli { cli, DisputeValidCandidates { fake_validation, fake_validation_error, percentage }, finality_delay, + true, )? }, } diff --git a/scripts/ci/dockerfiles/malus_injected.Dockerfile b/scripts/ci/dockerfiles/malus_injected.Dockerfile index 7ab3d74e3b83..1594350096f8 100644 --- a/scripts/ci/dockerfiles/malus_injected.Dockerfile +++ b/scripts/ci/dockerfiles/malus_injected.Dockerfile @@ -38,8 +38,8 @@ RUN apt-get update && \ --uid 10000 nonroot -# add adder-collator binary to docker image -COPY ./malus ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin +# add malus binary to docker image +COPY ./malus /usr/local/bin USER nonroot diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index a7f942c79a85..73702be0efc2 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -83,8 +83,6 @@ build-malus: # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. - - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" From 508669c2a1fc7a10968a6904127cf2910c412405 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Mon, 24 Jul 2023 19:07:25 +0200 Subject: [PATCH 37/48] Try to fix clippy errors --- node/core/pvf/src/execute/queue.rs | 2 +- node/core/pvf/src/prepare/pool.rs | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/node/core/pvf/src/execute/queue.rs b/node/core/pvf/src/execute/queue.rs index 57c19cd04533..33a1c6f89709 100644 --- a/node/core/pvf/src/execute/queue.rs +++ b/node/core/pvf/src/execute/queue.rs @@ -433,7 +433,7 @@ async fn spawn_worker_task( &program_path, job.executor_params.clone(), spawn_timeout, - node_version.as_ref().map(|v| v.as_str()), + node_version.as_deref(), ) .await { diff --git a/node/core/pvf/src/prepare/pool.rs b/node/core/pvf/src/prepare/pool.rs index 3903409a870b..1e8ccc7365bf 100644 --- a/node/core/pvf/src/prepare/pool.rs +++ b/node/core/pvf/src/prepare/pool.rs @@ -264,13 +264,7 @@ async fn spawn_worker_task( use futures_timer::Delay; loop { - match worker_intf::spawn( - &program_path, - spawn_timeout, - node_version.as_ref().map(|v| v.as_str()), - ) - .await - { + match worker_intf::spawn(&program_path, spawn_timeout, node_version.as_deref()).await { Ok((idle, handle)) => break PoolEvent::Spawn(idle, handle), Err(err) => { gum::warn!(target: LOG_TARGET, "failed to spawn a prepare worker: {:?}", err); From 432dcf3a8897ba70c8403838f7f710d427acbcfa Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 25 Jul 2023 15:38:29 +0200 Subject: [PATCH 38/48] Address `cargo run` issue - [X] Add default-run for polkadot - [X] Add note about installation to error --- Cargo.toml | 1 + node/service/src/lib.rs | 3 +- node/service/src/workers.rs | 58 ++++++++++++++++++++++--------------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a4ce09489351..260cdbd71c82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ description = "Implementation of a `https://polkadot.network` node in Rust based license = "GPL-3.0-only" rust-version = "1.64.0" # workspace properties readme = "README.md" +default-run = "polkadot" authors.workspace = true edition.workspace = true version.workspace = true diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index f2150c621e05..b33c4a7e3171 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -245,9 +245,10 @@ pub enum Error { InvalidWorkerBinaries { prep_worker_path: PathBuf, exec_worker_path: PathBuf }, #[cfg(feature = "full-node")] - #[error("Worker binaries could not be found at given workers path ({given_workers_path:?}), polkadot binary directory, or /usr/lib/polkadot, workers names: {workers_names:?}")] + #[error("Worker binaries could not be found, make sure polkadot was built/installed correctly. Searched given workers path ({given_workers_path:?}), polkadot binary path ({current_exe_path:?}), and lib path (/usr/lib/polkadot), workers names: {workers_names:?}")] MissingWorkerBinaries { given_workers_path: Option, + current_exe_path: PathBuf, workers_names: Option<(String, String)>, }, diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index 9f0591d0fb7c..d1c9e0bc4a13 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -58,7 +58,12 @@ pub fn determine_workers_paths( ) -> Result<(PathBuf, PathBuf), Error> { let mut workers_paths = list_workers_paths(given_workers_path.clone(), workers_names.clone())?; if workers_paths.is_empty() { - return Err(Error::MissingWorkerBinaries { given_workers_path, workers_names }) + let current_exe_path = get_exe_path()?; + return Err(Error::MissingWorkerBinaries { + given_workers_path, + current_exe_path, + workers_names, + }) } else if workers_paths.len() > 1 { log::warn!("multiple sets of worker binaries found ({:?})", workers_paths,); } @@ -128,12 +133,7 @@ fn list_workers_paths( // Consider the polkadot binary directory. { - let mut exe_path = std::env::current_exe()?; - let _ = exe_path.pop(); // executable file will always have a parent directory. - #[cfg(test)] - if let Some(ref path_override) = *workers_exe_path_override().lock().unwrap() { - exe_path = path_override.clone(); - } + let exe_path = get_exe_path()?; let (prep_worker, exec_worker) = build_worker_paths(exe_path.clone(), workers_names.clone()); @@ -176,6 +176,16 @@ fn list_workers_paths( Ok(workers_paths) } +fn get_exe_path() -> Result { + let mut exe_path = std::env::current_exe()?; + let _ = exe_path.pop(); // executable file will always have a parent directory. + #[cfg(test)] + if let Some(ref path_override) = *workers_exe_path_override().lock().unwrap() { + exe_path = path_override.clone(); + } + Ok(exe_path) +} + fn build_worker_paths( worker_dir: PathBuf, workers_names: Option<(String, String)>, @@ -244,7 +254,7 @@ echo {} /// Sets up an empty temp dir structure where the workers can be put by tests. Uses the temp dir /// to override the standard locations where the node searches for the workers. fn with_temp_dir_structure( - f: impl FnOnce(PathBuf) -> Result<(), Box>, + f: impl FnOnce(PathBuf, PathBuf) -> Result<(), Box>, ) -> Result<(), Box> { // Set up /usr/lib/polkadot and /usr/bin, both empty. @@ -257,29 +267,29 @@ echo {} let exe_path = tempdir.join("usr/bin"); let _ = fs::remove_dir_all(&exe_path); fs::create_dir_all(&exe_path)?; - *workers_exe_path_override().lock()? = Some(exe_path); + *workers_exe_path_override().lock()? = Some(exe_path.clone()); // Set up custom path at /usr/local/bin. let custom_path = tempdir.join("usr/local/bin"); let _ = fs::remove_dir_all(&custom_path); fs::create_dir_all(&custom_path)?; - f(tempdir) + f(tempdir, exe_path) } #[test] #[serial] fn test_given_worker_path() { - with_temp_dir_structure(|tempdir| { + with_temp_dir_structure(|tempdir, exe_path| { let given_workers_path = tempdir.join("usr/local/bin"); // Try with provided workers path that has missing binaries. assert_matches!( determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())), - Err(Error::MissingWorkerBinaries { given_workers_path: Some(p), workers_names: None }) if p == given_workers_path + Err(Error::MissingWorkerBinaries { given_workers_path: Some(p1), current_exe_path: p2, workers_names: None }) if p1 == given_workers_path && p2 == exe_path ); - // Try with provided workers path that has not executable binaries. + // Try with provided workers path that has non-executable binaries. let prepare_worker_path = given_workers_path.join("polkadot-prepare-worker"); write_worker_exe(&prepare_worker_path)?; fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o644))?; @@ -315,11 +325,11 @@ echo {} #[test] #[serial] fn missing_workers_paths_throws_error() { - with_temp_dir_structure(|tempdir| { + with_temp_dir_structure(|tempdir, exe_path| { // Try with both binaries missing. assert_matches!( determine_workers_paths(None, None, Some(NODE_VERSION.into())), - Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path ); // Try with only prep worker (at bin location). @@ -327,7 +337,7 @@ echo {} write_worker_exe(&prepare_worker_path)?; assert_matches!( determine_workers_paths(None, None, Some(NODE_VERSION.into())), - Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path ); // Try with only exec worker (at bin location). @@ -336,7 +346,7 @@ echo {} write_worker_exe(&execute_worker_path)?; assert_matches!( determine_workers_paths(None, None, Some(NODE_VERSION.into())), - Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path ); // Try with only prep worker (at lib location). @@ -345,7 +355,7 @@ echo {} write_worker_exe(&prepare_worker_path)?; assert_matches!( determine_workers_paths(None, None, Some(NODE_VERSION.into())), - Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path ); // Try with only exec worker (at lib location). @@ -354,7 +364,7 @@ echo {} write_worker_exe(execute_worker_path)?; assert_matches!( determine_workers_paths(None, None, Some(NODE_VERSION.into())), - Err(Error::MissingWorkerBinaries { given_workers_path: None, workers_names: None }) + Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path ); Ok(()) @@ -365,7 +375,7 @@ echo {} #[test] #[serial] fn should_find_workers_at_all_locations() { - with_temp_dir_structure(|tempdir| { + with_temp_dir_structure(|tempdir, _| { let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); write_worker_exe(&prepare_worker_bin_path)?; @@ -391,7 +401,7 @@ echo {} #[test] #[serial] fn should_find_workers_with_custom_names_at_all_locations() { - with_temp_dir_structure(|tempdir| { + with_temp_dir_structure(|tempdir, _| { let (prep_worker_name, exec_worker_name) = ("test-prepare", "test-execute"); let prepare_worker_bin_path = tempdir.join("usr/bin").join(prep_worker_name); @@ -421,7 +431,7 @@ echo {} fn workers_version_mismatch_throws_error() { let bad_version = "v9.9.9.9"; - with_temp_dir_structure(|tempdir| { + with_temp_dir_structure(|tempdir, _| { // Workers at bin location return bad version. let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); let execute_worker_bin_path = tempdir.join("usr/bin/polkadot-execute-worker"); @@ -472,7 +482,7 @@ echo {} #[serial] fn should_find_valid_workers() { // Test bin location. - with_temp_dir_structure(|tempdir| { + with_temp_dir_structure(|tempdir, _| { let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); write_worker_exe(&prepare_worker_bin_path)?; @@ -489,7 +499,7 @@ echo {} .unwrap(); // Test lib location. - with_temp_dir_structure(|tempdir| { + with_temp_dir_structure(|tempdir, _| { let prepare_worker_lib_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker"); write_worker_exe(&prepare_worker_lib_path)?; From 7407507b6c55e3ac1810b10b80cb53e81b60d143 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 25 Jul 2023 15:39:01 +0200 Subject: [PATCH 39/48] Update readme (installation instructions) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f64c941b5ad..a7aae888f5fd 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,9 @@ git checkout cargo build --release ``` -Note that compilation is a memory intensive process. We recommend having 4 GiB of physical RAM or swap available (keep in mind that if a build hits swap it tends to be very slow). +**Note:** compilation is a memory intensive process. We recommend having 4 GiB of physical RAM or swap available (keep in mind that if a build hits swap it tends to be very slow). + +**Note:** if you want to move the built `polkadot` binary somewhere (e.g. into $PATH) you will also need to move `polkadot-execute-worker` and `polkadot-prepare-worker`. You can let cargo do this for you with `cargo install --path .`. #### Build from Source with Docker From 5d9c711244b244fc58c8d5b7a2173153bdd5d9e1 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 25 Jul 2023 17:31:04 +0200 Subject: [PATCH 40/48] Allow disabling external workers for local/testing setups + [X] cli flag to enable single-binary mode + [X] Add message to error --- README.md | 2 +- cli/src/cli.rs | 10 +++++++--- cli/src/command.rs | 17 ++++++++++------- node/service/src/lib.rs | 2 +- zombienet_tests/README.md | 2 +- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a7aae888f5fd..f3d1f5e276cd 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ cargo build --release **Note:** compilation is a memory intensive process. We recommend having 4 GiB of physical RAM or swap available (keep in mind that if a build hits swap it tends to be very slow). -**Note:** if you want to move the built `polkadot` binary somewhere (e.g. into $PATH) you will also need to move `polkadot-execute-worker` and `polkadot-prepare-worker`. You can let cargo do this for you with `cargo install --path .`. +**Note:** if you want to move the built `polkadot` binary somewhere (e.g. into $PATH) you will also need to move `polkadot-execute-worker` and `polkadot-prepare-worker`. You can let cargo do all this for you by running `cargo install --path .`. #### Build from Source with Docker diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 1c640873cf2e..bc6ca19f7784 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -154,11 +154,15 @@ pub struct RunCmd { pub overseer_channel_capacity_override: Option, /// Path to the directory where auxiliary worker binaries reside. If not specified, the main - /// binary's directory is searched first, then `/usr/lib/polkadot` is searched. If the path - /// points to an executable rather then directory, that executable is used both as preparation - /// and execution worker (supposed to be used for tests only). + /// binary's directory is searched first, then `/usr/lib/polkadot` is searched. TESTING ONLY: if + /// the path points to an executable rather then directory, that executable is used both as + /// preparation and execution worker. #[arg(long, value_name = "PATH")] pub workers_path: Option, + + /// TESTING ONLY: don't use secure external PVF worker binaries. + #[arg(long, hide = true)] + pub dont_use_external_workers: bool, } #[allow(missing_docs)] diff --git a/cli/src/command.rs b/cli/src/command.rs index a098731511d3..7fddf0b6946f 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -351,13 +351,16 @@ pub fn run() -> Result<()> { } match &cli.subcommand { - None => run_node_inner( - cli, - service::RealOverseerGen, - None, - polkadot_node_metrics::logger_hook(), - false, - ), + None => { + let dont_use_external_workers = cli.run.dont_use_external_workers; + run_node_inner( + cli, + service::RealOverseerGen, + None, + polkadot_node_metrics::logger_hook(), + dont_use_external_workers, + ) + }, Some(Subcommand::BuildSpec(cmd)) => { let runner = cli.create_runner(cmd)?; Ok(runner.sync_run(|config| cmd.run(config.chain_spec, config.network))?) diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index b33c4a7e3171..8fe27cbff499 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -253,7 +253,7 @@ pub enum Error { }, #[cfg(feature = "full-node")] - #[error("Version of worker binary ({worker_version}) is different from node version ({node_version}), worker_path: {worker_path}")] + #[error("Version of worker binary ({worker_version}) is different from node version ({node_version}), worker_path: {worker_path}. TESTING ONLY: this check can be disabled with --dont-use-external-workers")] WorkerBinaryVersionMismatch { worker_version: String, node_version: String, diff --git a/zombienet_tests/README.md b/zombienet_tests/README.md index 5a4c97355f09..84334c3e1cfe 100644 --- a/zombienet_tests/README.md +++ b/zombienet_tests/README.md @@ -18,7 +18,7 @@ To run any test locally use the native provider (`zombienet test -p native ...`) * adder-collator -> polkadot/target/testnet/adder-collator * malus -> polkadot/target/testnet/malus -* polkadot -> polkadot/target/testnet/polkadot +* polkadot -> polkadot/target/testnet/polkadot, polkadot/target/testnet/polkadot-prepare-worker, polkadot/target/testnet/polkadot-execute-worker * polkadot-collator -> cumulus/target/release/polkadot-parachain * undying-collator -> polkadot/target/testnet/undying-collator From 1ce0c3ea6fb6cd859152a29b6f9f1e5eda1b25ea Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 27 Jul 2023 09:48:07 +0200 Subject: [PATCH 41/48] Revert unnecessary Cargo.lock changes --- Cargo.lock | 2120 ++++++++++++++++++++++++++-------------------------- 1 file changed, 1049 insertions(+), 1071 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 962f61d3cdb6..26e896204c24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,15 +21,6 @@ dependencies = [ "gimli", ] -[[package]] -name = "addr2line" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" -dependencies = [ - "gimli", -] - [[package]] name = "adler" version = "1.0.2" @@ -48,7 +39,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", ] [[package]] @@ -58,7 +49,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array 0.14.7", + "generic-array 0.14.6", ] [[package]] @@ -118,28 +109,28 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.8", "once_cell", "version_check", ] [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" dependencies = [ "cfg-if", - "getrandom 0.2.10", + "getrandom 0.2.8", "once_cell", "version_check", ] [[package]] name = "aho-corasick" -version = "1.0.2" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -152,24 +143,9 @@ checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "always-assert" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4436e0292ab1bb631b42973c61205e704475fe8126af845c8d923c0996328127" - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] +checksum = "fbf688625d06217d5b1bb0ea9d9c44a1635fd0ee3534466388d18203174f4d11" [[package]] name = "anes" @@ -203,15 +179,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" dependencies = [ "utf8parse", ] @@ -237,15 +213,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "approx" -version = "0.5.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +checksum = "072df7202e63b127ab55acfe16ce97013d5b97bf160489336d3f1840fd78e99e" dependencies = [ "num-traits", ] @@ -266,9 +242,9 @@ dependencies = [ [[package]] name = "arbitrary" -version = "1.3.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +checksum = "29d47fbf90d5149a107494b15a7dc8d69b351be2db3bb9691740e88ec17fd880" [[package]] name = "array-bytes" @@ -278,9 +254,9 @@ checksum = "d9b1c5a481ec30a5abd8dfbd94ab5cf1bb4e9a66be7f1b3b322f2f1170c200fd" [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" [[package]] name = "arrayvec" @@ -296,14 +272,13 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "assert_cmd" -version = "2.0.12" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" +checksum = "93ae1ddd39efd67689deb1979d80bad3bf7f2b09c6e6117c8d1f2443b5e2f83e" dependencies = [ - "anstyle", "bstr", "doc-comment", - "predicates 3.0.3", + "predicates", "predicates-core", "predicates-tree", "wait-timeout", @@ -317,40 +292,39 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-channel" -version = "1.9.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" dependencies = [ - "concurrent-queue", + "concurrent-queue 2.1.0", "event-listener", "futures-core", ] [[package]] name = "async-io" -version = "1.13.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +checksum = "a811e6a479f2439f0c04038796b5cfb3d2ad56c230e0f2d3f7b04d68cfee607b" dependencies = [ - "async-lock", - "autocfg", - "cfg-if", - "concurrent-queue", + "concurrent-queue 1.2.2", "futures-lite", + "libc", "log", + "once_cell", "parking", "polling", - "rustix 0.37.23", "slab", "socket2 0.4.9", "waker-fn", + "winapi", ] [[package]] name = "async-lock" -version = "2.7.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" +checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b" dependencies = [ "event-listener", ] @@ -363,31 +337,31 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] name = "async-trait" -version = "0.1.72" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] name = "asynchronous-codec" -version = "0.6.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568" +checksum = "06a0daa378f5fd10634e44b0a29b2a87b890657658e072a30d6f26e57ddee182" dependencies = [ "bytes", "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", ] [[package]] @@ -409,24 +383,24 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ - "addr2line 0.20.0", + "addr2line", "cc", "cfg-if", "libc", "miniz_oxide", - "object 0.31.1", + "object", "rustc-demangle", ] [[package]] name = "base-x" -version = "0.2.11" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" [[package]] name = "base16ct" @@ -436,36 +410,27 @@ checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" -version = "0.13.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "basic-toml" -version = "0.1.4" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bfc506e7a2370ec239e1d072507b2a80c833083699d3c6fa176fbb4de8448c6" -dependencies = [ - "serde", -] +checksum = "ea2b2456fd614d856680dcd9fcc660a51a820fa09daef2e49772b56a193c8474" [[package]] name = "beef" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +checksum = "bed554bd50246729a1ec158d08aa3235d1b69d94ad120ebe187e28894787e736" dependencies = [ "serde", ] @@ -494,19 +459,19 @@ version = "0.65.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cexpr", "clang-sys", "lazy_static", "lazycell", "peeking_take_while", - "prettyplease 0.2.12", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -515,12 +480,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitflags" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" - [[package]] name = "bitvec" version = "1.0.1" @@ -550,31 +509,31 @@ checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" dependencies = [ "arrayref", "arrayvec 0.7.4", - "constant_time_eq 0.2.6", + "constant_time_eq 0.2.4", ] [[package]] name = "blake2s_simd" -version = "1.0.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6637f448b9e61dfadbdcbae9a885fadee1f3eaffb1f8d3c1965d3ade8bdfd44f" +checksum = "db539cc2b5f6003621f1cd9ef92d7ded8ea5232c7de0f9faa2de251cd98730d4" dependencies = [ "arrayref", "arrayvec 0.7.4", - "constant_time_eq 0.2.6", + "constant_time_eq 0.1.5", ] [[package]] name = "blake3" -version = "1.4.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" +checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" dependencies = [ "arrayref", "arrayvec 0.7.4", "cc", "cfg-if", - "constant_time_eq 0.3.0", + "constant_time_eq 0.1.5", "digest 0.10.7", ] @@ -596,16 +555,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", ] [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", ] [[package]] @@ -655,13 +614,13 @@ dependencies = [ [[package]] name = "bstr" -version = "1.6.0" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" dependencies = [ + "lazy_static", "memchr", - "regex-automata 0.3.3", - "serde", + "regex-automata", ] [[package]] @@ -675,9 +634,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byte-slice-cast" @@ -693,9 +652,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "bytemuck" -version = "1.13.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "c041d3eab048880cb0b86b256447da3f18859a163c3b8d8893f4e6368abe6393" [[package]] name = "byteorder" @@ -720,20 +679,26 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + [[package]] name = "camino" -version = "1.1.6" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055" dependencies = [ "serde", ] [[package]] name = "cargo-platform" -version = "0.1.3" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfa25e60aea747ec7e1124f238816749faa93759c6ff5b31f1ccdda137f4479" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" dependencies = [ "serde", ] @@ -746,7 +711,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver 1.0.18", + "semver 1.0.16", "serde", "serde_json", "thiserror", @@ -778,9 +743,9 @@ dependencies = [ [[package]] name = "cfg-expr" -version = "0.15.3" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "215c0072ecc28f92eeb0eea38ba63ddfcb65c2828c46311d646f1a3ff5f9841c" +checksum = "c8790cf1286da485c72cf5fc7aeba308438800036ec67d89425924c4807268c9" dependencies = [ "smallvec", ] @@ -824,24 +789,22 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", + "libc", + "num-integer", "num-traits", "time", - "wasm-bindgen", "winapi", ] [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" dependencies = [ "ciborium-io", "ciborium-ll", @@ -850,15 +813,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" dependencies = [ "ciborium-io", "half", @@ -883,7 +846,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", ] [[package]] @@ -907,9 +870,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "fa66045b9cb23c2e9c1520732030608b02ee07e5cfaa5a521ec15ded7fa24c90" dependencies = [ "glob", "libc", @@ -918,15 +881,15 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.25" +version = "3.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", - "bitflags 1.3.2", - "clap_derive 3.2.25", + "bitflags", + "clap_derive 3.2.18", "clap_lex 0.2.4", - "indexmap 1.9.3", + "indexmap", "once_cell", "strsim", "termcolor", @@ -935,32 +898,33 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.19" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" +checksum = "8a1f23fa97e1d1641371b51f35535cb26959b8e27ab50d167a8b996b5bada819" dependencies = [ "clap_builder", - "clap_derive 4.3.12", + "clap_derive 4.2.0", "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.19" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" +checksum = "0fdc5d93c358224b4d6867ef1356d740de2303e9892edc06c5340daeccd96bab" dependencies = [ "anstream", "anstyle", - "clap_lex 0.5.0", + "bitflags", + "clap_lex 0.4.1", "strsim", ] [[package]] name = "clap_derive" -version = "3.2.25" +version = "3.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" dependencies = [ "heck", "proc-macro-error", @@ -971,14 +935,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -992,15 +956,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" [[package]] name = "coarsetime" -version = "0.1.23" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90d114103adbc625300f346d4d09dfb4ab1c4a8df6868435dd903392ecf4354" +checksum = "454038500439e141804c655b4cd1bc6a70bcb95cd2bc9463af5661b6956f0e46" dependencies = [ "libc", "once_cell", @@ -1020,9 +984,9 @@ dependencies = [ [[package]] name = "color-eyre" -version = "0.6.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" +checksum = "8ebf286c900a6d5867aeff75cfee3192857bb7f24b547d4f0df2ed6baa812c90" dependencies = [ "backtrace", "eyre", @@ -1039,9 +1003,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "f9e1f7e5d046697d34b593bdba8ee31f4649366e452a2ccabb3baf3511e503d1" dependencies = [ "strum", "strum_macros", @@ -1056,31 +1020,40 @@ checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" [[package]] name = "concurrent-queue" -version = "2.2.0" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "concurrent-queue" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" dependencies = [ "crossbeam-utils", ] [[package]] name = "console" -version = "0.15.7" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.42.0", ] [[package]] name = "const-oid" -version = "0.9.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "795bc6e66a8e340f075fcf6227e417a2dc976b92b91f3cdc778bb858778b6747" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" [[package]] name = "const-random" @@ -1098,7 +1071,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.8", "once_cell", "proc-macro-hack", "tiny-keccak", @@ -1106,15 +1079,15 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" [[package]] name = "convert_case" @@ -1124,9 +1097,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" dependencies = [ "core-foundation-sys", "libc", @@ -1134,9 +1107,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "core2" @@ -1149,18 +1122,9 @@ dependencies = [ [[package]] name = "cpp_demangle" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "cpp_demangle" -version = "0.4.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee34052ee3d93d6d8f3e6f81d85c47921f6653a19a7b70e939e3e602d893a674" +checksum = "931ab2a3e6330a07900b8e7ca4e106cdcbb93f2b9a52df55e54ee53d8305b55d" dependencies = [ "cfg-if", ] @@ -1177,9 +1141,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" dependencies = [ "libc", ] @@ -1301,7 +1265,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.25", + "clap 3.2.23", "criterion-plot", "itertools", "lazy_static", @@ -1327,9 +1291,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1337,9 +1301,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -1348,22 +1312,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ - "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", + "lazy_static", + "memoffset 0.6.4", "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1371,9 +1335,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if", ] @@ -1390,7 +1354,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4c2f4e1afd912bc40bfd6fed5d9dc1f288e0ba01bfcc835cc5bc3eb13efe15" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", "rand_core 0.6.4", "subtle", "zeroize", @@ -1402,7 +1366,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", "rand_core 0.6.4", "typenum", ] @@ -1413,7 +1377,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", "subtle", ] @@ -1423,10 +1387,20 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", "subtle", ] +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "ctr" version = "0.8.0" @@ -1487,9 +1461,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.102" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f68e12e817cb19eaab81aaec582b4052d07debd3c3c6b083b9d361db47c7dc9d" +checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" dependencies = [ "cc", "cxxbridge-flags", @@ -1499,9 +1473,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.102" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e789217e4ab7cf8cc9ce82253180a9fe331f35f5d339f0ccfe0270b39433f397" +checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" dependencies = [ "cc", "codespan-reporting", @@ -1509,37 +1483,37 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.27", + "syn 1.0.109", ] [[package]] name = "cxxbridge-flags" -version = "1.0.102" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78a19f4c80fd9ab6c882286fa865e92e07688f4387370a209508014ead8751d0" +checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" [[package]] name = "cxxbridge-macro" -version = "1.0.102" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fcfa71f66c8563c4fa9dd2bb68368d50267856f831ac5d85367e0805f9606c" +checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 1.0.109", ] [[package]] name = "dashmap" -version = "5.5.0" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.12.3", "lock_api", "once_cell", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.6", ] [[package]] @@ -1550,9 +1524,9 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "data-encoding-macro" -version = "0.1.13" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c904b33cc60130e1aeea4956ab803d08a3f4a0ca82d64ed757afac3891f2bb99" +checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -1560,9 +1534,9 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.11" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fdf3fce3ce863539ec1d7fd1b6dcc3c645663376b43ed376bbf887733e4f772" +checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" dependencies = [ "data-encoding", "syn 1.0.109", @@ -1624,9 +1598,9 @@ dependencies = [ [[package]] name = "diff" -version = "0.1.13" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" [[package]] name = "difflib" @@ -1649,7 +1623,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", ] [[package]] @@ -1658,7 +1632,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer 0.10.3", "const-oid", "crypto-common", "subtle", @@ -1685,9 +1659,9 @@ dependencies = [ [[package]] name = "dirs-sys" -version = "0.3.7" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" dependencies = [ "libc", "redox_users", @@ -1707,9 +1681,9 @@ dependencies = [ [[package]] name = "dissimilar" -version = "1.0.7" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" +checksum = "31ad93652f40969dead8d4bf897a41e9462095152eb21c56e5830537e41179dd" [[package]] name = "dlmalloc" @@ -1747,7 +1721,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.27", + "syn 2.0.20", "termcolor", "walkdir", ] @@ -1760,9 +1734,9 @@ checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" [[package]] name = "dtoa" -version = "1.0.9" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" +checksum = "5caaa75cbd2b960ff1e5392d2cfb1f44717fffe12fc1f32b7b5d1267f99732a6" [[package]] name = "dyn-clonable" @@ -1787,15 +1761,15 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.12" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" +checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" [[package]] name = "ecdsa" -version = "0.16.8" +version = "0.16.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4b1e0c257a9e9f25f90ff76d7a68360ed497ee519c8e428d1825ef0000799d4" +checksum = "0997c976637b606099b9985693efa3581e84e41f5c11ba5255f88711058ad428" dependencies = [ "der", "digest 0.10.7", @@ -1824,7 +1798,7 @@ dependencies = [ "ed25519", "rand 0.7.3", "serde", - "sha2 0.9.9", + "sha2 0.9.8", "zeroize", ] @@ -1838,7 +1812,7 @@ dependencies = [ "hashbrown 0.12.3", "hex", "rand_core 0.6.4", - "sha2 0.9.9", + "sha2 0.9.8", "zeroize", ] @@ -1858,7 +1832,7 @@ dependencies = [ "crypto-bigint", "digest 0.10.7", "ff", - "generic-array 0.14.7", + "generic-array 0.14.6", "group", "pkcs8", "rand_core 0.6.4", @@ -1875,9 +1849,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" dependencies = [ "cfg-if", ] @@ -1911,28 +1885,28 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] name = "enumn" -version = "0.1.11" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b893c4eb2dc092c811165f84dc7447fae16fb66521717968c34c509b39b1a5c5" +checksum = "48016319042fb7c87b78d2993084a831793a897a5cd1a2a67cab9d1eeb4b7d76" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] name = "env_logger" -version = "0.9.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", - "humantime", + "humantime 1.3.0", "log", "regex", "termcolor", @@ -1940,12 +1914,12 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ - "humantime", - "is-terminal", + "atty", + "humantime 2.1.0", "log", "regex", "termcolor", @@ -1957,17 +1931,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48c92028aaa870e83d51c64e5d4e0b6981b360c522198c23959f219a4e1b15b" -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - [[package]] name = "erased-serde" -version = "0.3.28" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da96524cc884f6558f1769b6c46686af2fe8e8b4cd253bd5a3cdba8181b8e070" +checksum = "ad132dd8d0d0b546348d7d86cb3191aad14b34e5f979781fc005c80d4ac67ffd" dependencies = [ "serde", ] @@ -1982,6 +1950,17 @@ dependencies = [ "polkadot-primitives", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + [[package]] name = "errno" version = "0.3.1" @@ -2005,9 +1984,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "2.5.3" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" [[package]] name = "exit-future" @@ -2058,14 +2037,14 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] name = "eyre" -version = "0.6.8" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +checksum = "221239d1d5ea86bf5d6f91c9d6bc3646ffe471b08ff9b0f91c44f115ac969d2b" dependencies = [ "indenter", "once_cell", @@ -2085,19 +2064,13 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.9.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" dependencies = [ "instant", ] -[[package]] -name = "fastrand" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" - [[package]] name = "fatality" version = "0.0.6" @@ -2115,7 +2088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5aa1e3ae159e592ad222dc90c5acbad632b527779ba88486abe92782ab268bd" dependencies = [ "expander 0.0.4", - "indexmap 1.9.3", + "indexmap", "proc-macro-crate", "proc-macro2", "quote", @@ -2166,24 +2139,24 @@ checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" [[package]] name = "file-per-thread-logger" -version = "0.1.6" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f2e425d9790201ba4af4630191feac6dcc98765b118d4d18e91d23c2353866" +checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" dependencies = [ - "env_logger 0.10.0", + "env_logger 0.7.1", "log", ] [[package]] name = "filetime" -version = "0.2.21" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", - "windows-sys 0.48.0", + "redox_syscall", + "windows-sys 0.36.1", ] [[package]] @@ -2228,9 +2201,9 @@ dependencies = [ [[package]] name = "fixedbitset" -version = "0.4.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" [[package]] name = "float-cmp" @@ -2257,9 +2230,9 @@ dependencies = [ [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ "percent-encoding", ] @@ -2303,7 +2276,7 @@ dependencies = [ "Inflector", "array-bytes", "chrono", - "clap 4.3.19", + "clap 4.2.5", "comfy-table", "frame-benchmarking", "frame-support", @@ -2351,7 +2324,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -2427,7 +2400,7 @@ version = "4.0.0-dev" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ "aquamarine", - "bitflags 1.3.2", + "bitflags", "environmental", "frame-metadata", "frame-support-procedural", @@ -2473,7 +2446,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -2485,7 +2458,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -2495,7 +2468,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844a dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -2595,9 +2568,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "2.9.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" +checksum = "5ebd3504ad6116843b8375ad70df74e7bfe83cac77a1f3fe73200c844d43bfe0" [[package]] name = "fs2" @@ -2611,14 +2584,21 @@ dependencies = [ [[package]] name = "fs4" -version = "0.6.6" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" +checksum = "8ea55201cc351fdb478217c0fb641b59813da9b4efe4c414a9d8f989a657d149" dependencies = [ - "rustix 0.38.4", - "windows-sys 0.48.0", + "libc", + "rustix 0.35.13", + "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -2676,16 +2656,16 @@ checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-lite" -version = "1.13.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" dependencies = [ - "fastrand 1.9.0", + "fastrand", "futures-core", "futures-io", "memchr", "parking", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "waker-fn", ] @@ -2697,7 +2677,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -2707,7 +2687,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd" dependencies = [ "futures-io", - "rustls 0.20.8", + "rustls 0.20.7", "webpki", ] @@ -2742,7 +2722,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "pin-utils", "slab", ] @@ -2781,9 +2761,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", @@ -2813,9 +2793,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", @@ -2844,26 +2824,26 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" dependencies = [ "fallible-iterator", - "indexmap 1.9.3", + "indexmap", "stable_deref_trait", ] [[package]] name = "glob" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "globset" -version = "0.4.11" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1391ab1f92ffcc08911957149833e682aa3fe252b9f45f966d2ef972274c97df" +checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" dependencies = [ "aho-corasick", "bstr", @@ -2885,9 +2865,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.20" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" dependencies = [ "bytes", "fnv", @@ -2895,7 +2875,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap", "slab", "tokio", "tokio-util", @@ -2910,16 +2890,16 @@ checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] name = "handlebars" -version = "4.3.7" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" +checksum = "99d6a30320f094710245150395bc763ad23128d6a1ebbad7594dc4164b62c56b" dependencies = [ "log", "pest", "pest_derive", + "quick-error 2.0.1", "serde", "serde_json", - "thiserror", ] [[package]] @@ -2952,7 +2932,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.2", ] [[package]] @@ -2961,15 +2941,15 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.2", "allocator-api2", ] [[package]] name = "heck" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" [[package]] name = "hermit-abi" @@ -2982,9 +2962,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" [[package]] name = "hex" @@ -3049,7 +3029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" dependencies = [ "digest 0.9.0", - "generic-array 0.14.7", + "generic-array 0.14.6", "hmac 0.8.1", ] @@ -3078,9 +3058,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", @@ -3095,20 +3075,20 @@ checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", ] [[package]] name = "http-range-header" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" [[package]] name = "httparse" -version = "1.8.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" [[package]] name = "httpdate" @@ -3116,6 +3096,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error 1.2.3", +] + [[package]] name = "humantime" version = "2.1.0" @@ -3124,9 +3113,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ "bytes", "futures-channel", @@ -3138,7 +3127,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "socket2 0.4.9", "tokio", "tower-service", @@ -3148,59 +3137,35 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" dependencies = [ "http", "hyper", "log", - "rustls 0.20.8", + "rustls 0.20.7", "rustls-native-certs", "tokio", - "tokio-rustls 0.23.4", - "webpki-roots 0.22.6", + "tokio-rustls 0.23.2", + "webpki-roots 0.22.2", ] [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ - "futures-util", "http", "hyper", "log", - "rustls 0.21.5", + "rustls 0.21.2", "rustls-native-certs", "tokio", "tokio-rustls 0.24.1", ] -[[package]] -name = "iana-time-zone" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows 0.48.0", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "idna" version = "0.2.3" @@ -3214,9 +3179,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -3248,7 +3213,7 @@ dependencies = [ "rtnetlink", "system-configuration", "tokio", - "windows 0.34.0", + "windows", ] [[package]] @@ -3307,33 +3272,22 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexmap" -version = "1.9.3" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", "hashbrown 0.12.3", "serde", ] -[[package]] -name = "indexmap" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" -dependencies = [ - "equivalent", - "hashbrown 0.14.0", -] - [[package]] name = "indicatif" -version = "0.17.5" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" dependencies = [ "console", - "instant", "number_prefix", "portable-atomic", "unicode-width", @@ -3345,7 +3299,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", ] [[package]] @@ -3359,9 +3313,9 @@ dependencies = [ [[package]] name = "integer-encoding" -version = "3.0.4" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +checksum = "90c11140ffea82edce8dcd74137ce9324ec24b3cf0175fc9d7e29164da9915b8" [[package]] name = "integer-sqrt" @@ -3374,11 +3328,17 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.11" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" + +[[package]] +name = "io-lifetimes" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.3.1", "libc", "windows-sys 0.48.0", ] @@ -3391,30 +3351,31 @@ checksum = "aa2f047c0a98b2f299aa5d6d7088443570faae494e9ae1305e48be000c9e0eb1" [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" dependencies = [ - "socket2 0.5.3", + "socket2 0.4.9", "widestring", - "windows-sys 0.48.0", - "winreg 0.50.0", + "winapi", + "winreg 0.7.0", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ - "hermit-abi 0.3.2", - "rustix 0.38.4", + "hermit-abi 0.3.1", + "io-lifetimes 1.0.10", + "rustix 0.37.18", "windows-sys 0.48.0", ] @@ -3429,18 +3390,18 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.5" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jobserver" @@ -3496,10 +3457,10 @@ dependencies = [ "soketto", "thiserror", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls 0.23.2", "tokio-util", "tracing", - "webpki-roots 0.22.6", + "webpki-roots 0.22.2", ] [[package]] @@ -3538,7 +3499,7 @@ checksum = "cc345b0a43c6bc49b947ebeb936e886a419ee3d894421790c969cc56040542ad" dependencies = [ "async-trait", "hyper", - "hyper-rustls 0.23.2", + "hyper-rustls 0.23.0", "jsonrpsee-core", "jsonrpsee-types", "rustc-hash", @@ -3625,12 +3586,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" -dependencies = [ - "cpufeatures", -] +checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" [[package]] name = "kusama-runtime" @@ -3825,9 +3783,9 @@ checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libflate" -version = "1.4.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" +checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" dependencies = [ "adler32", "crc32fast", @@ -3845,9 +3803,9 @@ dependencies = [ [[package]] name = "libloading" -version = "0.7.4" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "afe203d669ec979b7128619bae5a63b7b42e9203c1b29146079ee05e2f604b52" dependencies = [ "cfg-if", "winapi", @@ -3868,7 +3826,7 @@ dependencies = [ "bytes", "futures", "futures-timer", - "getrandom 0.2.10", + "getrandom 0.2.8", "instant", "libp2p-allow-block-list", "libp2p-connection-limits", @@ -3972,7 +3930,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "lru 0.10.1", + "lru 0.10.0", "quick-protobuf", "quick-protobuf-codec", "smallvec", @@ -4157,7 +4115,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -4241,12 +4199,12 @@ dependencies = [ [[package]] name = "libsecp256k1" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b09eff1b35ed3b33b877ced3a691fc7a481919c7e29c53c906226fcf55e2a1" +checksum = "b0452aac8bab02242429380e9b2f94ea20cea2b37e2c1777a1358799bbe97f37" dependencies = [ "arrayref", - "base64 0.13.1", + "base64 0.13.0", "digest 0.9.0", "hmac-drbg", "libsecp256k1-core", @@ -4254,7 +4212,7 @@ dependencies = [ "libsecp256k1-gen-genmult", "rand 0.8.5", "serde", - "sha2 0.9.9", + "sha2 0.9.8", "typenum", ] @@ -4289,9 +4247,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.10" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e6ab01971eb092ffe6a7d42f49f9ff42662f17604681e2843ad65077ba47dc" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" dependencies = [ "cc", "pkg-config", @@ -4300,18 +4258,18 @@ dependencies = [ [[package]] name = "link-cplusplus" -version = "1.0.9" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" dependencies = [ "cc", ] [[package]] name = "linked-hash-map" -version = "0.5.6" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "linked_hash_set" @@ -4324,36 +4282,36 @@ dependencies = [ [[package]] name = "linregress" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de0b5f52a9f84544d268f5fabb71b38962d6aa3c6600b8bcd27d44ccf9c9c45" +checksum = "475015a7f8f017edb28d2e69813be23500ad4b32cfe3421c4148efc97324ee52" dependencies = [ "nalgebra", ] [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.4.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" +checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -4371,9 +4329,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.10.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718e8fae447df0c7e1ba7f5189829e63fd536945c8988d61444c19039f16b670" +checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e" dependencies = [ "hashbrown 0.13.2", ] @@ -4434,7 +4392,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -4448,7 +4406,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -4459,7 +4417,7 @@ checksum = "c12469fc165526520dff2807c2975310ab47cf7190a45b99b49a7dc8befab17b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -4470,7 +4428,7 @@ checksum = "b8fb85ec1620619edf2984a7693497d4ec88a9665d8b87e942856884c92dbf2a" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -4491,7 +4449,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -4500,38 +4458,37 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] name = "matches" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "matrixmultiply" -version = "0.3.7" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090126dc04f95dc0d1c1c91f61bdd474b3930ca064c1edc8a849da2c6cbe1e77" +checksum = "add85d4dd35074e6fedc608f8c8f513a3548619a9024b751949ef0e8e45a4d84" dependencies = [ - "autocfg", "rawpointer", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memfd" -version = "0.6.3" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc89ccdc6e10d6907450f753537ebc5c5d3460d2e4e62ea74bd571db62c0f9e" +checksum = "b20a59d985586e4a5aef64564ac77299f8586d8be6cf9106a5a40207e8908efb" dependencies = [ - "rustix 0.37.23", + "rustix 0.36.7", ] [[package]] @@ -4545,27 +4502,27 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.7.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ "autocfg", ] [[package]] name = "memoffset" -version = "0.8.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" dependencies = [ "autocfg", ] [[package]] name = "memoffset" -version = "0.9.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -4604,9 +4561,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.17" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "minimal-lexical" @@ -4616,9 +4573,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -4671,24 +4628,24 @@ dependencies = [ [[package]] name = "mockall" -version = "0.11.4" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326" dependencies = [ "cfg-if", "downcast", "fragile", "lazy_static", "mockall_derive", - "predicates 2.1.5", + "predicates", "predicates-tree", ] [[package]] name = "mockall_derive" -version = "0.11.4" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" dependencies = [ "cfg-if", "proc-macro2", @@ -4766,7 +4723,7 @@ dependencies = [ "digest 0.10.7", "multihash-derive 0.9.0", "ripemd", - "sha-1 0.10.1", + "sha-1 0.10.0", "sha2 0.10.7", "sha3", "strobe-rs", @@ -4833,9 +4790,9 @@ dependencies = [ [[package]] name = "nalgebra" -version = "0.32.3" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307ed9b18cc2423f29e83f84fd23a8e73628727990181f18641a8b5dc2ab1caa" +checksum = "f6515c882ebfddccaa73ead7320ca28036c4bc84c9bcca3cc0cbba8efe89223a" dependencies = [ "approx", "matrixmultiply", @@ -4849,9 +4806,9 @@ dependencies = [ [[package]] name = "nalgebra-macros" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91761aed67d03ad966ef783ae962ef9bbaca728d2dd7ceb7939ec110fffad998" +checksum = "d232c68884c0c99810a5a4d333ef7e47689cfd0edc85efc9e54e1e6bf5212766" dependencies = [ "proc-macro2", "quote", @@ -4864,16 +4821,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d66043b25d4a6cccb23619d10c19c25304b355a7dccd4a8e11423dd2382146" dependencies = [ - "rand 0.8.5", -] - -[[package]] -name = "names" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bddcd3bf5144b6392de80e04c347cd7fab2508f6df16a85fc496ecd5cec39bc" -dependencies = [ - "clap 3.2.25", + "clap 3.2.23", "rand 0.8.5", ] @@ -4902,7 +4850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab" dependencies = [ "anyhow", - "bitflags 1.3.2", + "bitflags", "byteorder", "libc", "netlink-packet-core", @@ -4911,9 +4859,9 @@ dependencies = [ [[package]] name = "netlink-packet-utils" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" +checksum = "25af9cf0dc55498b7bd94a1508af7a78706aa0ab715a73c5169273e03c84845e" dependencies = [ "anyhow", "byteorder", @@ -4938,9 +4886,9 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" +checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027" dependencies = [ "bytes", "futures", @@ -4951,11 +4899,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.3" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg-if", "libc", ] @@ -4966,7 +4914,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg-if", "libc", "memoffset 0.7.1", @@ -4982,12 +4930,13 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "nom" -version = "7.1.3" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" dependencies = [ "memchr", "minimal-lexical", + "version_check", ] [[package]] @@ -4996,16 +4945,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num-bigint" version = "0.4.3" @@ -5019,18 +4958,18 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" +checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085" dependencies = [ "num-traits", ] [[package]] name = "num-format" -version = "0.4.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +checksum = "54b862ff8df690cf089058c98b183676a7ed0f974cc08b426800093227cbff3b" dependencies = [ "arrayvec 0.7.4", "itoa", @@ -5038,9 +4977,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg", "num-traits", @@ -5060,20 +4999,20 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.1.19", "libc", ] @@ -5085,22 +5024,13 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.30.4" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" dependencies = [ "crc32fast", "hashbrown 0.13.2", - "indexmap 1.9.3", - "memchr", -] - -[[package]] -name = "object" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" -dependencies = [ + "indexmap", "memchr", ] @@ -5130,9 +5060,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "orchestra" @@ -5177,21 +5107,24 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.5.1" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" [[package]] -name = "overload" -version = "0.1.1" +name = "output_vt100" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" +dependencies = [ + "winapi", +] [[package]] name = "owo-colors" -version = "3.5.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +checksum = "20448fd678ec04e6ea15bbe0476874af65e98a01515d667aa49f1434dc44ebf4" [[package]] name = "packed_simd_2" @@ -5970,7 +5903,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -6233,9 +6166,9 @@ dependencies = [ [[package]] name = "parity-db" -version = "0.4.10" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f19d20a0d2cc52327a88d131fa1c4ea81ea4a04714aedcfeca2dd410049cf8" +checksum = "4890dcb9556136a4ec2b0c51fa4a08c8b733b829506af8fff2e853f3a065985b" dependencies = [ "blake2", "crc32fast", @@ -6253,9 +6186,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.6.4" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8e946cc0cc711189c0b0249fb8b599cbeeab9784d83c415719368bb8d4ac64" +checksum = "2287753623c76f953acd29d15d8100bcab84d29db78fb6f352adb3c53e83b967" dependencies = [ "arrayvec 0.7.4", "bitvec", @@ -6268,9 +6201,9 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.6.4" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a296c3079b5fefbc499e1de58dc26c09b1b9a5952d26694ee89f04a43ebbb3e" +checksum = "2b6937b5e67bfba3351b87b040d48352a2fcb6ad72f81855412ce97b45c8f110" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -6286,9 +6219,9 @@ checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" [[package]] name = "parking" -version = "2.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" [[package]] name = "parking_lot" @@ -6298,7 +6231,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core 0.8.6", + "parking_lot_core 0.8.5", ] [[package]] @@ -6308,34 +6241,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.6", ] [[package]] name = "parking_lot_core" -version = "0.8.6" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ "cfg-if", "instant", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", "winapi", ] [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", - "windows-targets 0.48.1", + "windows-sys 0.42.0", ] [[package]] @@ -6346,9 +6279,9 @@ checksum = "7924d1d0ad836f665c9065e26d016c673ece3993f30d340068b16f282afc1156" [[package]] name = "paste" -version = "1.0.14" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" [[package]] name = "pbkdf2" @@ -6376,25 +6309,24 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.7.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d2d1d55045829d65aad9d389139882ad623b33b904e7c9f1b10c5b8927298e5" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" dependencies = [ - "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f94bca7e7a599d89dea5dfa309e217e7906c3c007fb9c3299c40b10d6a315d3" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" dependencies = [ "pest", "pest_generator", @@ -6402,36 +6334,36 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d490fe7e8556575ff6911e45567ab95e71617f43781e5c05490dc8d75c965c" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.27", + "syn 1.0.109", ] [[package]] name = "pest_meta" -version = "2.7.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674c66ebb4b4d9036012091b537aae5878970d6999f81a265034d85b136b341" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" dependencies = [ - "once_cell", + "maplit", "pest", - "sha2 0.10.7", + "sha-1 0.8.2", ] [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap", ] [[package]] @@ -6451,7 +6383,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -6462,9 +6394,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -6484,9 +6416,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "12295df4f294471248581bc09bef3c38a5e46f1e36d6a37353621a0c6c357e1f" [[package]] name = "platforms" @@ -6521,7 +6453,7 @@ name = "polkadot-approval-distribution" version = "0.9.43" dependencies = [ "assert_matches", - "env_logger 0.9.3", + "env_logger 0.9.0", "futures", "futures-timer", "log", @@ -6549,7 +6481,7 @@ version = "0.9.43" dependencies = [ "assert_matches", "bitvec", - "env_logger 0.9.3", + "env_logger 0.9.0", "futures", "futures-timer", "log", @@ -6603,7 +6535,7 @@ name = "polkadot-availability-recovery" version = "0.9.43" dependencies = [ "assert_matches", - "env_logger 0.9.3", + "env_logger 0.9.0", "fatality", "futures", "futures-timer", @@ -6631,7 +6563,7 @@ dependencies = [ name = "polkadot-cli" version = "0.9.43" dependencies = [ - "clap 4.3.19", + "clap 4.2.5", "frame-benchmarking-cli", "futures", "log", @@ -6664,7 +6596,7 @@ dependencies = [ "always-assert", "assert_matches", "bitvec", - "env_logger 0.9.3", + "env_logger 0.9.0", "fatality", "futures", "futures-timer", @@ -6708,7 +6640,7 @@ dependencies = [ "fatality", "futures", "futures-timer", - "indexmap 1.9.3", + "indexmap", "lazy_static", "lru 0.11.0", "parity-scale-codec", @@ -6865,7 +6797,7 @@ version = "0.9.43" dependencies = [ "assert_matches", "bitvec", - "env_logger 0.9.3", + "env_logger 0.9.0", "futures", "futures-timer", "kvdb", @@ -7348,7 +7280,7 @@ dependencies = [ "assert_matches", "async-trait", "derive_more", - "env_logger 0.9.3", + "env_logger 0.9.0", "fatality", "futures", "futures-channel", @@ -7428,7 +7360,7 @@ dependencies = [ name = "polkadot-performance-test" version = "0.9.43" dependencies = [ - "env_logger 0.9.3", + "env_logger 0.9.0", "kusama-runtime", "log", "polkadot-erasure-coding", @@ -7693,7 +7625,7 @@ name = "polkadot-runtime-parachains" version = "0.9.43" dependencies = [ "assert_matches", - "bitflags 1.3.2", + "bitflags", "bitvec", "derive_more", "frame-benchmarking", @@ -7748,7 +7680,7 @@ version = "0.9.43" dependencies = [ "assert_matches", "async-trait", - "env_logger 0.9.3", + "env_logger 0.9.0", "frame-benchmarking", "frame-benchmarking-cli", "frame-support", @@ -7879,7 +7811,7 @@ dependencies = [ "fatality", "futures", "futures-timer", - "indexmap 1.9.3", + "indexmap", "parity-scale-codec", "polkadot-node-network-protocol", "polkadot-node-primitives", @@ -7945,7 +7877,7 @@ version = "0.9.43" dependencies = [ "assert_matches", "async-trait", - "clap 4.3.19", + "clap 4.2.5", "color-eyre", "futures", "futures-timer", @@ -8087,7 +8019,7 @@ dependencies = [ name = "polkadot-voter-bags" version = "0.9.43" dependencies = [ - "clap 4.3.19", + "clap 4.2.5", "generate-bags", "kusama-runtime", "polkadot-runtime", @@ -8097,18 +8029,15 @@ dependencies = [ [[package]] name = "polling" -version = "2.8.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +checksum = "685404d509889fade3e86fe3a5803bca2ec09b0c0778d5ada6ec8bf7a8de5259" dependencies = [ - "autocfg", - "bitflags 1.3.2", "cfg-if", - "concurrent-queue", "libc", "log", - "pin-project-lite 0.2.10", - "windows-sys 0.48.0", + "wepoll-ffi", + "winapi", ] [[package]] @@ -8148,22 +8077,22 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.4.1" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc55135a600d700580e406b4de0d59cb9ad25e344a3a091a97ded2622ec4ec6" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" [[package]] name = "pprof" -version = "0.11.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" +checksum = "d6472bfed9475542ac46c518734a8d06d71b0f6cb2c17f904aa301711a57786f" dependencies = [ "backtrace", "cfg-if", "findshlibs", "libc", "log", - "nix 0.26.2", + "nix 0.24.1", "once_cell", "parking_lot 0.12.1", "smallvec", @@ -8180,9 +8109,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "predicates" -version = "2.1.5" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +checksum = "95e5a7689e456ab905c22c2b48225bb921aba7c8dfa58440d68ba13f6222a715" dependencies = [ "difflib", "float-cmp", @@ -8192,29 +8121,17 @@ dependencies = [ "regex", ] -[[package]] -name = "predicates" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" -dependencies = [ - "anstyle", - "difflib", - "itertools", - "predicates-core", -] - [[package]] name = "predicates-core" -version = "1.0.6" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" +checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" [[package]] name = "predicates-tree" -version = "1.0.9" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +checksum = "338c7be2905b732ae3984a2f40032b5e94fd8f52505b186c7d4d68d193445df7" dependencies = [ "predicates-core", "termtree", @@ -8222,32 +8139,24 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" dependencies = [ + "ctor", "diff", + "output_vt100", "yansi", ] [[package]] name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "prettyplease" -version = "0.2.12" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ "proc-macro2", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -8327,29 +8236,29 @@ checksum = "70550716265d1ec349c41f70dd4f964b4fd88394efe4405f0c1da679c4799a07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ "unicode-ident", ] [[package]] name = "prometheus" -version = "0.13.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" dependencies = [ "cfg-if", "fnv", "lazy_static", "memchr", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "thiserror", ] @@ -8378,31 +8287,40 @@ dependencies = [ [[package]] name = "prometheus-parse" -version = "0.2.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2aa5feb83bf4b2c8919eaf563f51dbab41183de73ba2353c0e03cd7b6bd892" +checksum = "c996f3caea1c51aa034c0d2dfd8447a12c555f4567b02677ef8a865ac4cce712" dependencies = [ "chrono", - "itertools", - "once_cell", + "lazy_static", "regex", ] [[package]] name = "prost" -version = "0.11.9" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" +dependencies = [ + "bytes", + "prost-derive 0.10.1", +] + +[[package]] +name = "prost" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.11.0", ] [[package]] name = "prost-build" -version = "0.11.9" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" dependencies = [ "bytes", "heck", @@ -8411,20 +8329,31 @@ dependencies = [ "log", "multimap", "petgraph", - "prettyplease 0.1.25", - "prost", + "prost 0.11.0", "prost-types", "regex", - "syn 1.0.109", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-derive" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" dependencies = [ "anyhow", "itertools", @@ -8435,34 +8364,35 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.9" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e" dependencies = [ - "prost", + "bytes", + "prost 0.11.0", ] [[package]] name = "psm" -version = "0.1.21" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +checksum = "cd136ff4382c4753fc061cb9e4712ab2af263376b95bbd5bd8cd50c020b78e69" dependencies = [ "cc", ] [[package]] name = "pyroscope" -version = "0.5.6" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ce54d81c50f7fd6442ee671597f661a068ccebd82ed1557775b6791b14aba7" +checksum = "6636d352280fb587c8716f10e1d61fe88cb002660e0a8b0d3e47de17f3b5aaed" dependencies = [ "json", "libc", "libflate", "log", - "names 0.14.0", - "prost", + "names", + "prost 0.10.4", "reqwest", "thiserror", "url", @@ -8471,9 +8401,9 @@ dependencies = [ [[package]] name = "pyroscope_pprofrs" -version = "0.2.6" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57add45daa57783490913a5d3d88e3249126971b61ac97ee0c7bac293ef0114a" +checksum = "14e699bf3e7da41b3a7573d5944d77b1bd96a187aa72f5fa96afb4ed5609cc45" dependencies = [ "log", "pprof", @@ -8487,6 +8417,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-protobuf" version = "0.8.1" @@ -8522,9 +8458,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.32" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -8594,7 +8530,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.8", ] [[package]] @@ -8623,23 +8559,26 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.7.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ + "autocfg", + "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", + "lazy_static", "num_cpus", ] @@ -8649,27 +8588,17 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", - "thiserror", + "getrandom 0.2.8", + "redox_syscall", ] [[package]] @@ -8687,22 +8616,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.19" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ef7e18e8841942ddb1cf845054f8008410030a3997875d9e49b7a363063df1" +checksum = "300f2a835d808734ee295d45007adacb9ebb29dd3ae2424acfa17930cae541da" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.19" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfaf0c85b766276c797f3791f5bc6d5bd116b41d53049af2789666b0c0bc9fa" +checksum = "4c38e3aecd2b21cb3959637b883bb3714bc7e43f0268b9a29d3743ee3e55cdd2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 1.0.109", ] [[package]] @@ -8719,14 +8648,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.3", - "regex-syntax 0.7.4", + "regex-syntax", ] [[package]] @@ -8735,37 +8663,20 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax 0.6.29", -] - -[[package]] -name = "regex-automata" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax 0.7.4", + "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.7.4" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remote-ext-tests-bags-list" version = "0.9.43" dependencies = [ - "clap 4.3.19", + "clap 4.2.5", "frame-system", "kusama-runtime", "kusama-runtime-constants", @@ -8780,13 +8691,22 @@ dependencies = [ "westend-runtime-constants", ] +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" dependencies = [ - "base64 0.21.2", + "base64 0.21.0", "bytes", "encoding_rs", "futures-core", @@ -8795,27 +8715,27 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls 0.23.0", "ipnet", "js-sys", "log", "mime", "once_cell", "percent-encoding", - "pin-project-lite 0.2.10", - "rustls 0.21.5", - "rustls-pemfile", + "pin-project-lite 0.2.9", + "rustls 0.20.7", + "rustls-pemfile 1.0.2", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls 0.23.2", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.22.2", "winreg 0.10.1", ] @@ -8826,7 +8746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" dependencies = [ "hostname", - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -8988,12 +8908,11 @@ dependencies = [ [[package]] name = "rpassword" -version = "7.2.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" +checksum = "26b763cb66df1c928432cc35053f8bd4cec3335d8559fc16010017d16b3c1680" dependencies = [ "libc", - "rtoolbox", "winapi", ] @@ -9007,26 +8926,16 @@ dependencies = [ "log", "netlink-packet-route", "netlink-proto", - "nix 0.24.3", + "nix 0.24.1", "thiserror", "tokio", ] -[[package]] -name = "rtoolbox" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] name = "rustc-hash" @@ -9046,55 +8955,56 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.18", + "semver 1.0.16", ] [[package]] name = "rustix" -version = "0.36.15" +version = "0.35.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941" +checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", + "bitflags", + "errno 0.2.8", + "io-lifetimes 0.7.5", "libc", - "linux-raw-sys 0.1.4", - "windows-sys 0.45.0", + "linux-raw-sys 0.0.46", + "windows-sys 0.42.0", ] [[package]] name = "rustix" -version = "0.37.23" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", + "bitflags", + "errno 0.2.8", + "io-lifetimes 1.0.10", "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", + "linux-raw-sys 0.1.4", + "windows-sys 0.42.0", ] [[package]] name = "rustix" -version = "0.38.4" +version = "0.37.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" dependencies = [ - "bitflags 2.3.3", - "errno", + "bitflags", + "errno 0.3.1", + "io-lifetimes 1.0.10", "libc", - "linux-raw-sys 0.4.3", + "linux-raw-sys 0.3.6", "windows-sys 0.48.0", ] [[package]] name = "rustls" -version = "0.20.8" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" dependencies = [ "log", "ring", @@ -9104,52 +9014,51 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.5" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" dependencies = [ "log", "ring", - "rustls-webpki 0.101.1", + "rustls-webpki", "sct", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 0.2.1", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" dependencies = [ - "base64 0.21.2", + "base64 0.13.0", ] [[package]] -name = "rustls-webpki" -version = "0.100.1" +name = "rustls-pemfile" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "ring", - "untrusted", + "base64 0.21.0", ] [[package]] name = "rustls-webpki" -version = "0.101.1" +version = "0.100.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" dependencies = [ "ring", "untrusted", @@ -9157,9 +9066,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "rw-stream-sink" @@ -9174,15 +9083,15 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "3c9613b5a66ab9ba26415184cfc41156594925a9cf3a2057e57f31ff145f6568" [[package]] name = "safe_arch" -version = "0.7.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f398075ce1e6a179b46f51bd88d0598b92b00d3551f1a2d4ac49e771b56ac354" +checksum = "794821e4ccb0d9f979512f9c1973480123f9bd62a90d74ab0f9426fcf8f4a529" dependencies = [ "bytemuck", ] @@ -9220,7 +9129,7 @@ dependencies = [ "log", "multihash-codetable", "parity-scale-codec", - "prost", + "prost 0.11.0", "prost-build", "rand 0.8.5", "sc-client-api", @@ -9300,7 +9209,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -9310,12 +9219,12 @@ source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844a dependencies = [ "array-bytes", "chrono", - "clap 4.3.19", + "clap 4.2.5", "fdlimit", "futures", "libp2p-identity", "log", - "names 0.13.0", + "names", "parity-scale-codec", "rand 0.8.5", "regex", @@ -9548,7 +9457,7 @@ name = "sc-consensus-grandpa" version = "0.10.0-dev" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.2", "array-bytes", "async-trait", "dyn-clone", @@ -9670,7 +9579,7 @@ dependencies = [ "cfg-if", "libc", "log", - "rustix 0.36.15", + "rustix 0.36.7", "sc-allocator", "sc-executor-common", "sp-runtime-interface", @@ -9761,7 +9670,7 @@ dependencies = [ "futures", "libp2p-identity", "log", - "prost", + "prost 0.11.0", "prost-build", "sc-client-api", "sc-network", @@ -9777,7 +9686,7 @@ version = "0.10.0-dev" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ "async-trait", - "bitflags 1.3.2", + "bitflags", "futures", "libp2p-identity", "parity-scale-codec", @@ -9793,7 +9702,7 @@ name = "sc-network-gossip" version = "0.10.0-dev" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.2", "futures", "futures-timer", "libp2p-identity", @@ -9818,7 +9727,7 @@ dependencies = [ "libp2p-identity", "log", "parity-scale-codec", - "prost", + "prost 0.11.0", "prost-build", "sc-client-api", "sc-network", @@ -9843,7 +9752,7 @@ dependencies = [ "log", "mockall", "parity-scale-codec", - "prost", + "prost 0.11.0", "prost-build", "sc-client-api", "sc-consensus", @@ -9891,7 +9800,7 @@ dependencies = [ "futures", "futures-timer", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls 0.24.0", "libp2p", "log", "num_cpus", @@ -10094,7 +10003,7 @@ name = "sc-storage-monitor" version = "0.1.0" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ - "clap 4.3.19", + "clap 4.2.5", "fs4", "log", "sc-client-db", @@ -10197,7 +10106,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -10259,9 +10168,9 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.9.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0a159d0c45c12b20c5a844feb1fe4bea86e28f17b92a5f0c42193634d3782" +checksum = "0cfdffd972d76b22f3d7f81c8be34b2296afd3a25e0a547bd9abe340a4dbbe97" dependencies = [ "bitvec", "cfg-if", @@ -10273,9 +10182,9 @@ dependencies = [ [[package]] name = "scale-info-derive" -version = "2.9.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912e55f6d20e0e80d63733872b40e1227c0bce1e1ab81ba67d696339bfd7fd29" +checksum = "61fa974aea2d63dd18a4ec3a49d59af9f34178c73a4f56d2f18205628d00681e" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -10285,11 +10194,12 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" dependencies = [ - "windows-sys 0.48.0", + "lazy_static", + "winapi", ] [[package]] @@ -10298,7 +10208,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "772575a524feeb803e5b0fcbc6dd9f367e579488197c94c6e4023aad2305774d" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.2", "cfg-if", "hashbrown 0.13.2", ] @@ -10323,15 +10233,15 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.2.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.7" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] name = "sct" @@ -10345,13 +10255,13 @@ dependencies = [ [[package]] name = "sec1" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +checksum = "48518a2b5775ba8ca5b46596aae011caa431e6ce7e4a67ead66d92f08884220e" dependencies = [ "base16ct", "der", - "generic-array 0.14.7", + "generic-array 0.14.6", "pkcs8", "subtle", "zeroize", @@ -10359,18 +10269,18 @@ dependencies = [ [[package]] name = "secp256k1" -version = "0.24.3" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" +checksum = "d9512ffd81e3a3503ed401f79c33168b9148c75038956039166cd750eaa037c3" dependencies = [ "secp256k1-sys", ] [[package]] name = "secp256k1-sys" -version = "0.6.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" +checksum = "7058dc8eaf3f2810d7828680320acda0b25a288f6d288e19278e249bbf74226b" dependencies = [ "cc", ] @@ -10386,11 +10296,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.2" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" dependencies = [ - "bitflags 1.3.2", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -10399,9 +10309,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" dependencies = [ "core-foundation-sys", "libc", @@ -10418,9 +10328,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" dependencies = [ "serde", ] @@ -10445,38 +10355,38 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.175" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.175" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] name = "serde_fmt" -version = "1.0.3" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d4ddca14104cd60529e8c7f7ba71a2c8acd8f7f5cfcdc2faf97eeb7c3010a4" +checksum = "2963a69a2b3918c1dc75a45a18bd3fcd1120e31d3f59deb1b2f9b5d5ffb8baa4" dependencies = [ "serde", ] [[package]] name = "serde_json" -version = "1.0.103" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -10485,9 +10395,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" dependencies = [ "serde", ] @@ -10526,7 +10436,19 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", ] [[package]] @@ -10544,9 +10466,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.10.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ "cfg-if", "cpufeatures", @@ -10567,9 +10489,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.9" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" dependencies = [ "block-buffer 0.9.0", "cfg-if", @@ -10591,9 +10513,9 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.8" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +checksum = "31f935e31cf406e8c0e96c2815a5516181b7004ae8c5f296293221e9b1e356bd" dependencies = [ "digest 0.10.7", "keccak", @@ -10616,9 +10538,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook" -version = "0.3.17" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" dependencies = [ "libc", "signal-hook-registry", @@ -10626,9 +10548,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] @@ -10663,9 +10585,9 @@ dependencies = [ [[package]] name = "simba" -version = "0.8.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061507c94fc6ab4ba1c9a0305018408e312e17c041eb63bef8aa726fa33aceae" +checksum = "50582927ed6f77e4ac020c057f37a268fc6aebc29225050365aacbb9deeeddc4" dependencies = [ "approx", "num-complex", @@ -10682,18 +10604,15 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" [[package]] name = "slice-group-by" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" +checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" [[package]] name = "slot-range-helper" @@ -10770,7 +10689,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ - "base64 0.13.1", + "base64 0.13.0", "bytes", "futures", "http", @@ -10812,7 +10731,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -10990,7 +10909,7 @@ version = "21.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ "array-bytes", - "bitflags 1.3.2", + "bitflags", "blake2", "bounded-collections", "bs58 0.4.0", @@ -11049,7 +10968,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844a dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -11068,7 +10987,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844a dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -11161,7 +11080,7 @@ version = "4.1.0-dev" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ "thiserror", - "zstd 0.12.4", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] @@ -11286,7 +11205,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -11435,7 +11354,7 @@ name = "sp-trie" version = "22.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.2", "hash-db", "hashbrown 0.13.2", "lazy_static", @@ -11478,7 +11397,7 @@ dependencies = [ "parity-scale-codec", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -11538,9 +11457,9 @@ dependencies = [ [[package]] name = "ss58-registry" -version = "1.41.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfc443bad666016e012538782d9e3006213a7db43e9fb1dda91657dc06a6fa08" +checksum = "23d92659e7d18d82b803824a9ba5a6022cff101c3491d027c1c1d8d30e749284" dependencies = [ "Inflector", "num-format", @@ -11562,7 +11481,7 @@ name = "staking-miner" version = "0.9.43" dependencies = [ "assert_cmd", - "clap 4.3.19", + "clap 4.2.5", "exitcode", "frame-election-provider-support", "frame-remote-externalities", @@ -11594,7 +11513,7 @@ dependencies = [ "sub-tokens", "thiserror", "tokio", - "tracing-subscriber 0.3.17", + "tracing-subscriber 0.3.11", "westend-runtime", ] @@ -11622,11 +11541,11 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a2a1c578e98c1c16fc3b8ec1328f7659a500737d7a0c6d625e73e830ff9c1f6" dependencies = [ - "bitflags 1.3.2", + "bitflags", "cfg_aliases", "libc", "parking_lot 0.11.2", - "parking_lot_core 0.8.6", + "parking_lot_core 0.8.5", "static_init_macro 1.0.2", "winapi", ] @@ -11663,7 +11582,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabb238a1cccccfa4c4fb703670c0d157e1256c1ba695abf1b93bd2bb14bab2d" dependencies = [ - "bitflags 1.3.2", + "bitflags", "byteorder", "keccak", "subtle", @@ -11687,9 +11606,9 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ "heck", "proc-macro2", @@ -11715,7 +11634,7 @@ dependencies = [ "hmac 0.11.0", "pbkdf2 0.8.0", "schnorrkel", - "sha2 0.9.9", + "sha2 0.9.8", "zeroize", ] @@ -11829,7 +11748,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -11845,7 +11764,7 @@ dependencies = [ "sp-maybe-compressed-blob", "strum", "tempfile", - "toml 0.7.6", + "toml 0.7.3", "walkdir", "wasm-opt", ] @@ -11926,9 +11845,9 @@ dependencies = [ [[package]] name = "symbolic-common" -version = "10.2.1" +version = "9.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" +checksum = "800963ba330b09a2ae4a4f7c6392b81fbc2784099a98c1eac68c3437aa9382b2" dependencies = [ "debugid", "memmap2", @@ -11938,11 +11857,11 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "10.2.1" +version = "9.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" +checksum = "2b940a1fdbc72bb3369e38714efe6cd332dbbe46d093cf03d668b9ac390d1ad0" dependencies = [ - "cpp_demangle 0.4.2", + "cpp_demangle", "rustc-demangle", "symbolic-common", ] @@ -11960,9 +11879,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.27" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +checksum = "fcb8d4cebc40aa517dfb69618fa647a346562e67228e2236ae0042ee6ac14775" dependencies = [ "proc-macro2", "quote", @@ -11983,11 +11902,11 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd" dependencies = [ - "bitflags 1.3.2", + "bitflags", "core-foundation", "system-configuration-sys", ] @@ -12010,37 +11929,38 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.12.10" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2faeef5759ab89935255b1a4cd98e0baf99d1085e37d36599c625dac49ae8e" +checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d" [[package]] name = "tempfile" -version = "3.7.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if", - "fastrand 2.0.0", - "redox_syscall 0.3.5", - "rustix 0.38.4", - "windows-sys 0.48.0", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" dependencies = [ "winapi-util", ] [[package]] name = "termtree" -version = "0.4.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +checksum = "13a4ec180a2de59b57434704ccfad967f789b12737738798fa08798cd5824c16" [[package]] name = "test-parachain-adder" @@ -12059,7 +11979,7 @@ dependencies = [ name = "test-parachain-adder-collator" version = "0.9.43" dependencies = [ - "clap 4.3.19", + "clap 4.2.5", "futures", "futures-timer", "log", @@ -12076,7 +11996,6 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", - "substrate-build-script-utils", "substrate-test-utils", "test-parachain-adder", "tokio", @@ -12108,7 +12027,7 @@ dependencies = [ name = "test-parachain-undying-collator" version = "0.9.43" dependencies = [ - "clap 4.3.19", + "clap 4.2.5", "futures", "futures-timer", "log", @@ -12125,7 +12044,6 @@ dependencies = [ "sc-service", "sp-core", "sp-keyring", - "substrate-build-script-utils", "substrate-test-utils", "test-parachain-undying", "tokio", @@ -12163,22 +12081,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -12189,11 +12107,10 @@ checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" dependencies = [ - "cfg-if", "once_cell", ] @@ -12232,11 +12149,12 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.5.3+5.3.0-patched" +version = "0.5.2+5.3.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a678df20055b43e57ef8cddde41cdfda9a3c1a060b67f4c5836dfb1d78543ba8" +checksum = "ec45c14da997d0925c7835883e4d5c181f196fa142f8c19d7643d1e9af2592c3" dependencies = [ "cc", + "fs_extra", "libc", ] @@ -12252,9 +12170,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.45" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -12310,9 +12228,9 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.1" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" @@ -12327,7 +12245,7 @@ dependencies = [ "mio", "num_cpus", "parking_lot 0.12.1", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "signal-hook-registry", "socket2 0.4.9", "tokio-macros", @@ -12342,7 +12260,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -12358,11 +12276,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" dependencies = [ - "rustls 0.20.8", + "rustls 0.20.7", "tokio", "webpki", ] @@ -12373,27 +12291,27 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.5", + "rustls 0.21.2", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" dependencies = [ "futures-core", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "tokio", "tokio-util", ] [[package]] name = "tokio-tungstenite" -version = "0.17.2" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "06cda1232a49558c46f8a504d5b93101d42c0bf7f911f12a105ba48168f821ae" dependencies = [ "futures-util", "log", @@ -12403,15 +12321,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "tokio", "tracing", ] @@ -12427,9 +12345,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.6" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" dependencies = [ "serde", "serde_spanned", @@ -12439,20 +12357,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.19.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" dependencies = [ - "indexmap 2.0.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -12472,18 +12390,18 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82" +checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" dependencies = [ - "bitflags 2.3.3", + "bitflags", "bytes", "futures-core", "futures-util", "http", "http-body", "http-range-header", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "tower-layer", "tower-service", ] @@ -12496,9 +12414,9 @@ checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" @@ -12508,27 +12426,27 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", - "pin-project-lite 0.2.10", + "pin-project-lite 0.2.9", "tracing-attributes", "tracing-core", ] [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 1.0.109", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -12564,7 +12482,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -12580,9 +12498,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.3" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" dependencies = [ "serde", "tracing-core", @@ -12613,13 +12531,13 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ + "ansi_term", + "lazy_static", "matchers 0.1.0", - "nu-ansi-term", - "once_cell", "regex", "sharded-slab", "smallvec", @@ -12699,9 +12617,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "try-runtime-cli" @@ -12709,7 +12627,7 @@ version = "0.10.0-dev" source = "git+https://github.com/paritytech/substrate?branch=master#d38d176b844aab1338ce79eb71cd6df86c97d4a0" dependencies = [ "async-trait", - "clap 4.3.19", + "clap 4.2.5", "frame-remote-externalities", "frame-try-runtime", "hex", @@ -12736,16 +12654,15 @@ dependencies = [ "sp-version", "sp-weights", "substrate-rpc-client", - "zstd 0.12.4", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] name = "trybuild" -version = "1.0.82" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84e0202ea606ba5ebee8507ab2bfbe89b98551ed9b8f0be198109275cff284b" +checksum = "f1212c215a87a183687a7cc7065901b1a98da6b37277d51a1b5faedbb4efd4f3" dependencies = [ - "basic-toml", "dissimilar", "glob", "once_cell", @@ -12753,28 +12670,29 @@ dependencies = [ "serde_derive", "serde_json", "termcolor", + "toml 0.5.11", ] [[package]] name = "tt-call" -version = "1.0.9" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f195fd851901624eee5a58c4bb2b4f06399148fcd0ed336e6f1cb60a9881df" +checksum = "5e66dcbec4290c69dd03c57e76c2469ea5c7ce109c6dd4351c13055cf71ea055" [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5" dependencies = [ - "base64 0.13.1", + "base64 0.13.0", "byteorder", "bytes", "http", "httparse", "log", "rand 0.8.5", - "sha-1 0.10.1", + "sha-1 0.10.0", "thiserror", "url", "utf-8", @@ -12800,15 +12718,15 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" [[package]] name = "uint" -version = "0.9.5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +checksum = "6470ab50f482bde894a037a57064480a246dbfdd5960bd65a44824693f08da5f" dependencies = [ "byteorder", "crunchy", @@ -12818,36 +12736,36 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" dependencies = [ "tinyvec", ] [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "universal-hash" @@ -12855,7 +12773,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.6", "subtle", ] @@ -12889,12 +12807,12 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna 0.4.0", + "idna 0.3.0", "percent-encoding", ] @@ -12912,9 +12830,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" [[package]] name = "valuable" @@ -12993,20 +12911,22 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", + "winapi", "winapi-util", ] [[package]] name = "want" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ + "log", "try-lock", ] @@ -13051,7 +12971,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", "wasm-bindgen-shared", ] @@ -13085,7 +13005,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -13166,7 +13086,7 @@ version = "0.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48134de3d7598219ab9eaf6b91b15d8e50d31da76b8519fe4ecfcec2cf35104b" dependencies = [ - "indexmap 1.9.3", + "indexmap", "url", ] @@ -13179,10 +13099,10 @@ dependencies = [ "anyhow", "bincode", "cfg-if", - "indexmap 1.9.3", + "indexmap", "libc", "log", - "object 0.30.4", + "object", "once_cell", "paste", "psm", @@ -13214,12 +13134,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c86437fa68626fe896e5afc69234bb2b5894949083586535f200385adfd71213" dependencies = [ "anyhow", - "base64 0.21.2", + "base64 0.21.0", "bincode", "directories-next", "file-per-thread-logger", "log", - "rustix 0.36.15", + "rustix 0.36.7", "serde", "sha2 0.10.7", "toml 0.5.11", @@ -13241,7 +13161,7 @@ dependencies = [ "cranelift-wasm", "gimli", "log", - "object 0.30.4", + "object", "target-lexicon", "thiserror", "wasmparser", @@ -13259,7 +13179,7 @@ dependencies = [ "cranelift-codegen", "cranelift-native", "gimli", - "object 0.30.4", + "object", "target-lexicon", "wasmtime-environ", ] @@ -13273,9 +13193,9 @@ dependencies = [ "anyhow", "cranelift-entity", "gimli", - "indexmap 1.9.3", + "indexmap", "log", - "object 0.30.4", + "object", "serde", "target-lexicon", "thiserror", @@ -13289,14 +13209,14 @@ version = "8.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de48df552cfca1c9b750002d3e07b45772dd033b0b206d5c0968496abf31244" dependencies = [ - "addr2line 0.19.0", + "addr2line", "anyhow", "bincode", "cfg-if", - "cpp_demangle 0.3.5", + "cpp_demangle", "gimli", "log", - "object 0.30.4", + "object", "rustc-demangle", "serde", "target-lexicon", @@ -13313,9 +13233,9 @@ version = "8.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e0554b84c15a27d76281d06838aed94e13a77d7bf604bbbaf548aa20eb93846" dependencies = [ - "object 0.30.4", + "object", "once_cell", - "rustix 0.36.15", + "rustix 0.36.7", ] [[package]] @@ -13338,7 +13258,7 @@ dependencies = [ "anyhow", "cc", "cfg-if", - "indexmap 1.9.3", + "indexmap", "libc", "log", "mach", @@ -13346,7 +13266,7 @@ dependencies = [ "memoffset 0.8.0", "paste", "rand 0.8.5", - "rustix 0.36.15", + "rustix 0.36.7", "wasmtime-asm-macros", "wasmtime-environ", "wasmtime-jit-debug", @@ -13367,9 +13287,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" dependencies = [ "js-sys", "wasm-bindgen", @@ -13387,9 +13307,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" dependencies = [ "webpki", ] @@ -13400,7 +13320,16 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "rustls-webpki 0.100.1", + "rustls-webpki", +] + +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", ] [[package]] @@ -13516,20 +13445,20 @@ dependencies = [ [[package]] name = "which" -version = "4.4.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" dependencies = [ "either", + "lazy_static", "libc", - "once_cell", ] [[package]] name = "wide" -version = "0.7.11" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa469ffa65ef7e0ba0f164183697b89b854253fd31aeb92358b7b6155177d62f" +checksum = "feff0a412894d67223777b6cc8d68c0dab06d52d95e9890d5f2d47f10dd9366c" dependencies = [ "bytemuck", "safe_arch", @@ -13537,9 +13466,9 @@ dependencies = [ [[package]] name = "widestring" -version = "1.0.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" [[package]] name = "winapi" @@ -13586,12 +13515,31 @@ dependencies = [ ] [[package]] -name = "windows" -version = "0.48.0" +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows-targets 0.48.1", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -13609,7 +13557,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.1", + "windows-targets 0.48.0", ] [[package]] @@ -13629,9 +13577,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ "windows_aarch64_gnullvm 0.48.0", "windows_aarch64_msvc 0.48.0", @@ -13660,6 +13608,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -13678,6 +13632,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -13696,6 +13656,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -13714,6 +13680,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -13744,6 +13716,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -13758,30 +13736,29 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.5.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b5872fa2e10bd067ae946f927e726d7d603eaeb6e02fa6a350e0722d2b8c11" +checksum = "deac0939bd6e4f24ab5919fbf751c97a8cfc8543bb083a305ed5c0c10bb241d1" dependencies = [ "memchr", ] [[package]] name = "winreg" -version = "0.10.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi", ] [[package]] name = "winreg" -version = "0.50.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "cfg-if", - "windows-sys 0.48.0", + "winapi", ] [[package]] @@ -13906,7 +13883,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.20", ] [[package]] @@ -14010,13 +13987,14 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.4.2" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 1.0.109", + "synstructure", ] [[package]] @@ -14047,11 +14025,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe 6.0.6", + "zstd-safe 6.0.5+zstd.1.5.4", ] [[package]] @@ -14066,9 +14044,9 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.6" +version = "6.0.5+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" dependencies = [ "libc", "zstd-sys", From 09f48402aedcb7f2aa9be3837d76fdaeeca7479c Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 27 Jul 2023 09:48:35 +0200 Subject: [PATCH 42/48] Remove unnecessary build scripts from collators --- .../test-parachains/adder/collator/Cargo.toml | 3 --- .../test-parachains/adder/collator/build.rs | 19 ------------------- .../adder/collator/src/main.rs | 6 +++++- .../undying/collator/Cargo.toml | 3 --- .../test-parachains/undying/collator/build.rs | 19 ------------------- .../undying/collator/src/main.rs | 6 +++++- 6 files changed, 10 insertions(+), 46 deletions(-) delete mode 100644 parachain/test-parachains/adder/collator/build.rs delete mode 100644 parachain/test-parachains/undying/collator/build.rs diff --git a/parachain/test-parachains/adder/collator/Cargo.toml b/parachain/test-parachains/adder/collator/Cargo.toml index 79fff4566496..29a10069e3e0 100644 --- a/parachain/test-parachains/adder/collator/Cargo.toml +++ b/parachain/test-parachains/adder/collator/Cargo.toml @@ -45,6 +45,3 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } - -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/parachain/test-parachains/adder/collator/build.rs b/parachain/test-parachains/adder/collator/build.rs deleted file mode 100644 index 40e9f832586e..000000000000 --- a/parachain/test-parachains/adder/collator/build.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); -} diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index a6c9d89ffcf7..c6f917897f35 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -64,12 +64,16 @@ fn main() -> Result<()> { enable_beefy: false, jaeger_agent: None, telemetry_worker_handle: None, - node_version: Some(env!("SUBSTRATE_CLI_IMPL_VERSION").into()), + + // Cumulus doesn't spawn PVF workers, so we can disable version checks and secure + // external workers. + node_version: None, workers_path: None, workers_names: None, // Don't use workers to make this test binary self-contained and easier to // use. External binaries are for increased security in production. dont_use_external_workers: true, + overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, diff --git a/parachain/test-parachains/undying/collator/Cargo.toml b/parachain/test-parachains/undying/collator/Cargo.toml index 6669f0f47ed2..f63757a20958 100644 --- a/parachain/test-parachains/undying/collator/Cargo.toml +++ b/parachain/test-parachains/undying/collator/Cargo.toml @@ -45,6 +45,3 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } - -[build-dependencies] -substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/parachain/test-parachains/undying/collator/build.rs b/parachain/test-parachains/undying/collator/build.rs deleted file mode 100644 index 40e9f832586e..000000000000 --- a/parachain/test-parachains/undying/collator/build.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -fn main() { - substrate_build_script_utils::generate_cargo_keys(); -} diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 7c7ad53f7216..36d31dd7eb0f 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -64,12 +64,16 @@ fn main() -> Result<()> { enable_beefy: false, jaeger_agent: None, telemetry_worker_handle: None, - node_version: Some(env!("SUBSTRATE_CLI_IMPL_VERSION").into()), + + // Cumulus doesn't spawn PVF workers, so we can disable version checks and secure + // external workers. + node_version: None, workers_path: None, workers_names: None, // Don't use workers to make this test binary self-contained and easier to // use. External binaries are for increased security in production. dont_use_external_workers: true, + overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, From 8851c201b5674bf8f93d6a56d8e4b6e56e160600 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 27 Jul 2023 11:13:05 +0200 Subject: [PATCH 43/48] Add back missing malus commands (should fix failing ZN job) --- node/malus/src/malus.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/node/malus/src/malus.rs b/node/malus/src/malus.rs index 791112079425..e881161dc080 100644 --- a/node/malus/src/malus.rs +++ b/node/malus/src/malus.rs @@ -36,6 +36,14 @@ enum NemesisVariant { BackGarbageCandidate(BackGarbageCandidateOptions), /// Delayed disputing of ancestors that are perfectly fine. DisputeAncestor(DisputeAncestorOptions), + + #[allow(missing_docs)] + #[command(name = "prepare-worker", hide = true)] + PvfPrepareWorker(polkadot_cli::ValidationWorkerCommand), + + #[allow(missing_docs)] + #[command(name = "execute-worker", hide = true)] + PvfExecuteWorker(polkadot_cli::ValidationWorkerCommand), } #[derive(Debug, Parser)] @@ -87,6 +95,35 @@ impl MalusCli { true, )? }, + NemesisVariant::PvfPrepareWorker(cmd) => { + #[cfg(target_os = "android")] + { + return Err("PVF preparation workers are not supported under this platform") + .into() + } + + #[cfg(not(target_os = "android"))] + { + polkadot_node_core_pvf_prepare_worker::worker_entrypoint( + &cmd.socket_path, + None, + ); + } + }, + NemesisVariant::PvfExecuteWorker(cmd) => { + #[cfg(target_os = "android")] + { + return Err("PVF execution workers are not supported under this platform").into() + } + + #[cfg(not(target_os = "android"))] + { + polkadot_node_core_pvf_execute_worker::worker_entrypoint( + &cmd.socket_path, + None, + ); + } + }, } Ok(()) } From fd0f018c0fc27f2229f9699d8cc3bcb4f3b5a8aa Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 27 Jul 2023 14:06:24 +0200 Subject: [PATCH 44/48] Some minor fixes --- Cargo.lock | 5 ++--- Cargo.toml | 1 - node/core/pvf/common/src/worker/mod.rs | 4 +--- node/malus/Cargo.toml | 2 ++ 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95d3bbee0007..7841b8d608fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6436,11 +6436,8 @@ dependencies = [ "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", - "sp-tracing", "substrate-build-script-utils", "substrate-rpc-client", "tempfile", @@ -7886,6 +7883,8 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", + "polkadot-node-core-pvf-execute-worker", + "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", diff --git a/Cargo.toml b/Cargo.toml index 260cdbd71c82..760c6ce39533 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,6 @@ polkadot-overseer = { path = "node/overseer" } # Needed for worker binaries. polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } -sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } [dev-dependencies] assert_cmd = "2.0.4" diff --git a/node/core/pvf/common/src/worker/mod.rs b/node/core/pvf/common/src/worker/mod.rs index 0d78b4f60e3d..8dd99fc762d8 100644 --- a/node/core/pvf/common/src/worker/mod.rs +++ b/node/core/pvf/common/src/worker/mod.rs @@ -41,9 +41,7 @@ macro_rules! decl_worker_main { } fn main() { - // NOTE: We don't want sp_tracing as a dependency in this crate to reduce this size. - // This means callers of this macro must depend on it. - ::sp_tracing::try_init_simple(); + $crate::sp_tracing::try_init_simple(); let args = std::env::args().collect::>(); if args.len() == 1 { diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index e9ed7406367a..32db361de372 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -20,6 +20,8 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator" } polkadot-node-core-candidate-validation = { path = "../core/candidate-validation" } polkadot-node-core-backing = { path = "../core/backing" } +polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } color-eyre = { version = "0.6.1", default-features = false } From e5771a0ba393b0446830eb2c50620becdbec65f6 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 27 Jul 2023 14:10:16 +0200 Subject: [PATCH 45/48] Update Cargo.lock --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 7841b8d608fb..8da607f937a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6436,6 +6436,8 @@ dependencies = [ "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", "substrate-build-script-utils", From 0a04dd4625f937d87548dc59c95f7ff7202298e8 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Thu, 27 Jul 2023 16:25:55 +0200 Subject: [PATCH 46/48] Fix some build errors --- node/core/pvf/common/src/lib.rs | 4 ++++ node/malus/src/malus.rs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/node/core/pvf/common/src/lib.rs b/node/core/pvf/common/src/lib.rs index 028fd9b17947..e5737a66aaec 100644 --- a/node/core/pvf/common/src/lib.rs +++ b/node/core/pvf/common/src/lib.rs @@ -25,6 +25,10 @@ pub mod worker; pub use cpu_time::ProcessTime; +// Used by `decl_worker_main!`. +#[doc(hidden)] +pub use sp_tracing; + const LOG_TARGET: &str = "parachain::pvf-common"; use std::mem; diff --git a/node/malus/src/malus.rs b/node/malus/src/malus.rs index e881161dc080..54abc81a7714 100644 --- a/node/malus/src/malus.rs +++ b/node/malus/src/malus.rs @@ -107,6 +107,7 @@ impl MalusCli { polkadot_node_core_pvf_prepare_worker::worker_entrypoint( &cmd.socket_path, None, + None, ); } }, @@ -121,6 +122,7 @@ impl MalusCli { polkadot_node_core_pvf_execute_worker::worker_entrypoint( &cmd.socket_path, None, + None, ); } }, From 9f05299d57abf076945add559512465527609b64 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Fri, 28 Jul 2023 15:31:35 +0200 Subject: [PATCH 47/48] Undo self-contained binaries; cli flag to disable version check + [X] Remove --dont-run-external-workers + [X] Add --disable-worker-version-check + [X] Remove PVF subcommands + [X] Redo malus changes --- Cargo.lock | 5 +- cli/Cargo.toml | 4 - cli/src/cli.rs | 12 +-- cli/src/command.rs | 78 +++---------------- node/core/candidate-validation/Cargo.toml | 1 + node/core/candidate-validation/src/lib.rs | 16 ++-- node/malus/Cargo.toml | 23 +++++- node/malus/build.rs | 22 ++++++ node/malus/src/malus.rs | 48 +----------- node/service/src/lib.rs | 44 +++++------ node/service/src/overseer.rs | 2 +- node/service/src/workers.rs | 2 + node/test/service/src/lib.rs | 13 +++- .../adder/collator/src/main.rs | 6 +- .../undying/collator/src/main.rs | 6 +- .../ci/dockerfiles/malus_injected.Dockerfile | 2 +- scripts/ci/gitlab/pipeline/build.yml | 2 + 17 files changed, 110 insertions(+), 176 deletions(-) create mode 100644 node/malus/build.rs diff --git a/Cargo.lock b/Cargo.lock index 8da607f937a2..1fc6175ad3a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6566,8 +6566,6 @@ dependencies = [ "frame-benchmarking-cli", "futures", "log", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-node-metrics", "polkadot-performance-test", "polkadot-service", @@ -6877,6 +6875,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-parachain", "polkadot-primitives", "polkadot-primitives-test-helpers", @@ -7885,6 +7884,7 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", + "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", @@ -7896,6 +7896,7 @@ dependencies = [ "rand 0.8.5", "sp-core", "sp-keystore", + "substrate-build-script-utils", "tracing-gum", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e7aa562880cc..7b782644125a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -22,8 +22,6 @@ pyro = { package = "pyroscope", version = "0.5.3", optional = true } pyroscope_pprofrs = { version = "0.2", optional = true } service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true } -polkadot-node-core-pvf-execute-worker = { path = "../node/core/pvf/execute-worker", optional = true } -polkadot-node-core-pvf-prepare-worker = { path = "../node/core/pvf/prepare-worker", optional = true } polkadot-performance-test = { path = "../node/test/performance-test", optional = true } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -53,8 +51,6 @@ cli = [ "sc-tracing", "frame-benchmarking-cli", "try-runtime-cli", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "service", ] runtime-benchmarks = [ diff --git a/cli/src/cli.rs b/cli/src/cli.rs index bc6ca19f7784..00ec54f8d969 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -46,14 +46,6 @@ pub enum Subcommand { /// Revert the chain to a previous state. Revert(sc_cli::RevertCmd), - #[allow(missing_docs)] - #[command(name = "prepare-worker", hide = true)] - PvfPrepareWorker(ValidationWorkerCommand), - - #[allow(missing_docs)] - #[command(name = "execute-worker", hide = true)] - PvfExecuteWorker(ValidationWorkerCommand), - /// Sub-commands concerned with benchmarking. /// The pallet benchmarking moved to the `pallet` sub-command. #[command(subcommand)] @@ -160,9 +152,9 @@ pub struct RunCmd { #[arg(long, value_name = "PATH")] pub workers_path: Option, - /// TESTING ONLY: don't use secure external PVF worker binaries. + /// TESTING ONLY: disable the version check between nodes and workers. #[arg(long, hide = true)] - pub dont_use_external_workers: bool, + pub disable_worker_version_check: bool, } #[allow(missing_docs)] diff --git a/cli/src/command.rs b/cli/src/command.rs index 7fddf0b6946f..ee71bb0840dc 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -222,15 +222,8 @@ pub fn run_node( run: Cli, overseer_gen: impl service::OverseerGen, malus_finality_delay: Option, - dont_use_external_workers: bool, ) -> Result<()> { - run_node_inner( - run, - overseer_gen, - malus_finality_delay, - |_logger_builder, _config| {}, - dont_use_external_workers, - ) + run_node_inner(run, overseer_gen, malus_finality_delay, |_logger_builder, _config| {}) } fn run_node_inner( @@ -238,7 +231,6 @@ fn run_node_inner( overseer_gen: impl service::OverseerGen, maybe_malus_finality_delay: Option, logger_hook: F, - dont_use_external_workers: bool, ) -> Result<()> where F: FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration), @@ -280,6 +272,9 @@ where None }; + let node_version = + if cli.run.disable_worker_version_check { None } else { Some(NODE_VERSION.to_string()) }; + runner.run_node_until_exit(move |config| async move { let hwbench = (!cli.run.no_hardware_benchmarks) .then_some(config.database.path().map(|database_path| { @@ -297,10 +292,9 @@ where enable_beefy, jaeger_agent, telemetry_worker_handle: None, - node_version: Some(NODE_VERSION.to_string()), + node_version, workers_path: cli.run.workers_path, workers_names: None, - dont_use_external_workers, overseer_enable_anyways: false, overseer_gen, overseer_message_channel_capacity_override: cli @@ -351,16 +345,12 @@ pub fn run() -> Result<()> { } match &cli.subcommand { - None => { - let dont_use_external_workers = cli.run.dont_use_external_workers; - run_node_inner( - cli, - service::RealOverseerGen, - None, - polkadot_node_metrics::logger_hook(), - dont_use_external_workers, - ) - }, + None => run_node_inner( + cli, + service::RealOverseerGen, + None, + polkadot_node_metrics::logger_hook(), + ), Some(Subcommand::BuildSpec(cmd)) => { let runner = cli.create_runner(cmd)?; Ok(runner.sync_run(|config| cmd.run(config.chain_spec, config.network))?) @@ -439,52 +429,6 @@ pub fn run() -> Result<()> { )) })?) }, - Some(Subcommand::PvfPrepareWorker(cmd)) => { - let mut builder = sc_cli::LoggerBuilder::new(""); - builder.with_colors(false); - let _ = builder.init(); - - #[cfg(target_os = "android")] - { - return Err(sc_cli::Error::Input( - "PVF preparation workers are not supported under this platform".into(), - ) - .into()) - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_prepare_worker::worker_entrypoint( - &cmd.socket_path, - Some(NODE_VERSION), - Some(&cmd.node_impl_version), - ); - Ok(()) - } - }, - Some(Subcommand::PvfExecuteWorker(cmd)) => { - let mut builder = sc_cli::LoggerBuilder::new(""); - builder.with_colors(false); - let _ = builder.init(); - - #[cfg(target_os = "android")] - { - return Err(sc_cli::Error::Input( - "PVF execution workers are not supported under this platform".into(), - ) - .into()) - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_execute_worker::worker_entrypoint( - &cmd.socket_path, - Some(NODE_VERSION), - Some(&cmd.node_impl_version), - ); - Ok(()) - } - }, Some(Subcommand::Benchmark(cmd)) => { let runner = cli.create_runner(cmd)?; let chain_spec = &runner.config().chain_spec; diff --git a/node/core/candidate-validation/Cargo.toml b/node/core/candidate-validation/Cargo.toml index 515aabbb3b41..c0fca9a49996 100644 --- a/node/core/candidate-validation/Cargo.toml +++ b/node/core/candidate-validation/Cargo.toml @@ -19,6 +19,7 @@ polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-node-metrics = { path = "../../metrics" } +polkadot-overseer = { path = "../../overseer" } [target.'cfg(not(any(target_os = "android", target_os = "unknown")))'.dependencies] polkadot-node-core-pvf = { path = "../pvf" } diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 26efad33ce10..93a7e05c8724 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -107,7 +107,7 @@ pub struct CandidateValidationSubsystem { pub metrics: Metrics, #[allow(missing_docs)] pub pvf_metrics: polkadot_node_core_pvf::Metrics, - config: Config, + config: Option, } impl CandidateValidationSubsystem { @@ -116,7 +116,7 @@ impl CandidateValidationSubsystem { /// /// Check out [`IsolationStrategy`] to get more details. pub fn with_config( - config: Config, + config: Option, metrics: Metrics, pvf_metrics: polkadot_node_core_pvf::Metrics, ) -> Self { @@ -127,10 +127,14 @@ impl CandidateValidationSubsystem { #[overseer::subsystem(CandidateValidation, error=SubsystemError, prefix=self::overseer)] impl CandidateValidationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { - let future = run(ctx, self.metrics, self.pvf_metrics, self.config) - .map_err(|e| SubsystemError::with_origin("candidate-validation", e)) - .boxed(); - SpawnedSubsystem { name: "candidate-validation-subsystem", future } + if let Some(config) = self.config { + let future = run(ctx, self.metrics, self.pvf_metrics, config) + .map_err(|e| SubsystemError::with_origin("candidate-validation", e)) + .boxed(); + SpawnedSubsystem { name: "candidate-validation-subsystem", future } + } else { + polkadot_overseer::DummySubsystem.start(ctx) + } } } diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 32db361de372..7e0bf0d8dd08 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -12,6 +12,19 @@ publish = false name = "malus" path = "src/malus.rs" +# Use artifact dependencies once stable. +# See https://github.com/rust-lang/cargo/issues/9096. +[[bin]] +name = "polkadot-execute-worker" +path = "../../src/bin/execute-worker.rs" +# Prevent rustdoc error. Already documented from top-level Cargo.toml. +doc = false +[[bin]] +name = "polkadot-prepare-worker" +path = "../../src/bin/prepare-worker.rs" +# Prevent rustdoc error. Already documented from top-level Cargo.toml. +doc = false + [dependencies] polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } polkadot-node-subsystem = { path = "../subsystem" } @@ -20,8 +33,6 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator" } polkadot-node-core-candidate-validation = { path = "../core/candidate-validation" } polkadot-node-core-backing = { path = "../core/backing" } -polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } color-eyre = { version = "0.6.1", default-features = false } @@ -36,11 +47,19 @@ gum = { package = "tracing-gum", path = "../gum/" } erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } rand = "0.8.5" +# Required for worker binaries to build. +polkadot-node-core-pvf-common = { path = "../core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } +polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } + [dev-dependencies] polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = { version = "0.3.21", features = ["thread-pool"] } +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + [features] default = [] fast-runtime = ["polkadot-cli/fast-runtime"] diff --git a/node/malus/build.rs b/node/malus/build.rs new file mode 100644 index 000000000000..84fe22e23ed6 --- /dev/null +++ b/node/malus/build.rs @@ -0,0 +1,22 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the node and binary workers + // when the version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); +} diff --git a/node/malus/src/malus.rs b/node/malus/src/malus.rs index 54abc81a7714..69dd7c869fc0 100644 --- a/node/malus/src/malus.rs +++ b/node/malus/src/malus.rs @@ -36,14 +36,6 @@ enum NemesisVariant { BackGarbageCandidate(BackGarbageCandidateOptions), /// Delayed disputing of ancestors that are perfectly fine. DisputeAncestor(DisputeAncestorOptions), - - #[allow(missing_docs)] - #[command(name = "prepare-worker", hide = true)] - PvfPrepareWorker(polkadot_cli::ValidationWorkerCommand), - - #[allow(missing_docs)] - #[command(name = "execute-worker", hide = true)] - PvfExecuteWorker(polkadot_cli::ValidationWorkerCommand), } #[derive(Debug, Parser)] @@ -63,12 +55,7 @@ impl MalusCli { NemesisVariant::BackGarbageCandidate(opts) => { let BackGarbageCandidateOptions { percentage, cli } = opts; - polkadot_cli::run_node( - cli, - BackGarbageCandidates { percentage }, - finality_delay, - true, - )? + polkadot_cli::run_node(cli, BackGarbageCandidates { percentage }, finality_delay)? }, NemesisVariant::SuggestGarbageCandidate(opts) => { let SuggestGarbageCandidateOptions { percentage, cli } = opts; @@ -77,7 +64,6 @@ impl MalusCli { cli, SuggestGarbageCandidates { percentage }, finality_delay, - true, )? }, NemesisVariant::DisputeAncestor(opts) => { @@ -92,40 +78,8 @@ impl MalusCli { cli, DisputeValidCandidates { fake_validation, fake_validation_error, percentage }, finality_delay, - true, )? }, - NemesisVariant::PvfPrepareWorker(cmd) => { - #[cfg(target_os = "android")] - { - return Err("PVF preparation workers are not supported under this platform") - .into() - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_prepare_worker::worker_entrypoint( - &cmd.socket_path, - None, - None, - ); - } - }, - NemesisVariant::PvfExecuteWorker(cmd) => { - #[cfg(target_os = "android")] - { - return Err("PVF execution workers are not supported under this platform").into() - } - - #[cfg(not(target_os = "android"))] - { - polkadot_node_core_pvf_execute_worker::worker_entrypoint( - &cmd.socket_path, - None, - None, - ); - } - }, } Ok(()) } diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 0b51f91f4bc6..efdd01d27321 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -251,7 +251,7 @@ pub enum Error { }, #[cfg(feature = "full-node")] - #[error("Version of worker binary ({worker_version}) is different from node version ({node_version}), worker_path: {worker_path}. TESTING ONLY: this check can be disabled with --dont-use-external-workers")] + #[error("Version of worker binary ({worker_version}) is different from node version ({node_version}), worker_path: {worker_path}. TESTING ONLY: this check can be disabled with --disable-worker-version-check")] WorkerBinaryVersionMismatch { worker_version: String, node_version: String, @@ -632,15 +632,13 @@ pub struct NewFullParams { pub enable_beefy: bool, pub jaeger_agent: Option, pub telemetry_worker_handle: Option, - /// The version of the node. `None` can be passed to skip the node/worker version check (only - /// for tests). + /// The version of the node. TESTING ONLY: `None` can be passed to skip the node/worker version + /// check, both on startup and in the workers. pub node_version: Option, /// An optional path to a directory containing the workers. pub workers_path: Option, /// Optional custom names for the prepare and execute workers. pub workers_names: Option<(String, String)>, - /// Use the current binary instead of relying on external workers. *Only for tests.* - pub dont_use_external_workers: bool, pub overseer_enable_anyways: bool, pub overseer_gen: OverseerGenerator, pub overseer_message_channel_capacity_override: Option, @@ -719,7 +717,6 @@ pub fn new_full( node_version, workers_path, workers_names, - dont_use_external_workers, overseer_enable_anyways, overseer_gen, overseer_message_channel_capacity_override, @@ -912,25 +909,24 @@ pub fn new_full( slot_duration_millis: slot_duration.as_millis() as u64, }; - let (prep_worker_path, exec_worker_path) = if dont_use_external_workers { - // Use the current binary. - let program_path = std::env::current_exe()?; - (program_path.clone(), program_path) + let candidate_validation_config = if is_collator.is_collator() { + None } else { - workers::determine_workers_paths(workers_path, workers_names, node_version.clone())? - }; - log::info!("🚀 Using prepare-worker binary at: {:?}", prep_worker_path); - log::info!("🚀 Using execute-worker binary at: {:?}", exec_worker_path); - - let candidate_validation_config = CandidateValidationConfig { - artifacts_cache_path: config - .database - .path() - .ok_or(Error::DatabasePathRequired)? - .join("pvf-artifacts"), - node_version, - prep_worker_path, - exec_worker_path, + let (prep_worker_path, exec_worker_path) = + workers::determine_workers_paths(workers_path, workers_names, node_version.clone())?; + log::info!("🚀 Using prepare-worker binary at: {:?}", prep_worker_path); + log::info!("🚀 Using execute-worker binary at: {:?}", exec_worker_path); + + Some(CandidateValidationConfig { + artifacts_cache_path: config + .database + .path() + .ok_or(Error::DatabasePathRequired)? + .join("pvf-artifacts"), + node_version, + prep_worker_path, + exec_worker_path, + }) }; let chain_selection_config = ChainSelectionConfig { diff --git a/node/service/src/overseer.rs b/node/service/src/overseer.rs index b1172cd9a549..29122ddca162 100644 --- a/node/service/src/overseer.rs +++ b/node/service/src/overseer.rs @@ -114,7 +114,7 @@ where /// Configuration for the availability store subsystem. pub availability_config: AvailabilityConfig, /// Configuration for the candidate validation subsystem. - pub candidate_validation_config: CandidateValidationConfig, + pub candidate_validation_config: Option, /// Configuration for the chain selection subsystem. pub chain_selection_config: ChainSelectionConfig, /// Configuration for the dispute coordinator subsystem. diff --git a/node/service/src/workers.rs b/node/service/src/workers.rs index d1c9e0bc4a13..5f7cc1c2ed49 100644 --- a/node/service/src/workers.rs +++ b/node/service/src/workers.rs @@ -99,6 +99,8 @@ pub fn determine_workers_paths( worker_path: exec_worker_path, }) } + } else { + log::warn!("Skipping node/worker version checks. This could result in incorrect behavior in PVF workers."); } Ok((prep_worker_path, exec_worker_path)) diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 98f622ac0bd7..99ccacb78f7e 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -74,7 +74,7 @@ pub fn new_full( is_collator: IsCollator, workers_path: Option, ) -> Result { - let dont_use_external_workers = !workers_path.is_some(); + let workers_path = Some(workers_path.unwrap_or_else(get_relative_workers_path_for_test)); polkadot_service::new_full( config, @@ -87,7 +87,6 @@ pub fn new_full( node_version: None, workers_path, workers_names: None, - dont_use_external_workers, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, overseer_message_channel_capacity_override: None, @@ -97,6 +96,16 @@ pub fn new_full( ) } +fn get_relative_workers_path_for_test() -> PathBuf { + // If no explicit worker path is passed in, we need to specify it ourselves as test binaries + // are in the "deps/" directory, one level below where the worker binaries are generated. + let mut exe_path = std::env::current_exe() + .expect("for test purposes it's reasonable to expect that this will not fail"); + let _ = exe_path.pop(); + let _ = exe_path.pop(); + exe_path +} + /// Returns a prometheus config usable for testing. pub fn test_prometheus_config(port: u16) -> PrometheusConfig { PrometheusConfig::new_with_default_registry( diff --git a/parachain/test-parachains/adder/collator/src/main.rs b/parachain/test-parachains/adder/collator/src/main.rs index c6f917897f35..d4bfc50c8db7 100644 --- a/parachain/test-parachains/adder/collator/src/main.rs +++ b/parachain/test-parachains/adder/collator/src/main.rs @@ -65,14 +65,10 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, - // Cumulus doesn't spawn PVF workers, so we can disable version checks and secure - // external workers. + // Collators don't spawn PVF workers, so we can disable version checks. node_version: None, workers_path: None, workers_names: None, - // Don't use workers to make this test binary self-contained and easier to - // use. External binaries are for increased security in production. - dont_use_external_workers: true, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, diff --git a/parachain/test-parachains/undying/collator/src/main.rs b/parachain/test-parachains/undying/collator/src/main.rs index 36d31dd7eb0f..3b6b4259aaec 100644 --- a/parachain/test-parachains/undying/collator/src/main.rs +++ b/parachain/test-parachains/undying/collator/src/main.rs @@ -65,14 +65,10 @@ fn main() -> Result<()> { jaeger_agent: None, telemetry_worker_handle: None, - // Cumulus doesn't spawn PVF workers, so we can disable version checks and secure - // external workers. + // Collators don't spawn PVF workers, so we can disable version checks. node_version: None, workers_path: None, workers_names: None, - // Don't use workers to make this test binary self-contained and easier to - // use. External binaries are for increased security in production. - dont_use_external_workers: true, overseer_enable_anyways: false, overseer_gen: polkadot_service::RealOverseerGen, diff --git a/scripts/ci/dockerfiles/malus_injected.Dockerfile b/scripts/ci/dockerfiles/malus_injected.Dockerfile index 1594350096f8..fa429b5f142a 100644 --- a/scripts/ci/dockerfiles/malus_injected.Dockerfile +++ b/scripts/ci/dockerfiles/malus_injected.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && \ # add malus binary to docker image -COPY ./malus /usr/local/bin +COPY ./malus ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin USER nonroot diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index 73702be0efc2..dafca393cd4f 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -83,6 +83,8 @@ build-malus: # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" From 047755006e327223f615b9a5a0b10482e3b44ca5 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Fri, 28 Jul 2023 16:16:18 +0200 Subject: [PATCH 48/48] Try to fix failing job and add some docs for local tests --- node/metrics/README.md | 9 +++++++++ node/test/service/README.md | 9 +++++++++ scripts/ci/gitlab/pipeline/test.yml | 3 +++ 3 files changed, 21 insertions(+) create mode 100644 node/metrics/README.md create mode 100644 node/test/service/README.md diff --git a/node/metrics/README.md b/node/metrics/README.md new file mode 100644 index 000000000000..cc88884f2142 --- /dev/null +++ b/node/metrics/README.md @@ -0,0 +1,9 @@ +# polkadot-node-metrics + +## Testing + +Before running `cargo test` in this crate, make sure the worker binaries are built first. This can be done with: + +```sh +cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker +``` diff --git a/node/test/service/README.md b/node/test/service/README.md new file mode 100644 index 000000000000..2fdee46a7f93 --- /dev/null +++ b/node/test/service/README.md @@ -0,0 +1,9 @@ +# polkadot-test-service + +## Testing + +Before running `cargo test` in this crate, make sure the worker binaries are built first. This can be done with: + +```sh +cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker +``` diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 60886dc60cde..b45c4c1be890 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -89,6 +89,9 @@ test-node-metrics: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: + # Build the required workers. + - cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked + # Run tests. - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics test-deterministic-wasm: