From d783e4d300c505a3a081b731c30bf91d3f7987d3 Mon Sep 17 00:00:00 2001 From: Filip Bozic <70634661+fbozic@users.noreply.github.com> Date: Fri, 10 Jan 2025 12:08:43 +0100 Subject: [PATCH] feat(e2e-tests): support protocol test matrix, implement icp protocol --- Cargo.lock | 1 + e2e-tests/Cargo.toml | 1 + e2e-tests/config/config.json | 23 +- e2e-tests/src/config.rs | 21 +- e2e-tests/src/driver.rs | 266 +++++++++++++-------- e2e-tests/src/main.rs | 3 + e2e-tests/src/protocol.rs | 27 +++ e2e-tests/src/protocol/icp.rs | 63 +++++ e2e-tests/src/protocol/near.rs | 76 ++++++ e2e-tests/src/steps.rs | 23 ++ e2e-tests/src/steps/application_install.rs | 4 + e2e-tests/src/steps/context_create.rs | 4 + e2e-tests/src/steps/context_invite_join.rs | 4 + e2e-tests/src/steps/jsonrpc_call.rs | 4 + 14 files changed, 417 insertions(+), 103 deletions(-) create mode 100644 e2e-tests/src/protocol.rs create mode 100644 e2e-tests/src/protocol/icp.rs create mode 100644 e2e-tests/src/protocol/near.rs diff --git a/Cargo.lock b/Cargo.lock index 84dabeda7..32781536d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2105,6 +2105,7 @@ dependencies = [ "serde", "serde_json", "tokio 1.41.1", + "url", ] [[package]] diff --git a/e2e-tests/Cargo.toml b/e2e-tests/Cargo.toml index bc5eaab25..2dd0f7ed3 100644 --- a/e2e-tests/Cargo.toml +++ b/e2e-tests/Cargo.toml @@ -19,6 +19,7 @@ rand.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true tokio = { workspace = true, features = ["fs", "io-util", "macros", "process", "rt", "rt-multi-thread", "time"] } +url = { workspace = true } [lints] workspace = true diff --git a/e2e-tests/config/config.json b/e2e-tests/config/config.json index e4f5f1509..f17eb8c97 100644 --- a/e2e-tests/config/config.json +++ b/e2e-tests/config/config.json @@ -8,8 +8,23 @@ "merod": { "args": [] }, - "near": { - "contextConfigContract": "./contracts/near/context-config/res/calimero_context_config_near.wasm", - "proxyLibContract": "./contracts/near/context-proxy/res/calimero_context_proxy_near.wasm" - } + "protocolSandboxes": [ + { + "protocol": "near", + "config": { + "contextConfigContract": "./contracts/near/context-config/res/calimero_context_config_near.wasm", + "proxyLibContract": "./contracts/near/context-proxy/res/calimero_context_proxy_near.wasm" + } + }, + { + "protocol": "icp", + "config": { + "contextConfigContractId": "bkyz2-fmaaa-aaaaa-qaaaq-cai", + "rpcUrl": "http://127.0.0.1:4943", + "accountId": "fph2z-lxdui-xq3o6-6kuqy-rgkwi-hq7us-gkwlq-gxfgs-irrcq-hnm4e-6qe", + "publicKey": "e3a22f0dbbde552188995641e1fa48cab2e06b94d24462281dace13d02", + "secretKey": "c9a8e56920efd1c7b6694dce6ce871b661ae3922d5045d4a9f04e131eaa34164" + } + } + ] } diff --git a/e2e-tests/src/config.rs b/e2e-tests/src/config.rs index c97994851..1ee89dbd6 100644 --- a/e2e-tests/src/config.rs +++ b/e2e-tests/src/config.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; pub struct Config { pub network: Network, pub merod: MerodConfig, - pub near: Near, + pub protocol_sandboxes: Box<[ProtocolSandboxConfig]>, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -24,9 +24,26 @@ pub struct MerodConfig { pub args: Box<[String]>, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "protocol", content = "config", rename_all = "camelCase")] +pub enum ProtocolSandboxConfig { + Near(NearProtocolConfig), + Icp(IcpProtocolConfig), +} + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -pub struct Near { +pub struct NearProtocolConfig { pub context_config_contract: Utf8PathBuf, pub proxy_lib_contract: Utf8PathBuf, } + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct IcpProtocolConfig { + pub context_config_contract_id: String, + pub rpc_url: String, + pub account_id: String, + pub public_key: String, + pub secret_key: String, +} diff --git a/e2e-tests/src/driver.rs b/e2e-tests/src/driver.rs index 5ba78ab0c..5f5e64a74 100644 --- a/e2e-tests/src/driver.rs +++ b/e2e-tests/src/driver.rs @@ -3,20 +3,20 @@ use std::env; use std::path::PathBuf; use std::time::Duration; -use eyre::{bail, Result as EyreResult}; -use near_workspaces::network::Sandbox; -use near_workspaces::types::NearToken; -use near_workspaces::{Account, Contract, Worker}; +use eyre::{bail, OptionExt, Result as EyreResult}; use rand::seq::SliceRandom; use serde_json::from_slice; use tokio::fs::{read, read_dir}; use tokio::time::sleep; -use crate::config::Config; +use crate::config::{Config, ProtocolSandboxConfig}; use crate::meroctl::Meroctl; use crate::merod::Merod; use crate::output::OutputWriter; -use crate::steps::{TestScenario, TestStep}; +use crate::protocol::icp::IcpSandboxEnvironment; +use crate::protocol::near::NearSandboxEnvironment; +use crate::protocol::ProtocolSandboxEnvironment; +use crate::steps::TestScenario; use crate::TestEnvironment; pub struct TestContext<'a> { @@ -32,6 +32,7 @@ pub struct TestContext<'a> { pub trait Test { async fn run_assert(&self, ctx: &mut TestContext<'_>) -> EyreResult<()>; + fn display_name(&self) -> String; } impl<'a> TestContext<'a> { @@ -59,13 +60,6 @@ pub struct Driver { config: Config, meroctl: Meroctl, merods: HashMap, - near: Option, -} - -pub struct NearSandboxEnvironment { - pub worker: Worker, - pub root_account: Account, - pub contract: Contract, } impl Driver { @@ -76,61 +70,45 @@ impl Driver { config, meroctl, merods: HashMap::new(), - near: None, } } pub async fn run(&mut self) -> EyreResult<()> { self.environment.init().await?; - self.init_near_environment().await?; - - let result = { - self.boot_merods().await?; - self.run_scenarios().await - }; + let mut sandbox_environments: Vec = Default::default(); + for protocol_sandbox in self.config.protocol_sandboxes.iter() { + match protocol_sandbox { + ProtocolSandboxConfig::Near(config) => { + let near = NearSandboxEnvironment::init(config.clone()).await?; + sandbox_environments.push(ProtocolSandboxEnvironment::Near(near)); + } + ProtocolSandboxConfig::Icp(config) => { + let icp = IcpSandboxEnvironment::init(config.clone()).await?; + sandbox_environments.push(ProtocolSandboxEnvironment::Icp(icp)); + } + } + } - self.stop_merods().await; + let mut report = TestRunReport::new(); + for sandbox in sandbox_environments.iter() { + self.boot_merods(sandbox).await?; + report = self.run_scenarios(report, sandbox.name()).await?; + self.stop_merods().await; + } - if let Err(e) = &result { + if let Err(e) = report.result() { self.environment .output_writer .write_str("Error occurred during test run:"); self.environment.output_writer.write_string(e.to_string()); } - result + println!("{}", report.to_markdown()); + report.result() } - async fn init_near_environment(&mut self) -> EyreResult<()> { - let worker = near_workspaces::sandbox().await?; - - let wasm = read(&self.config.near.context_config_contract).await?; - let context_config_contract = worker.dev_deploy(&wasm).await?; - - let proxy_lib_contract = read(&self.config.near.proxy_lib_contract).await?; - drop( - context_config_contract - .call("set_proxy_code") - .args(proxy_lib_contract) - .max_gas() - .transact() - .await? - .into_result()?, - ); - - let root_account = worker.root_account()?; - - self.near = Some(NearSandboxEnvironment { - worker, - root_account, - contract: context_config_contract, - }); - - Ok(()) - } - - async fn boot_merods(&mut self) -> EyreResult<()> { + async fn boot_merods(&mut self, sandbox: &ProtocolSandboxEnvironment) -> EyreResult<()> { self.environment .output_writer .write_header("Starting merod nodes", 2); @@ -143,40 +121,7 @@ impl Driver { self.environment.test_id )]; - if let Some(ref near) = self.near { - let near_account = near - .root_account - .create_subaccount(&node_name) - .initial_balance(NearToken::from_near(30)) - .transact() - .await? - .into_result()?; - let near_secret_key = near_account.secret_key(); - - args.extend(vec![ - format!( - "context.config.new.contract_id=\"{}\"", - near.contract.as_account().id() - ), - format!("context.config.signer.use=\"{}\"", "self"), - format!( - "context.config.signer.self.near.testnet.rpc_url=\"{}\"", - near.worker.rpc_addr() - ), - format!( - "context.config.signer.self.near.testnet.account_id=\"{}\"", - near_account.id() - ), - format!( - "context.config.signer.self.near.testnet.public_key=\"{}\"", - near_secret_key.public_key() - ), - format!( - "context.config.signer.self.near.testnet.secret_key=\"{}\"", - near_secret_key - ), - ]); - } + args.extend(sandbox.node_args(&node_name).await?); let mut config_args = vec![]; config_args.extend(args.iter().map(|arg| &**arg)); @@ -219,7 +164,11 @@ impl Driver { self.merods.clear(); } - async fn run_scenarios(&self) -> EyreResult<()> { + async fn run_scenarios( + &self, + mut report: TestRunReport, + protocol_name: String, + ) -> EyreResult { let scenarios_dir = self.environment.input_dir.join("scenarios"); let mut entries = read_dir(scenarios_dir).await?; @@ -228,15 +177,35 @@ impl Driver { if path.is_dir() { let test_file_path = path.join("test.json"); if test_file_path.exists() { - self.run_scenario(test_file_path).await?; + let scenario_report = self + .run_scenario( + path.file_name() + .ok_or_eyre("failed")? + .to_str() + .ok_or_eyre("failed")?, + test_file_path, + ) + .await?; + + drop( + report + .scenario_matrix + .entry(scenario_report.scenario_name.clone()) + .or_insert_with(HashMap::new) + .insert(protocol_name.clone(), scenario_report), + ); } } } - Ok(()) + Ok(report) } - async fn run_scenario(&self, file_path: PathBuf) -> EyreResult<()> { + async fn run_scenario( + &self, + scenarion_name: &str, + file_path: PathBuf, + ) -> EyreResult { self.environment .output_writer .write_header("Running scenario", 2); @@ -269,6 +238,8 @@ impl Driver { self.environment.output_writer, ); + let mut report = TestScenarioReport::new(scenarion_name.to_owned()); + for step in scenario.steps.iter() { self.environment .output_writer @@ -276,15 +247,14 @@ impl Driver { self.environment.output_writer.write_str("Step spec:"); self.environment.output_writer.write_json(&step)?; - match step { - TestStep::ApplicationInstall(step) => step.run_assert(&mut ctx).await?, - TestStep::ContextCreate(step) => step.run_assert(&mut ctx).await?, - TestStep::ContextInviteJoin(step) => step.run_assert(&mut ctx).await?, - TestStep::JsonRpcCall(step) => step.run_assert(&mut ctx).await?, - }; + let result = step.run_assert(&mut ctx).await; + report.steps.push(TestStepReport { + step_name: step.display_name(), + result, + }); } - Ok(()) + Ok(report) } fn pick_inviter_node(&self) -> Option<(String, Vec)> { @@ -299,3 +269,105 @@ impl Driver { } } } + +struct TestRunReport { + scenario_matrix: HashMap>, +} + +impl TestRunReport { + fn new() -> Self { + Self { + scenario_matrix: Default::default(), + } + } + + fn result(&self) -> EyreResult<()> { + let mut errors = vec![]; + + for (_, scenarios) in &self.scenario_matrix { + for (_, scenario) in scenarios { + for step in &scenario.steps { + if let Err(e) = &step.result { + errors.push(e.to_string()); + } + } + } + } + + if errors.is_empty() { + Ok(()) + } else { + bail!("Errors occurred during test run: {:?}", errors) + } + } + + fn to_markdown(&self) -> String { + let mut markdown = String::new(); + + for (scenario, protocols) in &self.scenario_matrix { + markdown.push_str(&format!("## Scenario: {}\n", scenario)); + markdown.push_str("| Protocol/Step |"); + + // Collecting all step names + let mut step_names = vec![]; + for report in protocols.values() { + for step in &report.steps { + if !step_names.contains(&step.step_name) { + step_names.push(step.step_name.clone()); + } + } + } + + // Adding step names to the first row of the table + for step_name in &step_names { + markdown.push_str(&format!(" {} |", step_name)); + } + markdown.push_str("\n| :--- |"); + for _ in &step_names { + markdown.push_str(" :---: |"); + } + markdown.push_str("\n"); + + // Adding protocol rows + for (protocol, report) in protocols { + markdown.push_str(&format!("| {} |", protocol)); + for step_name in &step_names { + let result = report + .steps + .iter() + .find(|step| &step.step_name == step_name) + .map_or("N/A", |step| { + if step.result.is_ok() { + "Success" + } else { + "Failure" + } + }); + markdown.push_str(&format!(" {} |", result)); + } + markdown.push_str("\n"); + } + markdown.push_str("\n"); + } + markdown + } +} + +struct TestScenarioReport { + scenario_name: String, + steps: Vec, +} + +impl TestScenarioReport { + fn new(scenario_name: String) -> Self { + Self { + scenario_name, + steps: Default::default(), + } + } +} + +struct TestStepReport { + step_name: String, + result: EyreResult<()>, +} diff --git a/e2e-tests/src/main.rs b/e2e-tests/src/main.rs index 8f7de8b27..063e2a8f3 100644 --- a/e2e-tests/src/main.rs +++ b/e2e-tests/src/main.rs @@ -13,6 +13,7 @@ mod driver; mod meroctl; mod merod; mod output; +mod protocol; mod steps; pub const EXAMPLES: &str = r" @@ -69,6 +70,7 @@ pub struct TestEnvironment { pub output_dir: Utf8PathBuf, pub nodes_dir: Utf8PathBuf, pub logs_dir: Utf8PathBuf, + pub icp_dir: Utf8PathBuf, pub output_writer: OutputWriter, } @@ -84,6 +86,7 @@ impl Into for Args { output_dir: self.output_dir.clone(), nodes_dir: self.output_dir.join("nodes"), logs_dir: self.output_dir.join("logs"), + icp_dir: self.output_dir.join("icp"), output_writer: OutputWriter::new(self.output_format), } } diff --git a/e2e-tests/src/protocol.rs b/e2e-tests/src/protocol.rs new file mode 100644 index 000000000..a03676800 --- /dev/null +++ b/e2e-tests/src/protocol.rs @@ -0,0 +1,27 @@ +use eyre::Result as EyreResult; +use icp::IcpSandboxEnvironment; +use near::NearSandboxEnvironment; + +pub mod icp; +pub mod near; + +pub enum ProtocolSandboxEnvironment { + Near(NearSandboxEnvironment), + Icp(IcpSandboxEnvironment), +} + +impl ProtocolSandboxEnvironment { + pub async fn node_args(&self, node_name: &str) -> EyreResult> { + match self { + ProtocolSandboxEnvironment::Near(env) => env.node_args(node_name).await, + ProtocolSandboxEnvironment::Icp(env) => env.node_args(), + } + } + + pub fn name(&self) -> String { + match self { + ProtocolSandboxEnvironment::Near(_) => "near".to_string(), + ProtocolSandboxEnvironment::Icp(_) => "icp".to_string(), + } + } +} diff --git a/e2e-tests/src/protocol/icp.rs b/e2e-tests/src/protocol/icp.rs new file mode 100644 index 000000000..03ca65d08 --- /dev/null +++ b/e2e-tests/src/protocol/icp.rs @@ -0,0 +1,63 @@ +use std::{net::TcpStream, time::Duration}; + +use eyre::{bail, OptionExt, Result as EyreResult}; +use url::Url; + +use crate::config::IcpProtocolConfig; + +pub struct IcpSandboxEnvironment { + config: IcpProtocolConfig, +} + +impl IcpSandboxEnvironment { + pub async fn init(config: IcpProtocolConfig) -> EyreResult { + let rpc_url = Url::parse(&config.rpc_url)?; + let rpc_host = rpc_url + .host_str() + .ok_or_eyre("failed to get icp rpc host from config")?; + let rpc_port = rpc_url + .port() + .ok_or_eyre("failed to get icp rpc port from config")?; + + if let Err(err) = TcpStream::connect_timeout( + &format!("{}:{}", rpc_host, rpc_port).parse()?, + Duration::from_secs(3), + ) { + bail!( + "Failed to connect to icp rpc url '{}': {}", + &config.rpc_url, + err + ); + } + + return Ok(Self { config }); + } + + pub fn node_args(&self) -> EyreResult> { + return Ok(vec![ + format!("context.config.new.protocol=\"{}\"", "icp"), + format!("context.config.new.network=\"{}\"", "local"), + format!( + "context.config.new.contract_id=\"{}\"", + self.config.context_config_contract_id + ), + format!("context.config.signer.use=\"{}\"", "self"), + format!( + "context.config.signer.self.icp.local.rpc_url=\"{}\"", + self.config.rpc_url + ), + format!( + "context.config.signer.self.icp.local.account_id=\"{}\"", + self.config.account_id + ), + format!( + "context.config.signer.self.icp.local.public_key=\"{}\"", + self.config.public_key + ), + format!( + "context.config.signer.self.icp.local.secret_key=\"{}\"", + self.config.secret_key + ), + ]); + } +} diff --git a/e2e-tests/src/protocol/near.rs b/e2e-tests/src/protocol/near.rs new file mode 100644 index 000000000..b5378f4fb --- /dev/null +++ b/e2e-tests/src/protocol/near.rs @@ -0,0 +1,76 @@ +use eyre::Result as EyreResult; +use near_workspaces::network::Sandbox; +use near_workspaces::types::NearToken; +use near_workspaces::{Account, Contract, Worker}; +use tokio::fs::read; + +use crate::config::NearProtocolConfig; + +pub struct NearSandboxEnvironment { + pub worker: Worker, + pub root_account: Account, + pub contract: Contract, +} + +impl NearSandboxEnvironment { + pub async fn init(config: NearProtocolConfig) -> EyreResult { + let worker = near_workspaces::sandbox().await?; + + let wasm = read(&config.context_config_contract).await?; + let context_config_contract = worker.dev_deploy(&wasm).await?; + + let proxy_lib_contract = read(&config.proxy_lib_contract).await?; + drop( + context_config_contract + .call("set_proxy_code") + .args(proxy_lib_contract) + .max_gas() + .transact() + .await? + .into_result()?, + ); + + let root_account = worker.root_account()?; + + Ok(Self { + worker, + root_account, + contract: context_config_contract, + }) + } + + pub async fn node_args(&self, node_name: &str) -> EyreResult> { + let near_account = self + .root_account + .create_subaccount(node_name) + .initial_balance(NearToken::from_near(30)) + .transact() + .await? + .into_result()?; + let near_secret_key = near_account.secret_key(); + + return Ok(vec![ + format!( + "context.config.new.contract_id=\"{}\"", + self.contract.as_account().id() + ), + format!("context.config.signer.use=\"{}\"", "self"), + format!( + "context.config.signer.self.near.testnet.rpc_url=\"{}\"", + self.worker.rpc_addr() + ), + format!( + "context.config.signer.self.near.testnet.account_id=\"{}\"", + near_account.id() + ), + format!( + "context.config.signer.self.near.testnet.public_key=\"{}\"", + near_secret_key.public_key() + ), + format!( + "context.config.signer.self.near.testnet.secret_key=\"{}\"", + near_secret_key + ), + ]); + } +} diff --git a/e2e-tests/src/steps.rs b/e2e-tests/src/steps.rs index 2cbb3a46e..ff60869db 100644 --- a/e2e-tests/src/steps.rs +++ b/e2e-tests/src/steps.rs @@ -1,9 +1,12 @@ use application_install::ApplicationInstallStep; use context_create::ContextCreateStep; use context_invite_join::ContextInviteJoinStep; +use eyre::Result as EyreResult; use jsonrpc_call::JsonRpcCallStep; use serde::{Deserialize, Serialize}; +use crate::driver::{Test, TestContext}; + mod application_install; mod context_create; mod context_invite_join; @@ -23,3 +26,23 @@ pub enum TestStep { ContextInviteJoin(ContextInviteJoinStep), JsonRpcCall(JsonRpcCallStep), } + +impl Test for TestStep { + fn display_name(&self) -> String { + match self { + TestStep::ApplicationInstall(step) => step.display_name(), + TestStep::ContextCreate(step) => step.display_name(), + TestStep::ContextInviteJoin(step) => step.display_name(), + TestStep::JsonRpcCall(step) => step.display_name(), + } + } + + async fn run_assert(&self, ctx: &mut TestContext<'_>) -> EyreResult<()> { + match self { + TestStep::ApplicationInstall(step) => step.run_assert(ctx).await, + TestStep::ContextCreate(step) => step.run_assert(ctx).await, + TestStep::ContextInviteJoin(step) => step.run_assert(ctx).await, + TestStep::JsonRpcCall(step) => step.run_assert(ctx).await, + } + } +} diff --git a/e2e-tests/src/steps/application_install.rs b/e2e-tests/src/steps/application_install.rs index 1444095a4..11558b141 100644 --- a/e2e-tests/src/steps/application_install.rs +++ b/e2e-tests/src/steps/application_install.rs @@ -26,6 +26,10 @@ pub enum ApplicationInstallTarget { } impl Test for ApplicationInstallStep { + fn display_name(&self) -> String { + "ApplicationInstall".to_string() + } + async fn run_assert(&self, ctx: &mut TestContext<'_>) -> EyreResult<()> { if let ApplicationInstallTarget::AllMembers = self.target { for invitee in ctx.invitees.iter() { diff --git a/e2e-tests/src/steps/context_create.rs b/e2e-tests/src/steps/context_create.rs index 28eb55da6..2a73f062d 100644 --- a/e2e-tests/src/steps/context_create.rs +++ b/e2e-tests/src/steps/context_create.rs @@ -8,6 +8,10 @@ use crate::driver::{Test, TestContext}; pub struct ContextCreateStep; impl Test for ContextCreateStep { + fn display_name(&self) -> String { + "ContextCreate".to_string() + } + async fn run_assert(&self, ctx: &mut TestContext<'_>) -> EyreResult<()> { let Some(ref application_id) = ctx.application_id else { bail!("Application ID is required for ContextCreateStep"); diff --git a/e2e-tests/src/steps/context_invite_join.rs b/e2e-tests/src/steps/context_invite_join.rs index 7e360b038..f24b26329 100644 --- a/e2e-tests/src/steps/context_invite_join.rs +++ b/e2e-tests/src/steps/context_invite_join.rs @@ -11,6 +11,10 @@ use crate::driver::{Test, TestContext}; pub struct ContextInviteJoinStep; impl Test for ContextInviteJoinStep { + fn display_name(&self) -> String { + "ContextInviteJoin".to_string() + } + async fn run_assert(&self, ctx: &mut TestContext<'_>) -> EyreResult<()> { let Some(ref context_id) = ctx.context_id else { bail!("Context ID is required for InviteJoinContextStep"); diff --git a/e2e-tests/src/steps/jsonrpc_call.rs b/e2e-tests/src/steps/jsonrpc_call.rs index 89cb98bbb..1b7aa0c7d 100644 --- a/e2e-tests/src/steps/jsonrpc_call.rs +++ b/e2e-tests/src/steps/jsonrpc_call.rs @@ -22,6 +22,10 @@ pub enum JsonRpcCallTarget { } impl Test for JsonRpcCallStep { + fn display_name(&self) -> String { + "JsonRpcCall".to_string() + } + async fn run_assert(&self, ctx: &mut TestContext<'_>) -> EyreResult<()> { let Some(ref context_id) = ctx.context_id else { bail!("Context ID is required for JsonRpcExecuteStep");