From 786ca7215dc07f95a1a0b332098cc08d977f1270 Mon Sep 17 00:00:00 2001 From: Filip Bozic <70634661+fbozic@users.noreply.github.com> Date: Tue, 19 Nov 2024 19:08:26 +0100 Subject: [PATCH] ci: setup e2e-tests in github workflow --- .github/workflows/e2e_tests.yml | 75 +++++++++++++++++++ e2e-tests/README.md | 18 ++++- e2e-tests/config/scenarios/kv-store/test.json | 2 +- e2e-tests/src/driver.rs | 66 +++++++++++----- e2e-tests/src/main.rs | 9 +++ e2e-tests/src/meroctl.rs | 6 +- e2e-tests/src/merod.rs | 6 +- e2e-tests/src/output.rs | 65 ++++++++++++++++ e2e-tests/src/steps/context_create.rs | 5 +- e2e-tests/src/steps/context_invite_join.rs | 3 +- e2e-tests/src/steps/jsonrpc_call.rs | 3 +- 11 files changed, 232 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/e2e_tests.yml create mode 100644 e2e-tests/src/output.rs diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml new file mode 100644 index 000000000..5d2f985a0 --- /dev/null +++ b/.github/workflows/e2e_tests.yml @@ -0,0 +1,75 @@ +name: End-to-end tests + +on: + push: + branches: + - '**' + paths: + - Cargo.toml + - Cargo.lock + - 'contracts/**' + - 'crates/**' + - 'e2e-tests/**' + - '.github/workflows/e2e_tests.yml' + +jobs: + test: + name: Test + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup rust toolchain + run: rustup toolchain install stable --profile minimal + + - name: Setup rust cache + uses: Swatinem/rust-cache@v2 + + - name: Build apps + run: | + ./apps/kv-store/build.sh + + - name: Build contracts + run: | + ./contracts/context-config/build.sh + ./contracts/proxy-lib/build.sh + + - name: Build binaries + run: | + cargo build -p meroctl -p merod -p e2e-tests + + - name: Run e2e tests + run: | + export NO_COLOR=1 + echo "# E2E tests 🏗️" >> $GITHUB_STEP_SUMMARY + ./target/debug/e2e-tests \ + --input-dir ./e2e-tests/config \ + --output-dir ./e2e-tests/corpus \ + --merod-binary ./target/debug/merod \ + --meroctl-binary ./target/debug/meroctl >> $GITHUB_STEP_SUMMARY + + - name: Run e2e tests + if: success() || failure() + run: | + LOGS_DIR=./e2e-tests/corpus/logs + if [ ! -d "$LOGS_DIR" ]; then + echo "Directory $LOGS_DIR does not exist." + exit 1 + fi + + echo "# Node logs 📋" >> $GITHUB_STEP_SUMMARY + + for FOLDER in "$LOGS_DIR"/*; do + if [ -d "$FOLDER" ]; then + + RUN_LOG="$FOLDER/run.log" + if [ -f "$RUN_LOG" ]; then + echo "## Node logs: $(basename $FOLDER) 📋" >> $GITHUB_STEP_SUMMARY + cat "$RUN_LOG" >> $GITHUB_STEP_SUMMARY + else + echo "## No run.log found in $FOLDER ⚠️" >> $GITHUB_STEP_SUMMARY + fi + fi + done diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 299b0387d..dcc926fed 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -4,12 +4,26 @@ Binary crate which runs e2e tests for the merod node. ## Usage -Build the merod and meroctl binaries and run the e2e tests with the following -commands: +First build apps, contracts, and mero binaries. After that run the e2e tests. + +Example of running the e2e tests: ```bash +./apps/kv-store/build.sh + +./contracts/context-config/build.sh +./contracts/proxy-lib/build.sh + cargo build -p merod cargo build -p meroctl +export NO_COLOR=1 # Disable color output for merod logs cargo run -p e2e-tests -- --input-dir ./e2e-tests/config --output-dir ./e2e-tests/corpus --merod-binary ./target/debug/merod --meroctl-binary ./target/debug/meroctl ``` + +Useful env vars for debugging: + +- `RUST_LOG=debug` - enable debug logs + - `RUST_LOG=near_jsonrpc_client=debug` - or more specific logs +- `NEAR_ENABLE_SANDBOX_LOG=1` - enable near sandbox logs +- `NO_COLOR=1` - disable color output diff --git a/e2e-tests/config/scenarios/kv-store/test.json b/e2e-tests/config/scenarios/kv-store/test.json index d421d4342..101cc745a 100644 --- a/e2e-tests/config/scenarios/kv-store/test.json +++ b/e2e-tests/config/scenarios/kv-store/test.json @@ -7,7 +7,7 @@ }, { "jsonRpcCall": { - "methodName": "set", + "methodName": "sets", "argsJson": { "key": "foo", "value": "bar" }, "expectedResultJson": null, "target": "inviter" diff --git a/e2e-tests/src/driver.rs b/e2e-tests/src/driver.rs index 728acc814..d91c4f7a4 100644 --- a/e2e-tests/src/driver.rs +++ b/e2e-tests/src/driver.rs @@ -14,6 +14,7 @@ use tokio::time::sleep; use crate::config::Config; use crate::meroctl::Meroctl; use crate::merod::Merod; +use crate::output::OutputWriter; use crate::steps::{TestScenario, TestStep}; use crate::TestEnvironment; @@ -24,6 +25,7 @@ pub struct TestContext<'a> { pub context_id: Option, pub inviter_public_key: Option, pub invitees_public_keys: HashMap, + pub output_writer: OutputWriter, } pub trait Test { @@ -31,7 +33,12 @@ pub trait Test { } impl<'a> TestContext<'a> { - pub fn new(inviter: String, invitees: Vec, meroctl: &'a Meroctl) -> Self { + pub fn new( + inviter: String, + invitees: Vec, + meroctl: &'a Meroctl, + output_writer: OutputWriter, + ) -> Self { Self { inviter, invitees, @@ -39,6 +46,7 @@ impl<'a> TestContext<'a> { context_id: None, inviter_public_key: None, invitees_public_keys: HashMap::new(), + output_writer, } } } @@ -81,6 +89,13 @@ impl Driver { self.stop_merods().await; + if let Err(e) = &result { + self.environment + .output_writer + .write_str("Error occurred during test run:"); + self.environment.output_writer.write_string(e.to_string()); + } + result } @@ -113,7 +128,9 @@ impl Driver { } async fn boot_merods(&mut self) -> EyreResult<()> { - println!("========================= Starting nodes ==========================="); + self.environment + .output_writer + .write_header("Starting merod nodes", 2); for i in 0..self.config.network.node_count { let node_name = format!("node{}", i + 1); @@ -178,9 +195,7 @@ impl Driver { } // TODO: Implement health check? - sleep(Duration::from_secs(20)).await; - - println!("===================================================================="); + sleep(Duration::from_secs(10)).await; Ok(()) } @@ -213,35 +228,50 @@ impl Driver { } async fn run_scenario(&self, file_path: PathBuf) -> EyreResult<()> { - println!("================= Setting up scenario and context =================="); + self.environment + .output_writer + .write_header("Running scenario", 2); + let scenario: TestScenario = from_slice(&read(&file_path).await?)?; - println!( - "Loaded test scenario from file: {:?}\n{:?}", - file_path, scenario - ); + self.environment + .output_writer + .write_string(format!("Source file: {:?}", file_path)); + self.environment + .output_writer + .write_string(format!("Steps count: {}", scenario.steps.len())); let (inviter, invitees) = match self.pick_inviter_node() { Some((inviter, invitees)) => (inviter, invitees), None => bail!("Not enough nodes to run the test"), }; - println!("Picked inviter: {}", inviter); - println!("Picked invitees: {:?}", invitees); + self.environment + .output_writer + .write_string(format!("Picked inviter: {}", inviter)); + self.environment + .output_writer + .write_string(format!("Picked invitees: {:?}", invitees)); - let mut ctx = TestContext::new(inviter, invitees, &self.meroctl); - - println!("===================================================================="); + let mut ctx = TestContext::new( + inviter, + invitees, + &self.meroctl, + self.environment.output_writer, + ); for step in scenario.steps.iter() { - println!("======================== Starting step ============================="); - println!("Step: {:?}", step); + self.environment + .output_writer + .write_header("Running test step", 3); + self.environment.output_writer.write_str("Step spec:"); + self.environment.output_writer.write_json(&step)?; + match step { TestStep::ContextCreate(step) => step.run_assert(&mut ctx).await?, TestStep::ContextInviteJoin(step) => step.run_assert(&mut ctx).await?, TestStep::JsonRpcCall(step) => step.run_assert(&mut ctx).await?, }; - println!("===================================================================="); } Ok(()) diff --git a/e2e-tests/src/main.rs b/e2e-tests/src/main.rs index a566f8705..8f7de8b27 100644 --- a/e2e-tests/src/main.rs +++ b/e2e-tests/src/main.rs @@ -4,6 +4,7 @@ use config::Config; use const_format::concatcp; use driver::Driver; use eyre::Result as EyreResult; +use output::{OutputFormat, OutputWriter}; use rand::Rng; use tokio::fs::{create_dir_all, read_to_string, remove_dir_all}; @@ -11,6 +12,7 @@ mod config; mod driver; mod meroctl; mod merod; +mod output; mod steps; pub const EXAMPLES: &str = r" @@ -51,6 +53,11 @@ pub struct Args { #[arg(long, value_name = "PATH")] #[arg(env = "MEROCTL_BINARY", hide_env_values = true)] pub meroctl_binary: Utf8PathBuf, + + /// Format of the E2E test output. + #[arg(long, value_name = "OUTPUT_FORMAT", default_value_t, value_enum)] + #[arg(env = "E2E_OUTPUT_FORMAT", hide_env_values = true)] + pub output_format: OutputFormat, } #[derive(Debug)] @@ -62,6 +69,7 @@ pub struct TestEnvironment { pub output_dir: Utf8PathBuf, pub nodes_dir: Utf8PathBuf, pub logs_dir: Utf8PathBuf, + pub output_writer: OutputWriter, } impl Into for Args { @@ -76,6 +84,7 @@ impl Into for Args { output_dir: self.output_dir.clone(), nodes_dir: self.output_dir.join("nodes"), logs_dir: self.output_dir.join("logs"), + output_writer: OutputWriter::new(self.output_format), } } } diff --git a/e2e-tests/src/meroctl.rs b/e2e-tests/src/meroctl.rs index 3eca3da26..75b798eca 100644 --- a/e2e-tests/src/meroctl.rs +++ b/e2e-tests/src/meroctl.rs @@ -4,11 +4,13 @@ use camino::Utf8PathBuf; use eyre::{bail, eyre, OptionExt, Result as EyreResult}; use tokio::process::Command; +use crate::output::OutputWriter; use crate::TestEnvironment; pub struct Meroctl { nodes_dir: Utf8PathBuf, binary: Utf8PathBuf, + output_writer: OutputWriter, } impl Meroctl { @@ -16,6 +18,7 @@ impl Meroctl { Self { nodes_dir: environment.nodes_dir.clone(), binary: environment.meroctl_binary.clone(), + output_writer: environment.output_writer, } } @@ -148,7 +151,8 @@ impl Meroctl { root_args.extend(args); - println!("Command: '{:}' {:?}", &self.binary, root_args); + self.output_writer + .write_string(format!("Command: '{:}' {:?}", &self.binary, root_args)); let output = Command::new(&self.binary) .args(root_args) diff --git a/e2e-tests/src/merod.rs b/e2e-tests/src/merod.rs index f3edcf2aa..61c413c4d 100644 --- a/e2e-tests/src/merod.rs +++ b/e2e-tests/src/merod.rs @@ -7,6 +7,7 @@ use tokio::fs::{create_dir_all, File}; use tokio::io::copy; use tokio::process::{Child, Command}; +use crate::output::OutputWriter; use crate::TestEnvironment; pub struct Merod { @@ -15,6 +16,7 @@ pub struct Merod { nodes_dir: Utf8PathBuf, log_dir: Utf8PathBuf, binary: Utf8PathBuf, + output_writer: OutputWriter, } impl Merod { @@ -25,6 +27,7 @@ impl Merod { log_dir: environment.logs_dir.join(&name), binary: environment.merod_binary.clone(), name, + output_writer: environment.output_writer, } } @@ -92,7 +95,8 @@ impl Merod { let log_file = self.log_dir.join(format!("{}.log", log_suffix)); let mut log_file = File::create(&log_file).await?; - println!("Command: '{:}' {:?}", &self.binary, root_args); + self.output_writer + .write_string(format!("Command: '{:}' {:?}", &self.binary, root_args)); let mut child = Command::new(&self.binary) .args(root_args) diff --git a/e2e-tests/src/output.rs b/e2e-tests/src/output.rs new file mode 100644 index 000000000..08a8df1e7 --- /dev/null +++ b/e2e-tests/src/output.rs @@ -0,0 +1,65 @@ +use clap::ValueEnum; +use eyre::{Ok, Result as EyreResult}; +use serde::Serialize; + +#[derive(Clone, Copy, Debug)] +pub struct OutputWriter { + format: OutputFormat, +} + +#[derive(Clone, Copy, Debug, Default, ValueEnum)] +pub enum OutputFormat { + Markdown, + #[default] + PlainText, +} + +impl OutputWriter { + pub fn new(format: OutputFormat) -> Self { + Self { format } + } + + pub fn write_str(&self, line: &str) { + match self.format { + OutputFormat::Markdown => println!("{} ", line), + OutputFormat::PlainText => println!("{}", line), + } + } + + pub fn write_string(&self, line: String) { + match self.format { + OutputFormat::Markdown => println!("{} ", line), + OutputFormat::PlainText => println!("{}", line), + } + } + + pub fn write_header(&self, header: &str, level: usize) { + match self.format { + OutputFormat::Markdown => println!("{} {} ", "#".repeat(level), header), + OutputFormat::PlainText => { + println!( + "{}{}{}", + "-".repeat(level * 5), + header, + "-".repeat(level * 5), + ) + } + } + } + + pub fn write_json(&self, json: &T) -> EyreResult<()> + where + T: ?Sized + Serialize, + { + match self.format { + OutputFormat::Markdown => { + println!("```json\n{}\n```", serde_json::to_string_pretty(json)?); + } + OutputFormat::PlainText => { + println!("{}", serde_json::to_string(json)?); + } + } + + Ok(()) + } +} diff --git a/e2e-tests/src/steps/context_create.rs b/e2e-tests/src/steps/context_create.rs index 2f88de830..82ba9331c 100644 --- a/e2e-tests/src/steps/context_create.rs +++ b/e2e-tests/src/steps/context_create.rs @@ -30,7 +30,10 @@ impl Test for CreateContextStep { ctx.context_id = Some(context_id); ctx.inviter_public_key = Some(member_public_key); - println!("Report: Created context on '{}' node", &ctx.inviter); + ctx.output_writer.write_string(format!( + "Report: Created context on '{}' node", + &ctx.inviter + )); Ok(()) } diff --git a/e2e-tests/src/steps/context_invite_join.rs b/e2e-tests/src/steps/context_invite_join.rs index d76621154..912ca7d86 100644 --- a/e2e-tests/src/steps/context_invite_join.rs +++ b/e2e-tests/src/steps/context_invite_join.rs @@ -57,7 +57,8 @@ impl Test for InviteJoinContextStep { .insert(invitee.clone(), invitee_public_key), ); - println!("Report: Node '{}' joined the context", invitee) + ctx.output_writer + .write_string(format!("Report: Node '{}' joined the context", invitee)); } Ok(()) diff --git a/e2e-tests/src/steps/jsonrpc_call.rs b/e2e-tests/src/steps/jsonrpc_call.rs index 31a1c447d..89cb98bbb 100644 --- a/e2e-tests/src/steps/jsonrpc_call.rs +++ b/e2e-tests/src/steps/jsonrpc_call.rs @@ -75,7 +75,8 @@ impl Test for JsonRpcCallStep { } } - println!("Report: Call on '{}' node passed assertion", node) + ctx.output_writer + .write_string(format!("Report: Call on '{}' node passed assertion", node)); } Ok(())