Add target_os attribute to test framework

test/test-manager/src/config.rs

@@ -195,6 +195,16 @@ pub enum OsType {
     Macos,
 }
 
+impl From<OsType> for test_rpc::meta::Os {
+    fn from(ostype: OsType) -> Self {
+        match ostype {
+            OsType::Windows => Self::Windows,
+            OsType::Linux => Self::Linux,
+            OsType::Macos => Self::Macos,
+        }
+    }
+}
+
 #[derive(clap::ValueEnum, Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum PackageType {

test/test-manager/src/main.rs

@@ -208,15 +208,6 @@ async fn main() -> Result<()> {
             verbose,
             test_report,
         } => {
-            let summary_logger = match test_report {
-                Some(path) => Some(
-                    summary::SummaryLogger::new(&name, &path)
-                        .await
-                        .context("Failed to create summary logger")?,
-                ),
-                None => None,
-            };
-
             let mut config = config.clone();
             config.runtime_opts.display = match (display, vnc.is_some()) {
                 (false, false) => config::Display::None,
@@ -233,6 +224,19 @@ async fn main() -> Result<()> {
 
             let vm_config = vm::get_vm_config(&config, &name).context("Cannot get VM config")?;
 
+            let summary_logger = match test_report {
+                Some(path) => Some(
+                    summary::SummaryLogger::new(
+                        &name,
+                        test_rpc::meta::Os::from(vm_config.os_type),
+                        &path,
+                    )
+                    .await
+                    .context("Failed to create summary logger")?,
+                ),
+                None => None,
+            };
+
             let manifest = package::get_app_manifest(vm_config, current_app, previous_app)
                 .await
                 .context("Could not find the specified app packages")?;
@@ -273,6 +277,7 @@ async fn main() -> Result<()> {
                     host_bridge_name: crate::vm::network::macos::find_vm_bridge()?,
                     #[cfg(not(target_os = "macos"))]
                     host_bridge_name: crate::vm::network::linux::BRIDGE_NAME.to_owned(),
+                    os: test_rpc::meta::Os::from(vm_config.os_type),
                 },
                 &*instance,
                 &test_filters,

test/test-manager/src/run_tests.rs

@@ -1,5 +1,5 @@
 use crate::summary::{self, maybe_log_test_result};
-use crate::tests::TestContext;
+use crate::tests::{config::TEST_CONFIG, TestContext};
 use crate::{
     logging::{panic_as_string, TestOutput},
     mullvad_daemon, tests,
@@ -26,7 +26,7 @@ pub async fn run(
     mut summary_logger: Option<summary::SummaryLogger>,
 ) -> Result<()> {
     log::trace!("Setting test constants");
-    tests::config::TEST_CONFIG.init(config);
+    TEST_CONFIG.init(config);
 
     let pty_path = instance.get_pty();
 
@@ -47,7 +47,9 @@ pub async fn run(
     let mullvad_client =
         mullvad_daemon::new_rpc_client(connection_handle, mullvad_daemon_transport);
 
-    let mut tests: Vec<_> = inventory::iter::<tests::TestMetadata>().collect();
+    let mut tests: Vec<_> = inventory::iter::<tests::TestMetadata>()
+        .filter(|test| test.should_run_on_os(TEST_CONFIG.os))
+        .collect();
     tests.sort_by_key(|test| test.priority.unwrap_or(0));
 
     if !test_filters.is_empty() {

test/test-manager/src/summary.rs

@@ -1,4 +1,5 @@
 use std::{collections::BTreeMap, io, path::Path};
+use test_rpc::meta::Os;
 use tokio::{
     fs,
     io::{AsyncBufReadExt, AsyncWriteExt},
@@ -15,18 +16,24 @@ pub enum Error {
     Read(#[error(source)] io::Error),
     #[error(display = "Failed to parse log file")]
     Parse,
+    #[error(display = "Failed to serialize value")]
+    Serialize(#[error(source)] serde_json::Error),
+    #[error(display = "Failed to deserialize value")]
+    Deserialize(#[error(source)] serde_json::Error),
 }
 
 #[derive(Clone, Copy)]
 pub enum TestResult {
     Pass,
     Fail,
+    Skip,
     Unknown,
 }
 
 impl TestResult {
     const PASS_STR: &'static str = "✅";
     const FAIL_STR: &'static str = "❌";
+    const SKIP_STR: &'static str = "↪️";
     const UNKNOWN_STR: &'static str = " ";
 }
 
@@ -37,6 +44,7 @@ impl std::str::FromStr for TestResult {
         match s {
             TestResult::PASS_STR => Ok(TestResult::Pass),
             TestResult::FAIL_STR => Ok(TestResult::Fail),
+            TestResult::SKIP_STR => Ok(TestResult::Skip),
             _ => Ok(TestResult::Unknown),
         }
     }
@@ -47,6 +55,7 @@ impl std::fmt::Display for TestResult {
         match self {
             TestResult::Pass => f.write_str(TestResult::PASS_STR),
             TestResult::Fail => f.write_str(TestResult::FAIL_STR),
+            TestResult::Skip => f.write_str(TestResult::SKIP_STR),
             TestResult::Unknown => f.write_str(TestResult::UNKNOWN_STR),
         }
     }
@@ -60,7 +69,7 @@ pub struct SummaryLogger {
 impl SummaryLogger {
     /// Create a new logger and log to `path`. If `path` does not exist, it will be created. If it
     /// already exists, it is truncated and overwritten.
-    pub async fn new(name: &str, path: &Path) -> Result<SummaryLogger, Error> {
+    pub async fn new(name: &str, os: Os, path: &Path) -> Result<SummaryLogger, Error> {
         let mut file = fs::OpenOptions::new()
             .create(true)
             .write(true)
@@ -69,11 +78,14 @@ impl SummaryLogger {
             .await
             .map_err(|err| Error::Open(err, path.to_path_buf()))?;
 
-        // The first row is the summary name
         file.write_all(name.as_bytes())
             .await
             .map_err(Error::Write)?;
         file.write_u8(b'\n').await.map_err(Error::Write)?;
+        file.write_all(&serde_json::to_vec(&os).map_err(Error::Serialize)?)
+            .await
+            .map_err(Error::Write)?;
+        file.write_u8(b'\n').await.map_err(Error::Write)?;
 
         Ok(SummaryLogger { file })
     }
@@ -113,15 +125,18 @@ pub async fn maybe_log_test_result(
 
 /// Parsed summary results
 pub struct Summary {
-    /// Summary name
-    name: String,
+    /// Name of the configuration
+    config_name: String,
     /// Pairs of test names mapped to test results
     results: BTreeMap<String, TestResult>,
 }
 
 impl Summary {
     /// Read test summary from `path`.
-    pub async fn parse_log<P: AsRef<Path>>(path: P) -> Result<Summary, Error> {
+    pub async fn parse_log<P: AsRef<Path>>(
+        all_tests: &[&crate::tests::TestMetadata],
+        path: P,
+    ) -> Result<Summary, Error> {
         let file = fs::OpenOptions::new()
             .read(true)
             .open(&path)
@@ -130,11 +145,17 @@ impl Summary {
 
         let mut lines = tokio::io::BufReader::new(file).lines();
 
-        let name = lines
+        let config_name = lines
             .next_line()
             .await
             .map_err(Error::Read)?
             .ok_or(Error::Parse)?;
+        let os = lines
+            .next_line()
+            .await
+            .map_err(Error::Read)?
+            .ok_or(Error::Parse)?;
+        let os: Os = serde_json::from_str(&os).map_err(Error::Deserialize)?;
 
         let mut results = BTreeMap::new();
 
@@ -147,7 +168,20 @@ impl Summary {
             results.insert(test_name.to_owned(), test_result);
         }
 
-        Ok(Summary { name, results })
+        for test in all_tests {
+            // Add missing test results
+            let entry = results.entry(test.name.to_owned());
+            if test.should_run_on_os(os) {
+                entry.or_insert(TestResult::Unknown);
+            } else {
+                entry.or_insert(TestResult::Skip);
+            }
+        }
+
+        Ok(Summary {
+            config_name,
+            results,
+        })
     }
 
     // Return all tests which passed.
@@ -165,18 +199,18 @@ impl Summary {
 /// exist. If some log file which is expected to exist, but for any reason fails to
 /// be parsed, we should not abort the entire summarization.
 pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) {
-    let mut summaries = Vec::new();
-    let mut failed_to_parse = Vec::new();
+    // Collect test details
+    let tests: Vec<_> = inventory::iter::<crate::tests::TestMetadata>().collect();
+
+    let mut summaries = vec![];
+    let mut failed_to_parse = vec![];
     for sumfile in summary_files {
-        match Summary::parse_log(sumfile).await {
+        match Summary::parse_log(&tests, sumfile).await {
             Ok(summary) => summaries.push(summary),
             Err(_) => failed_to_parse.push(sumfile),
         }
     }
 
-    // Collect test details
-    let tests: Vec<_> = inventory::iter::<crate::tests::TestMetadata>().collect();
-
     // Print a table
     println!("<table>");
 
@@ -185,7 +219,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) {
     println!("<td style='text-align: center;'>Test ⬇️ / Platform ➡️ </td>");
 
     for summary in &summaries {
-        let total_tests = tests.len();
+        let total_tests = summary.results.len();
         let total_passed = summary.passed().len();
         let counter_text = if total_passed == total_tests {
             String::from(TestResult::PASS_STR)
@@ -194,7 +228,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) {
         };
         println!(
             "<td style='text-align: center;'>{} {}</td>",
-            summary.name, counter_text
+            summary.config_name, counter_text
         );
     }
 
@@ -203,15 +237,15 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) {
     println!("{}", {
         let oses_passed: Vec<_> = summaries
             .iter()
-            .filter(|summary| summary.passed().len() == tests.len())
+            .filter(|summary| summary.passed().len() == summary.results.len())
             .collect();
         if oses_passed.len() == summaries.len() {
             "🎉 All Platforms passed 🎉".to_string()
         } else {
             let failed: usize = summaries
                 .iter()
                 .map(|summary| {
-                    if summary.passed().len() == tests.len() {
+                    if summary.passed().len() == summary.results.len() {
                         0
                     } else {
                         1
@@ -246,9 +280,9 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) {
                 .unwrap_or(&TestResult::Unknown);
             match result {
                 TestResult::Fail | TestResult::Unknown => {
-                    failed_platforms.push(summary.name.clone())
+                    failed_platforms.push(summary.config_name.clone())
                 }
-                TestResult::Pass => (),
+                TestResult::Pass | TestResult::Skip => (),
             }
             println!("<td style='text-align: center;'>{}</td>", result);
         }
@@ -267,7 +301,7 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) {
         );
         println!("</td>");
 
-        // List the test name again (Useful for the summary accross the different platforms)
+        // List the test name again (Useful for the summary across the different platforms)
         println!("<td>{}</td>", test.name);
 
         // End row
@@ -279,4 +313,5 @@ pub async fn print_summary_table<P: AsRef<Path>>(summary_files: &[P]) {
     // Print explanation of test result
     println!("<p>{} = Test passed</p>", TestResult::PASS_STR);
     println!("<p>{} = Test failed</p>", TestResult::FAIL_STR);
+    println!("<p>{} = Test skipped</p>", TestResult::SKIP_STR);
 }

test/test-manager/src/tests/config.rs

@@ -1,5 +1,6 @@
 use once_cell::sync::OnceCell;
 use std::ops::Deref;
+use test_rpc::meta::Os;
 
 // Default `mullvad_host`. This should match the production env.
 pub const DEFAULT_MULLVAD_HOST: &str = "mullvad.net";
@@ -20,6 +21,8 @@ pub struct TestConfig {
     pub mullvad_host: String,
 
     pub host_bridge_name: String,
+
+    pub os: Os,
 }
 
 #[derive(Debug, Clone)]

test/test-manager/src/tests/install.rs

@@ -360,7 +360,7 @@ async fn replace_openvpn_cert(rpc: &ServiceClient) -> Result<(), Error> {
     const SOURCE_CERT_FILENAME: &str = "openvpn.ca.crt";
     const DEST_CERT_FILENAME: &str = "ca.crt";
 
-    let dest_dir = match rpc.get_os().await.expect("failed to get OS") {
+    let dest_dir = match TEST_CONFIG.os {
         Os::Windows => "C:\\Program Files\\Mullvad VPN\\resources",
         Os::Linux => "/opt/Mullvad VPN/resources",
         Os::Macos => "/Applications/Mullvad VPN.app/Contents/Resources",

test/test-manager/src/tests/test_metadata.rs

@@ -1,9 +1,11 @@
 use super::TestWrapperFunction;
+use test_rpc::meta::Os;
 use test_rpc::mullvad_daemon::MullvadClientVersion;
 
 pub struct TestMetadata {
     pub name: &'static str,
     pub command: &'static str,
+    pub target_os: Option<Os>,
     pub mullvad_client_version: MullvadClientVersion,
     pub func: TestWrapperFunction,
     pub priority: Option<i32>,
@@ -12,5 +14,13 @@ pub struct TestMetadata {
     pub cleanup: bool,
 }
 
+impl TestMetadata {
+    pub fn should_run_on_os(&self, os: Os) -> bool {
+        self.target_os
+            .map(|target_os| target_os == os)
+            .unwrap_or(true)
+    }
+}
+
 // Register our test metadata struct with inventory to allow submitting tests of this type.
 inventory::collect!(TestMetadata);

test/test-manager/src/tests/tunnel.rs

@@ -1,7 +1,7 @@
 use super::helpers::{
     self, connect_and_wait, disconnect_and_wait, set_bridge_settings, set_relay_settings,
 };
-use super::{Error, TestContext};
+use super::{config::TEST_CONFIG, Error, TestContext};
 
 use crate::network_monitor::{start_packet_monitor, MonitorOptions};
 use mullvad_management_interface::{types, ManagementServiceClient};
@@ -502,7 +502,7 @@ async fn check_tunnel_psk(
     mullvad_client: &ManagementServiceClient,
     should_have_psk: bool,
 ) {
-    match rpc.get_os().await.expect("failed to get OS") {
+    match TEST_CONFIG.os {
         Os::Linux => {
             let name = helpers::get_tunnel_interface(mullvad_client.clone())
                 .await

test/test-manager/src/tests/ui.rs

@@ -32,7 +32,7 @@ pub async fn run_test_env<
     let new_params: Vec<String>;
     let bin_path;
 
-    match rpc.get_os().await? {
+    match TEST_CONFIG.os {
         Os::Linux => {
             bin_path = PathBuf::from("/usr/bin/xvfb-run");