Fail fast for integration tests (#1827)

* Fail fast for integration tests

- Integration tesing should stop whenever there is a failure
- Add ability to create iml diagnostics with a provisioning script
- Add pub key to authorized keys for diagnostics upload
- Create diagnostic scripts in parallel on failure

Signed-off-by: johnsonw <[email protected]>

* avoid use of mod

Signed-off-by: Joe Grund <[email protected]>

* reduce SOS reporting to a single trait.

Signed-off-by: Joe Grund <[email protected]>

* - Remove hasher

Signed-off-by: johnsonw <[email protected]>

* - updates
- Add scp functionality
- Scp diagnostics from remote machines to /tmp on host

Signed-off-by: johnsonw <[email protected]>

Co-authored-by: Joe Grund <[email protected]>

iml-cmd/src/lib.rs

@@ -104,11 +104,15 @@ impl CheckedCommandExt for Command {
 }
 
 pub trait CheckedChildExt {
-    fn wait_with_checked_output(self) -> Pin<Box<dyn Future<Output = Result<Output, CmdError>>>>;
+    fn wait_with_checked_output(
+        self,
+    ) -> Pin<Box<dyn Future<Output = Result<Output, CmdError>> + Send>>;
 }
 
 impl CheckedChildExt for Child {
-    fn wait_with_checked_output(self) -> Pin<Box<dyn Future<Output = Result<Output, CmdError>>>> {
+    fn wait_with_checked_output(
+        self,
+    ) -> Pin<Box<dyn Future<Output = Result<Output, CmdError>> + Send>> {
         tracing::debug!("Child waiting for output: {:?}", self);
 
         self.wait_with_output()

iml-system-docker-tests/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.2.0"
 authors = ["IML Team <[email protected]>"]
 edition = "2018"
 
-[dev-dependencies]
+[dependencies]
 iml-cmd = { version = "0.2.0", path = "../iml-cmd" }
 iml-system-test-utils = { version = "0.2.0", path = "../iml-system-test-utils" }
 iml-systemd = { version = "0.2.0", path = "../iml-systemd" }

iml-system-docker-tests/src/lib.rs

@@ -0,0 +1,59 @@
+// Copyright (c) 2020 DDN. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+use iml_cmd::CmdError;
+use iml_system_test_utils::{docker, iml, ssh, vagrant, SetupConfigType};
+use iml_systemd::SystemdError;
+use std::{collections::HashMap, time::Duration};
+use tokio::time::delay_for;
+
+pub async fn setup() -> Result<(), SystemdError> {
+    // remove the stack if it is running and clean up volumes and network
+    docker::remove_iml_stack().await?;
+    docker::system_prune().await?;
+    docker::volume_prune().await?;
+    docker::configure_docker_overrides().await?;
+    docker::stop_swarm().await?;
+    docker::remove_password().await?;
+
+    docker::start_swarm().await?;
+    docker::set_password().await?;
+
+    // Destroy any vagrant nodes that are currently running
+    vagrant::destroy().await?;
+    vagrant::global_prune().await?;
+    vagrant::poweroff_running_vms().await?;
+    vagrant::unregister_vms().await?;
+    vagrant::clear_vbox_machine_folder().await?;
+
+    Ok(())
+}
+
+pub async fn run_fs_test<S: std::hash::BuildHasher>(
+    config: &vagrant::ClusterConfig,
+    docker_setup: &SetupConfigType,
+    server_map: HashMap<String, &[&str], S>,
+    fs_type: vagrant::FsType,
+) -> Result<(), SystemdError> {
+    setup().await?;
+    docker::configure_docker_setup(&docker_setup).await?;
+
+    docker::deploy_iml_stack().await?;
+
+    vagrant::setup_deploy_docker_servers(&config, server_map).await?;
+
+    vagrant::create_fs(fs_type, &config).await?;
+
+    delay_for(Duration::from_secs(30)).await;
+
+    iml::detect_fs().await?;
+
+    Ok(())
+}
+
+pub async fn wait_for_ntp(config: &vagrant::ClusterConfig) -> Result<(), CmdError> {
+    ssh::wait_for_ntp(&config.storage_server_ips()).await?;
+
+    Ok(())
+}

iml-system-docker-tests/tests/ldiskfs_test.rs

@@ -0,0 +1,36 @@
+// Copyright (c) 2020 DDN. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+use iml_system_test_utils::{vagrant, SetupConfig, SetupConfigType, SystemTestError, WithSos as _};
+use std::collections::HashMap;
+use iml_system_docker_tests::{run_fs_test, wait_for_ntp};
+
+async fn run_test(config: &vagrant::ClusterConfig) -> Result<(), SystemTestError> {
+    run_fs_test(
+        &config,
+        &SetupConfigType::DockerSetup(SetupConfig {
+            use_stratagem: false,
+            branding: iml_wire_types::Branding::Whamcloud,
+        }),
+        vec![("base_monitored".into(), &config.storage_servers()[..])]
+            .into_iter()
+            .collect::<HashMap<String, &[&str]>>(),
+        vagrant::FsType::LDISKFS,
+    )
+    .await?;
+
+    wait_for_ntp(&config).await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_docker_ldiskfs_setup() -> Result<(), SystemTestError> {
+    let config = vagrant::ClusterConfig::default();
+
+    run_test(&config)
+        .await
+        .handle_test_result(&config.storage_server_ips()[..], "docker_ldiskfs_test")
+        .await
+}

iml-system-docker-tests/tests/stratagem_test.rs

@@ -0,0 +1,40 @@
+// Copyright (c) 2020 DDN. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+use iml_system_docker_tests::{run_fs_test, wait_for_ntp};
+use iml_system_test_utils::{vagrant, SetupConfig, SetupConfigType, SystemTestError, WithSos as _};
+use std::collections::HashMap;
+
+async fn run_test(config: &vagrant::ClusterConfig) -> Result<(), SystemTestError> {
+    run_fs_test(
+        &config,
+        &SetupConfigType::DockerSetup(SetupConfig {
+            use_stratagem: true,
+            branding: iml_wire_types::Branding::DdnAi400,
+        }),
+        vec![
+            ("stratagem_server".into(), &config.mds_servers()[..]),
+            ("base_monitored".into(), &config.oss_servers()[..]),
+            ("stratagem_client".into(), &config.client_servers()[..]),
+        ]
+        .into_iter()
+        .collect::<HashMap<String, &[&str]>>(),
+        vagrant::FsType::LDISKFS,
+    )
+    .await?;
+
+    wait_for_ntp(&config).await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_docker_stratagem_setup() -> Result<(), SystemTestError> {
+    let config = vagrant::ClusterConfig::default();
+
+    run_test(&config)
+        .await
+        .handle_test_result(&config.storage_server_ips()[..], "docker_stratagem_test")
+        .await
+}