From af7b839034ea2a9a602a16174d63673191100a79 Mon Sep 17 00:00:00 2001 From: Konstantin Yarovoy Date: Mon, 8 Apr 2024 08:19:31 +0000 Subject: [PATCH] spec: remove backticks and use new logging REF: #1495 Replace all backtick command execution with ShellCmd module. Use Log instead of LOGGING module. Fix edgecase issues caused by logging refactor. Remove commented-out code. Signed-off-by: Konstantin Yarovoy --- spec/5g/core_spec.cr | 57 +- spec/5g/ran_spec.cr | 46 +- spec/airgap_task_spec.cr | 72 ++- spec/cluster_setup_spec.cr | 27 +- .../cnf_testsuite_config_lifecycle_spec.cr | 32 +- .../cnf_testsuite_container_chaos_spec.cr | 22 - .../cnf_testsuite_microservice_spec.cr | 30 +- .../cnf_testsuite_network_chaos_spec.cr | 24 - spec/cnf_testsuite_all/cnf_testsuite_spec.cr | 49 +- spec/curl_install_spec.cr | 24 +- spec/platform/cluster_api_spec.cr | 28 +- spec/platform/hardware_and_scheduler_spec.cr | 15 +- spec/platform/observability_spec.cr | 52 +- spec/platform/platform_spec.cr | 47 +- spec/platform/resilience_spec.cr | 14 +- spec/platform/security_spec.cr | 43 +- spec/prereqs_spec.cr | 28 +- spec/setup_spec.cr | 109 ++-- spec/spec_helper.cr | 18 +- spec/utils/cnf_manager_spec.cr | 132 ++--- spec/utils/k8s_instrumentation_spec.cr | 4 +- spec/utils/kubescape_spec.cr | 7 +- spec/utils/utils_spec.cr | 110 ++-- spec/workload/compatibility_spec.cr | 23 +- spec/workload/configuration_spec.cr | 509 ++++++++---------- spec/workload/cpu_hog_spec.cr | 34 +- spec/workload/installability_spec.cr | 86 ++- spec/workload/microservice_spec.cr | 271 +++++----- spec/workload/observability_spec.cr | 194 +++---- spec/workload/operator_spec.cr | 28 +- spec/workload/registry_spec.cr | 59 +- .../resilience/container_chaos_spec.cr | 50 +- spec/workload/resilience/disk_fill_spec.cr | 25 +- .../workload/resilience/network_chaos_spec.cr | 36 +- spec/workload/resilience/node_drain_spec.cr | 27 +- spec/workload/resilience/pod_delete_spec.cr | 25 +- .../workload/resilience/pod_dns_error_spec.cr | 27 +- .../workload/resilience/pod_io_stress_spec.cr | 25 +- .../resilience/pod_memory_hog_spec.cr | 25 +- .../resilience/pod_network_corruption_spec.cr | 25 +- .../pod_network_duplication_spec.cr | 25 +- .../resilience/pod_network_latency_spec.cr | 25 +- spec/workload/scalability_spec.cr | 27 +- spec/workload/security_spec.cr | 407 +++++++------- spec/workload/state_spec.cr | 113 ++-- src/tasks/utils/utils.cr | 19 +- 46 files changed, 1261 insertions(+), 1814 deletions(-) diff --git a/spec/5g/core_spec.cr b/spec/5g/core_spec.cr index 946289a24..a51a2133b 100644 --- a/spec/5g/core_spec.cr +++ b/spec/5g/core_spec.cr @@ -8,58 +8,55 @@ require "sam" describe "Core" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true end + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true + end it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"] do begin - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true - response_s = `./cnf-testsuite smf_upf_heartbeat verbose` - Log.info {"response: #{response_s}"} - (/PASSED: Chaos service degradation is less than 50%/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose") + (/PASSED: Chaos service degradation is less than 50%/ =~ result[:output]).should_not be_nil ensure - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"] do begin - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true - response_s = `./cnf-testsuite smf_upf_heartbeat verbose baseline_count=300` - Log.info {"response: #{response_s}"} - (/FAILED: Chaos service degradation is more than 50%/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose baseline_count=300") + (/FAILED: Chaos service degradation is more than 50%/ =~ result[:output]).should_not be_nil ensure - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"] do begin - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true - response_s = `./cnf-testsuite suci_enabled verbose` - Log.info {"response: #{response_s}"} - (/PASSED: Core uses SUCI 5g authentication/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("suci_enabled verbose") + (/PASSED: Core uses SUCI 5g authentication/ =~ result[:output]).should_not be_nil ensure - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'suci_enabled' should fail if the 5G core does not have suci enabled", tags: ["5g"] do begin - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml`} - $?.success?.should be_true - response_s = `./cnf-testsuite suci_enabled verbose` - Log.info {"response: #{response_s}"} - (/FAILED: Core does not use SUCI 5g authentication/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("suci_enabled verbose") + (/FAILED: Core does not use SUCI 5g authentication/ =~ result[:output]).should_not be_nil ensure - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml`} - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") + result[:status].success?.should be_true end end diff --git a/spec/5g/ran_spec.cr b/spec/5g/ran_spec.cr index 9eb48debe..b8eb66efc 100644 --- a/spec/5g/ran_spec.cr +++ b/spec/5g/ran_spec.cr @@ -8,42 +8,40 @@ require "sam" describe "5g" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'oran_e2_connection' should pass if the ORAN enabled RAN connects to the RIC using the e2 standard", tags: ["oran"] do begin - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml`} - $?.success?.should be_true - response_s = `./cnf-testsuite oran_e2_connection verbose` - Log.info {"response: #{response_s}"} - (/PASSED: RAN connects to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("oran_e2_connection verbose") + (/PASSED: RAN connects to a RIC using the e2 standard interface/ =~ result[:output]).should_not be_nil ensure - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml`} - $?.success?.should be_true - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'oran_e2_connection' should fail if the ORAN enabled RAN does not connect to the RIC using the e2 standard", tags: ["oran"] do begin - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true - Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml`} - $?.success?.should be_true - response_s = `./cnf-testsuite oran_e2_connection verbose` - Log.info {"response: #{response_s}"} - (/FAILED: RAN does not connect to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("oran_e2_connection verbose") + (/FAILED: RAN does not connect to a RIC using the e2 standard interface/ =~ result[:output]).should_not be_nil ensure - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} - $?.success?.should be_true - Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml`} - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") + result[:status].success?.should be_true end end diff --git a/spec/airgap_task_spec.cr b/spec/airgap_task_spec.cr index 3f6c724e3..6f9f03fd9 100644 --- a/spec/airgap_task_spec.cr +++ b/spec/airgap_task_spec.cr @@ -1,6 +1,6 @@ require "./spec_helper" require "colorize" -# require "../src/tasks/utils/utils.cr" +require "../src/tasks/utils/utils.cr" require "file_utils" require "sam" @@ -8,8 +8,7 @@ describe "AirGap" do before_all do Helm.helm_repo_add("chaos-mesh", "https://charts.chaos-mesh.org") - # AirGap::LOGGING.info `./cnf-testsuite airgapped output-file=/tmp/airgapped.tar.gz` unless File.exists?("/tmp/airgapped.tar.gz") - Log.info { `./cnf-testsuite airgapped output-file=/tmp/airgapped.tar.gz` } + result = ShellCmd.run_testsuite("airgapped output-file=/tmp/airgapped.tar.gz") (File.exists?("/tmp/airgapped.tar.gz")).should be_true end @@ -19,7 +18,7 @@ describe "AirGap" do end it "'setup' task should create an airgapped tarball with the necessary files", tags: ["airgap-setup"] do - file_list = `tar -tvf /tmp/airgapped.tar.gz` + file_list = ShellCmd.run("tar -tvf /tmp/airgapped.tar.gz")[:output] Log.info { "file_list: #{file_list}" } (file_list).match(/kubectl.tar/).should_not be_nil (file_list).match(/chaos-mesh.tar/).should_not be_nil @@ -36,9 +35,8 @@ describe "AirGap" do end it "'setup' task should install the necessary cri tools in the cluster", tags: ["airgap-setup"] do - response_s = `./cnf-testsuite -l info setup offline=/tmp/airgapped.tar.gz` - Log.info { response_s } - $?.success?.should be_true + result = ShellCmd.run_testsuite("-l info setup offline=/tmp/airgapped.tar.gz") + result[:status].success?.should be_true pods = KubectlClient::Get.pods_by_nodes(KubectlClient::Get.schedulable_nodes_list) pods = KubectlClient::Get.pods_by_label(pods, "name", "cri-tools") # Get the generated name of the cri-tools per node @@ -49,61 +47,51 @@ describe "AirGap" do sh = KubectlClient.exec("-ti #{pod_name} -- cat /usr/local/bin/ctr > /dev/null") sh[:status].success? end - (/All prerequisites found./ =~ response_s).should_not be_nil - (/Setup complete/ =~ response_s).should_not be_nil + (/All prerequisites found./ =~ result[:output]).should_not be_nil + (/Setup complete/ =~ result[:output]).should_not be_nil end it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf helm chart in airgapped mode", tags: ["airgap-repo"] do begin - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz` - Log.info { response_s } - file_list = `tar -tvf /tmp/airgapped.tar.gz` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz") + file_list = ShellCmd.run("tar -tvf /tmp/airgapped.tar.gz")[:output] Log.info { "file_list: #{file_list}" } (file_list).match(/coredns_1.8.0.tar/).should_not be_nil (file_list).match(/coredns_1.6.7.tar/).should_not be_nil - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz` - $?.success?.should be_true - Log.info { response_s } - (/Successfully setup coredns/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz") + result[:status].success?.should be_true + (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml wait_count=0` - $?.success?.should be_true - Log.info { response_s } - (/Successfully cleaned up/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml wait_count=0") + result[:status].success?.should be_true + (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf helm directory in airgapped mode", tags: ["airgap-directory"] do begin - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz` - Log.info { response_s } - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz` - Log.info { response_s } - $?.success?.should be_true - (/Successfully setup coredns/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz") + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz") + result[:status].success?.should be_true + (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml wait_count=0` - $?.success?.should be_true - Log.info { response_s } - (/Successfully cleaned up/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml wait_count=0") + result[:status].success?.should be_true + (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf manifest directory in airgapped mode", tags: ["airgap-manifest"] do begin - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz` - Log.info { response_s } - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz` - Log.info { response_s } - $?.success?.should be_true - Log.info { response_s } - (/Successfully setup nginx-webapp/ =~ response_s).should_not be_nil - (/exported_chart\" not found/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz") + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz") + result[:status].success?.should be_true + (/Successfully setup nginx-webapp/ =~ result[:output]).should_not be_nil + (/exported_chart\" not found/ =~ result[:output]).should be_nil ensure - response_s = `LOG_LEVEL=debug ./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml wait_count=0` - $?.success?.should be_true - Log.info { response_s } - (/Successfully cleaned up/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml wait_count=0", cmd_prefix: "LOG_LEVEL=debug") + result[:status].success?.should be_true + (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end diff --git a/spec/cluster_setup_spec.cr b/spec/cluster_setup_spec.cr index 4a232e698..cc2bef4ef 100644 --- a/spec/cluster_setup_spec.cr +++ b/spec/cluster_setup_spec.cr @@ -7,34 +7,31 @@ require "cluster_tools" describe "Cluster Setup" do before_each do - `./cnf-testsuite cleanup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cleanup") + result[:status].success?.should be_true end it "'install_cluster_tools' should give a message if namespace does not exist", tags: ["cluster_setup"] do KubectlClient::Delete.command("namespace #{ClusterTools.namespace}") - response_s = `./cnf-testsuite install_cluster_tools` - LOGGING.info response_s - $?.success?.should be_false - (/please run cnf-testsuite setup/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("install_cluster_tools") + result[:status].success?.should be_false + (/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil end it "'install_cluster_tools' should give a message if namespace does not exist even after setup", tags: ["cluster_setup"] do - `./cnf-testsuite setup` + result = ShellCmd.run_testsuite("setup") KubectlClient::Delete.command("namespace #{ClusterTools.namespace}") - response_s = `./cnf-testsuite install_cluster_tools` - LOGGING.info response_s - $?.success?.should be_false - (/please run cnf-testsuite setup/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("install_cluster_tools") + result[:status].success?.should be_false + (/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil end it "'uninstall_cluster_tools' should give a message if namespace does not exist", tags: ["cluster_setup"] do - response_s = `./cnf-testsuite uninstall_cluster_tools` - LOGGING.info response_s - $?.success?.should be_false - (/please run cnf-testsuite setup/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("uninstall_cluster_tools") + result[:status].success?.should be_false + (/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil end end diff --git a/spec/cnf_testsuite_all/cnf_testsuite_config_lifecycle_spec.cr b/spec/cnf_testsuite_all/cnf_testsuite_config_lifecycle_spec.cr index c838a27b9..5c958ac34 100644 --- a/spec/cnf_testsuite_all/cnf_testsuite_config_lifecycle_spec.cr +++ b/spec/cnf_testsuite_all/cnf_testsuite_config_lifecycle_spec.cr @@ -1,29 +1,7 @@ #TODO Update Test -# require "../spec_helper" -# require "../../src/tasks/utils/utils.cr" -# require "colorize" +require "../spec_helper" +require "../../src/tasks/utils/utils.cr" +require "colorize" -# describe CnfTestSuite do -# before_all do -# `./cnf-testsuite setup` -# $?.success?.should be_true -# end - -# after_all do -# `./cnf-testsuite samples_cleanup` -# $?.success?.should be_true -# end - -# it "'testsuite all' should run the configuration lifecycle tests", tags: ["testsuite-config-lifecycle"] do -# `./cnf-testsuite samples_cleanup` -# response_s = `./cnf-testsuite all ~reasonable_startup_time ~reasonable_image_size ~disk_fill ~pod_delete ~pod_io_stress ~pod_network_latency ~pod_network_corruption ~pod_network_duplication ~pod_memory_hog ~chaos_network_loss ~node_drain ~pod_dns_error ~chaos_cpu_hog ~chaos_container_kill ~platform ~volume_hostpath_not_found ~privileged ~increase_capacity ~decrease_capacity ~helm_chart_valid ~helm_chart_published "cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml" verbose` -# LOGGING.info response_s -# (/PASSED: Helm readiness probe found/ =~ response_s).should_not be_nil -# (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil -# (/Final workload score:/ =~ response_s).should_not be_nil -# (/Final score:/ =~ response_s).should_not be_nil -# (CNFManager::Points.all_result_test_names(CNFManager::Points.final_cnf_results_yml).sort).should eq(["ip_addresses", "liveness", "readiness", "rolling_update", "rolling_downgrade", "rolling_version_change", "nodeport_not_used", "hostport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "rollback", "secrets_used", "immutable_configmap"].sort) -# (/^.*\.cr:[0-9].*/ =~ response_s).should be_nil -# $?.success?.should be_true -# end -# end +describe CnfTestSuite do +end diff --git a/spec/cnf_testsuite_all/cnf_testsuite_container_chaos_spec.cr b/spec/cnf_testsuite_all/cnf_testsuite_container_chaos_spec.cr index 6a8b569a6..1e0477c57 100644 --- a/spec/cnf_testsuite_all/cnf_testsuite_container_chaos_spec.cr +++ b/spec/cnf_testsuite_all/cnf_testsuite_container_chaos_spec.cr @@ -3,26 +3,4 @@ require "../../src/tasks/utils/utils.cr" require "colorize" describe "CNF Test Suite all Container Chaos" do - # before_all do - # `./cnf-testsuite setup` - # $?.success?.should be_true - # end - - # after_all do - # `./cnf-testsuite samples_cleanup` - # $?.success?.should be_true - # end - - # it "'all ~platform ~compatibilty ~state ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_and_scheduling ~microservice ~chaos_network_loss' should run the chaos tests" do - # `./cnf-testsuite samples_cleanup` - # response_s = `./cnf-testsuite all ~platform ~compatibilty ~state ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_and_scheduling ~microservice ~chaos_network_loss cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false verbose` - # LOGGING.info response_s - # (/Final workload score:/ =~ response_s).should_not be_nil - # (/Final score:/ =~ response_s).should_not be_nil - # (CNFManager::Points.all_result_test_names(CNFManager.final_cnf_results_yml)).should eq([ "chaos_cpu_hog", "chaos_container_kill"]) - # $?.success?.should be_true - # ensure - # LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ` - # end - end diff --git a/spec/cnf_testsuite_all/cnf_testsuite_microservice_spec.cr b/spec/cnf_testsuite_all/cnf_testsuite_microservice_spec.cr index 23937af4c..5c958ac34 100644 --- a/spec/cnf_testsuite_all/cnf_testsuite_microservice_spec.cr +++ b/spec/cnf_testsuite_all/cnf_testsuite_microservice_spec.cr @@ -1,27 +1,7 @@ #TODO Update Test -# require "../spec_helper" -# require "../../src/tasks/utils/utils.cr" -# require "colorize" +require "../spec_helper" +require "../../src/tasks/utils/utils.cr" +require "colorize" -# describe CnfTestSuite do -# before_all do -# `./cnf-testsuite setup` -# $?.success?.should be_true -# end - -# after_all do -# `./cnf-testsuite samples_cleanup` -# $?.success?.should be_true -# end - -# it "'testsuite all' should run all the microservice tests", tags: ["testsuite-microservice"] do -# `./cnf-testsuite samples_cleanup` -# response_s = `./cnf-testsuite all ~disk_fill ~pod_delete ~pod_network_latency ~pod_network_corruption ~pod_network_duplication ~pod_io_stress ~pod_memory_hog ~node_drain ~pod_dns_error ~chaos_network_loss ~chaos_cpu_hog ~chaos_container_kill ~platform ~volume_hostpath_not_found ~privileged ~increase_capacity ~decrease_capacity ~ip_addresses ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~helm_chart_valid ~helm_chart_published ~rollback ~secrets_used ~immutable_configmap "cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml" verbose` -# LOGGING.info response_s -# (/Final workload score:/ =~ response_s).should_not be_nil -# (/Final score:/ =~ response_s).should_not be_nil -# (CNFManager::Points.all_result_test_names(CNFManager::Points.final_cnf_results_yml).sort).should eq(["reasonable_startup_time", "reasonable_image_size"].sort) -# (/^.*\.cr:[0-9].*/ =~ response_s).should be_nil -# $?.success?.should be_true -# end -# end +describe CnfTestSuite do +end diff --git a/spec/cnf_testsuite_all/cnf_testsuite_network_chaos_spec.cr b/spec/cnf_testsuite_all/cnf_testsuite_network_chaos_spec.cr index f88afc3e3..9e5796be2 100644 --- a/spec/cnf_testsuite_all/cnf_testsuite_network_chaos_spec.cr +++ b/spec/cnf_testsuite_all/cnf_testsuite_network_chaos_spec.cr @@ -3,28 +3,4 @@ require "../../src/tasks/utils/utils.cr" require "colorize" describe "CNF Test Suite all Network Chaos" do - # before_all do - # `./cnf-testsuite setup` - # $?.success?.should be_true - # end - - # after_all do - # `./cnf-testsuite samples_cleanup` - # $?.success?.should be_true - # end - - # it "'all' should run the whole test suite" do - # `./cnf-testsuite samples_cleanup` - - # response_s = `./cnf-testsuite all ~platform ~compatibilty ~state ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_and_scheduling ~microservice ~chaos_cpu_hog ~chaos_container_kill cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false verbose` - # response_s = `./cnf-testsuite all ~platform ~compatibilty ~state ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_and_scheduling ~microservice ~chaos_network_loss ~chaos_cpu_hog ~chaos_container_kill cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false verbose` - # LOGGING.info response_s - # (/Final workload score:/ =~ response_s).should_not be_nil - # (/Final score:/ =~ response_s).should_not be_nil - # (CNFManager::Points.all_result_test_names(CNFManager.final_cnf_results_yml)).should eq([ "chaos_network_loss"]) - # $?.success?.should be_true - # ensure - # LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ` - - # end end diff --git a/spec/cnf_testsuite_all/cnf_testsuite_spec.cr b/spec/cnf_testsuite_all/cnf_testsuite_spec.cr index db736c741..f110faa96 100644 --- a/spec/cnf_testsuite_all/cnf_testsuite_spec.cr +++ b/spec/cnf_testsuite_all/cnf_testsuite_spec.cr @@ -4,56 +4,19 @@ require "colorize" describe CnfTestSuite do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end after_all do - `./cnf-testsuite samples_cleanup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("samples_cleanup") + result[:status].success?.should be_true end - # Test is out of date - # it "'all' should run the workloads test suite", tags: ["testsuite-all"] do - # `./cnf-testsuite samples_cleanup` - # # the workload resilience tests are run in the chaos specs - # # the ommisions (i.e. ~resilience) are done for performance reasons for the spec suite - # # response_s = `./cnf-testsuite all ~platform ~resilience cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose` - # response_s = `./cnf-testsuite all ~disk_fill ~pod_delete ~pod_network_latency ~pod_io_stress ~pod_network_duplication ~pod_network_corruption ~pod_memory_hog ~node_drain ~pod_dns_error ~chaos_network_loss ~chaos_cpu_hog ~chaos_container_kill ~platform ~ip_addresses ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~rollback ~secrets_used ~immutable_configmap ~reasonable_startup_time ~reasonable_image_size "cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml" verbose` - # LOGGING.info response_s - # (/Lint Passed/ =~ response_s).should_not be_nil - # (/PASSED: Replicas increased to 3/ =~ response_s).should_not be_nil - # (/PASSED: Replicas decreased to 1/ =~ response_s).should_not be_nil - # (/PASSED: Published Helm Chart Found/ =~ response_s).should_not be_nil - # (/Final workload score:/ =~ response_s).should_not be_nil - # (/Final score:/ =~ response_s).should_not be_nil - # (CNFManager::Points.all_result_test_names(CNFManager::Points.final_cnf_results_yml).sort).should eq(["volume_hostpath_not_found", "privileged", "increase_capacity", "decrease_capacity", "helm_chart_valid", "helm_chart_published"].sort) - # (/^.*\.cr:[0-9].*/ =~ response_s).should be_nil - # $?.success?.should be_true - # end - - # todo add mock test for testing require tests (privilege mode is no longer required) - # it "'workload' should fail with an exit code when a required cnf fails", tags: ["security"] do - # begin - # LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose wait_count=0` - # $?.success?.should be_true - # response_s = `./cnf-testsuite workload ~automatic_cnf_install ~ensure_cnf_installed ~configuration_file_setup ~compatibility ~state ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_and_scheduling ~microservice ~resilience` - # LOGGING.info response_s - # $?.success?.should be_false - # (/Found.*privileged containers.*/ =~ response_s).should_not be_nil - # (/Privileged container (privileged-coredns) in.*/ =~ response_s).should_not be_nil - # response_s = `./cnf-testsuite privileged strict` - # $?.success?.should be_false - # ensure - # LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml` - # end - # end - it "a task should fail with an exit code of 2 when there is an exception", tags: ["security"] do begin - response_s = `./cnf-testsuite divide_by_zero strict` - LOGGING.info response_s - ($?.exit_code == 2).should be_true + result = ShellCmd.run_testsuite("divide_by_zero strict") + (result[:status].exit_code == 2).should be_true end end end diff --git a/spec/curl_install_spec.cr b/spec/curl_install_spec.cr index 70008186c..cd5d7603d 100644 --- a/spec/curl_install_spec.cr +++ b/spec/curl_install_spec.cr @@ -7,25 +7,23 @@ require "sam" describe "CurlInstall" do after_all do - LOGGING.info "Curl install tests finished. Building ./cnf-testsuite again".colorize(:green) - `crystal build src/cnf-testsuite.cr` - if $?.success? - LOGGING.info "Build Success!".colorize(:green) + Log.info { "Curl install tests finished. Building ./cnf-testsuite again".colorize(:green) } + result = ShellCmd.run("crystal build src/cnf-testsuite.cr") + if result[:status].success? + Log.info { "Build Success!".colorize(:green) } else - LOGGING.info "crystal build failed!".colorize(:red) + Log.info { "crystal build failed!".colorize(:red) } raise "crystal build failed! curl_install_spec.cr" end end it "'source curl_install.sh' should download a cnf-testsuite binary", tags: ["curl"] do - response_s = `/bin/bash -c "source ./curl_install.sh"` - LOGGING.info response_s - $?.success?.should be_true - (/cnf-testsuite/ =~ response_s).should_not be_nil + result = ShellCmd.run("/bin/bash -c 'source ./curl_install.sh'", force_output: true) + result[:status].success?.should be_true + (/cnf-testsuite/ =~ result[:output]).should_not be_nil end it "'curl_install.sh' should download a cnf-testsuite binary", tags: ["curl"] do - response_s = `./curl_install.sh` - LOGGING.info response_s - $?.success?.should be_true - (/To use the cnf-testsuite please restart you terminal session to load the new PATH/ =~ response_s).should_not be_nil + result = ShellCmd.run("./curl_install.sh", force_output: true) + result[:status].success?.should be_true + (/To use the cnf-testsuite please restart you terminal session to load the new PATH/ =~ result[:output]).should_not be_nil end end diff --git a/spec/platform/cluster_api_spec.cr b/spec/platform/cluster_api_spec.cr index fae506cb1..9db6eefa0 100644 --- a/spec/platform/cluster_api_spec.cr +++ b/spec/platform/cluster_api_spec.cr @@ -1,42 +1,38 @@ require "./../spec_helper" require "colorize" require "file_utils" -require "./../../src/tasks/utils/utils.cr" +require "../../src/tasks/utils/utils.cr" describe "Cluster API" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end after_each do - response_s = `./cnf-testsuite cluster_api_cleanup` - $?.success?.should be_true - Log.info { "#{response_s}" } + result = ShellCmd.run_testsuite("cluster_api_cleanup") + result[:status].success?.should be_true end it "'clusterapi_enabled' should pass if cluster api is installed", tags: ["cluster-api"] do begin - LOGGING.info `./cnf-testsuite cluster_api_setup` + result = ShellCmd.run_testsuite("cluster_api_setup") current_dir = FileUtils.pwd FileUtils.cd("#{current_dir}") - response_s = `./cnf-testsuite clusterapi_enabled poc` - LOGGING.info response_s - (/Cluster API is enabled/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("clusterapi_enabled poc") + (/Cluster API is enabled/ =~ result[:output]).should_not be_nil ensure Log.info { "Running Cleanup" } - `./cnf-testsuite cluster_api_cleanup` + result = ShellCmd.run_testsuite("cluster_api_cleanup") end end it "'clusterapi_enabled' should fail if cluster api is not installed", tags: ["cluster-api-fail"] do begin - response_s = `./cnf-testsuite clusterapi_enabled poc` - LOGGING.info response_s - (/Cluster API NOT enabled/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("clusterapi_enabled poc") + (/Cluster API NOT enabled/ =~ result[:output]).should_not be_nil ensure Log.info { "Running Cleanup" } - `./cnf-testsuite cluster_api_cleanup` + result = ShellCmd.run_testsuite("cluster_api_cleanup") end end end - diff --git a/spec/platform/hardware_and_scheduler_spec.cr b/spec/platform/hardware_and_scheduler_spec.cr index a565ebf4c..8f26fb3fc 100644 --- a/spec/platform/hardware_and_scheduler_spec.cr +++ b/spec/platform/hardware_and_scheduler_spec.cr @@ -4,18 +4,15 @@ require "./../../src/tasks/utils/utils.cr" describe "Platform" do before_all do - # LOGGING.debug `pwd` - # LOGGING.debug `echo $KUBECONFIG` - `./cnf-testsuite samples_cleanup` - $?.success?.should be_true - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("samples_cleanup") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'oci_compliant' should pass if all runtimes are oci_compliant", tags: ["platform:oci_compliant"] do - response_s = `./cnf-testsuite platform:oci_compliant` - LOGGING.info response_s - (/(PASSED){1}.*(which are OCI compliant runtimes){1}/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:oci_compliant") + (/(PASSED){1}.*(which are OCI compliant runtimes){1}/ =~ result[:output]).should_not be_nil end end diff --git a/spec/platform/observability_spec.cr b/spec/platform/observability_spec.cr index 23dca1766..bbab88c65 100644 --- a/spec/platform/observability_spec.cr +++ b/spec/platform/observability_spec.cr @@ -5,36 +5,32 @@ require "kubectl_client" describe "Platform Observability" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'kube_state_metrics' should return some json", tags: ["platform:observability"] do - LOGGING.info "Installing kube_state_metrics" + Log.info { "Installing kube_state_metrics" } helm = Helm::BinarySingleton.helm - `#{helm} repo add prometheus-community https://prometheus-community.github.io/helm-charts` - `#{helm} repo update` - resp = `#{helm} install --version 5.3.0 kube-state-metrics prometheus-community/kube-state-metrics` - LOGGING.info resp + result = ShellCmd.run("#{helm} repo add prometheus-community https://prometheus-community.github.io/helm-charts") + result = ShellCmd.run("#{helm} repo update") + result = ShellCmd.run("#{helm} install --version 5.3.0 kube-state-metrics prometheus-community/kube-state-metrics", force_output: true) KubectlClient::Get.wait_for_install("kube-state-metrics") - response_s = `./cnf-testsuite platform:kube_state_metrics poc` - LOGGING.info response_s - (/(PASSED){1}.*(Your platform is using the){1}.*(release for kube state metrics){1}/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:kube_state_metrics poc") + (/(PASSED){1}.*(Your platform is using the){1}.*(release for kube state metrics){1}/ =~ result[:output]).should_not be_nil ensure - resp = `#{helm} delete kube-state-metrics` - LOGGING.info resp - $?.success?.should be_true + result = ShellCmd.run("#{helm} delete kube-state-metrics", force_output: true) + result[:status].success?.should be_true end it "'node_exporter' should detect the named release of the installed node_exporter", tags: ["platform:observability"] do - LOGGING.info "Installing prometheus-node-exporter" + Log.info { "Installing prometheus-node-exporter" } helm = Helm::BinarySingleton.helm Helm.helm_repo_add("prometheus-community","https://prometheus-community.github.io/helm-charts") - resp = `#{helm} install node-exporter prometheus-community/prometheus-node-exporter` - LOGGING.info resp + result = ShellCmd.run("#{helm} install node-exporter prometheus-community/prometheus-node-exporter", force_output: true) pod_ready = "" pod_ready_timeout = 45 @@ -44,17 +40,15 @@ describe "Platform Observability" do sleep 1 pod_ready_timeout = pod_ready_timeout - 1 end - response_s = `./cnf-testsuite platform:node_exporter poc` - LOGGING.info response_s + result = ShellCmd.run_testsuite("platform:node_exporter poc") if check_containerd - (/(PASSED){1}.*(Your platform is using the node exporter){1}/ =~ response_s).should_not be_nil + (/(PASSED){1}.*(Your platform is using the node exporter){1}/ =~ result[:output]).should_not be_nil else - (/skipping node_exporter: This test only supports the Containerd Runtime./ =~ response_s).should_not be_nil + (/skipping node_exporter: This test only supports the Containerd Runtime./ =~ result[:output]).should_not be_nil end ensure - resp = `#{helm} delete node-exporter` - LOGGING.info resp - $?.success?.should be_true + result = ShellCmd.run("#{helm} delete node-exporter", force_output: true) + result[:status].success?.should be_true end it "'prometheus_adapter' should detect the named release of the installed prometheus_adapter", tags: ["platform:observability"] do @@ -69,9 +63,8 @@ describe "Platform Observability" do end KubectlClient::Get.wait_for_install("prometheus-adapter") - response_s = `./cnf-testsuite platform:prometheus_adapter poc` - Log.info { response_s } - (/(PASSED){1}.*(Your platform is using the prometheus adapter){1}/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:prometheus_adapter poc") + (/(PASSED){1}.*(Your platform is using the prometheus adapter){1}/ =~ result[:output]).should_not be_nil ensure resp = Helm.uninstall("prometheus-adapter") end @@ -88,12 +81,11 @@ describe "Platform Observability" do end Log.info { result } KubectlClient::Get.wait_for_install(deployment_name: "metrics-server") - response_s = `./cnf-testsuite platform:metrics_server poc` - LOGGING.info response_s - (/(PASSED){1}.*(Your platform is using the metrics server){1}/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:metrics_server poc") + (/(PASSED){1}.*(Your platform is using the metrics server){1}/ =~ result[:output]).should_not be_nil ensure resp = Helm.uninstall("metrics-server") - LOGGING.info resp + Log.info { resp } $?.success?.should be_true end end diff --git a/spec/platform/platform_spec.cr b/spec/platform/platform_spec.cr index a2fd96cad..40d9ab096 100644 --- a/spec/platform/platform_spec.cr +++ b/spec/platform/platform_spec.cr @@ -1,48 +1,33 @@ require "./../spec_helper" require "colorize" +require "../../src/tasks/utils/utils.cr" describe "Platform" do before_all do - # LOGGING.debug `pwd` - # LOGGING.debug `echo $KUBECONFIG` - `./cnf-testsuite samples_cleanup` - $?.success?.should be_true - LOGGING.info `./cnf-testsuite setup` - $?.success?.should be_true - # LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose` - # $?.success?.should be_true + result = ShellCmd.run_testsuite("samples_cleanup") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'platform:*' should not error out when no cnf is installed", tags: ["platform"] do - response_s = `./cnf-testsuite cleanup` - response_s = `./cnf-testsuite platform:oci_compliant` - LOGGING.info response_s - puts response_s - (/No cnf_testsuite.yml found/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cleanup") + result = ShellCmd.run_testsuite("platform:oci_compliant") + puts result[:output] + (/No cnf_testsuite.yml found/ =~ result[:output]).should be_nil end it "'platform' should not run prerequisites that are prefixed with a ~", tags: ["platform"] do - response_s = `./cnf-testsuite cleanup` - # response_s = `./cnf-testsuite platform` - stdout = IO::Memory.new - stderror = IO::Memory.new - # Capture all output from sonobuoy - process = Process.new("./cnf-testsuite", ["platform", "~k8s_conformance"], output: stdout, error: stderror) - status = process.wait - response_s = stdout.to_s - error = stderror.to_s - LOGGING.info "error: #{error}" - LOGGING.info "response #{response_s}" - (/kind=namespace namespace=sonobuoy/ =~ (response_s + error)).should be_nil + result = ShellCmd.run_testsuite("cleanup") + result = ShellCmd.run_testsuite("platform ~k8s_conformance") + (/kind=namespace namespace=sonobuoy/ =~ (result[:output] + result[:error])).should be_nil end it "'k8s_conformance' should pass if the sonobuoy tests pass", tags: ["platform"] do - response_s = `./cnf-testsuite k8s_conformance` - LOGGING.info response_s - (/PASSED: K8s conformance test has no failures/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("k8s_conformance") + (/PASSED: K8s conformance test has no failures/ =~ result[:output]).should_not be_nil end it "individual tasks like 'platform:exposed_dashboard' should not require an installed cnf to run", tags: ["platform"] do - response_s = `./cnf-testsuite platform:exposed_dashboard` - LOGGING.info response_s - (/You must install a CNF first./ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("platform:exposed_dashboard") + (/You must install a CNF first./ =~ result[:output]).should be_nil end end diff --git a/spec/platform/resilience_spec.cr b/spec/platform/resilience_spec.cr index 71f18a8d8..58d0b58e9 100644 --- a/spec/platform/resilience_spec.cr +++ b/spec/platform/resilience_spec.cr @@ -4,19 +4,17 @@ require "./../../src/tasks/utils/utils.cr" describe "Platform" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'worker_reboot_recovery' should pass if platform successfully recovers after reboot", tags: ["platform:worker_reboot_recovery"] do if check_destructive puts "Tests running in destructive mode".colorize(:red) - response_s = `./cnf-testsuite platform:worker_reboot_recovery destructive` - LOGGING.info response_s - (/(PASSED: Node came back online)/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:worker_reboot_recovery destructive") + (/(PASSED: Node came back online)/ =~ result[:output]).should_not be_nil else - response_s = `./cnf-testsuite platform:worker_reboot_recovery` - LOGGING.info response_s - (/SKIPPED/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:worker_reboot_recovery") + (/SKIPPED/ =~ result[:output]).should_not be_nil end end end diff --git a/spec/platform/security_spec.cr b/spec/platform/security_spec.cr index dcfa994e6..e1311da5e 100644 --- a/spec/platform/security_spec.cr +++ b/spec/platform/security_spec.cr @@ -4,26 +4,20 @@ require "./../../src/tasks/utils/utils.cr" describe "Platform" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'control_plane_hardening' should pass if the control plane has been hardened", tags: ["platform:security"] do - response_s = `./cnf-testsuite platform:control_plane_hardening` - Log.info { response_s } - (/(PASSED: Control plane hardened)/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:control_plane_hardening") + (/(PASSED: Control plane hardened)/ =~ result[:output]).should_not be_nil end it "'cluster_admin' should fail on a cnf that uses a cluster admin binding", tags: ["platform:security"] do begin - # LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` - # $?.success?.should be_true - response_s = `./cnf-testsuite platform:cluster_admin` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Users with cluster admin role found/ =~ response_s).should_not be_nil - # ensure - # `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("platform:cluster_admin") + result[:status].success?.should be_true + (/FAILED: Users with cluster admin role found/ =~ result[:output]).should_not be_nil end end @@ -31,8 +25,8 @@ describe "Platform" do dashboard_install_url = "https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0/aio/deploy/recommended.yaml" begin # Run the exposed_dashboard test to confirm no vulnerability before dashboard is installed - response_s = `./cnf-testsuite platform:exposed_dashboard` - (/PASSED: No exposed dashboard found in the cluster/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:exposed_dashboard") + (/PASSED: No exposed dashboard found in the cluster/ =~ result[:output]).should_not be_nil # Install the dashboard version 2.0.0. # According to the kubescape rule, anything less than v2.0.1 would fail. @@ -56,10 +50,9 @@ describe "Platform" do result = KubectlClient::Patch.spec("service", "kubernetes-dashboard", patch_spec.to_json, "kubernetes-dashboard") # Run the test again to confirm vulnerability with an exposed dashboard - response_s = `./cnf-testsuite platform:exposed_dashboard` - Log.info { response_s } - $?.success?.should be_true - (/FAILED: Found exposed dashboard in the cluster/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:exposed_dashboard") + result[:status].success?.should be_true + (/FAILED: Found exposed dashboard in the cluster/ =~ result[:output]).should_not be_nil ensure # Ensure to remove the Kubectl dashboard after the test KubectlClient::Delete.file(dashboard_install_url) @@ -69,9 +62,9 @@ describe "Platform" do it "'helm_tiller' should fail if Helm Tiller is running in the cluster", tags: ["platform:security"] do ShellCmd.run("kubectl run tiller --image=rancher/tiller:v2.11.0", "create_tiller") KubectlClient::Get.resource_wait_for_install("pod", "tiller") - response_s = `./cnf-testsuite platform:helm_tiller` - $?.success?.should be_true - (/FAILED: Containers with the Helm Tiller image are running/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:helm_tiller") + result[:status].success?.should be_true + (/FAILED: Containers with the Helm Tiller image are running/ =~ result[:output]).should_not be_nil ensure KubectlClient::Delete.command("pod/tiller") KubectlClient::Get.resource_wait_for_uninstall("pod", "tiller") @@ -79,8 +72,8 @@ describe "Platform" do it "'helm_tiller' should fail if Helm Tiller is running in the cluster", tags: ["platform:security"] do # By default we have nothing to setup for this task to pass since Helm v3 does not use Tiller. - response_s = `./cnf-testsuite platform:helm_tiller` - $?.success?.should be_true - (/PASSED: No Helm Tiller containers are running/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("platform:helm_tiller") + result[:status].success?.should be_true + (/PASSED: No Helm Tiller containers are running/ =~ result[:output]).should_not be_nil end end diff --git a/spec/prereqs_spec.cr b/spec/prereqs_spec.cr index 87cb6307b..e0eb1ed86 100644 --- a/spec/prereqs_spec.cr +++ b/spec/prereqs_spec.cr @@ -7,24 +7,20 @@ require "sam" describe "Prereq" do it "'prereq' should check the system for prerequisites", tags: ["points"] do - response_s = `./cnf-testsuite prereqs verbose` - LOGGING.info response_s - $?.success?.should be_true - (/helm found/ =~ response_s).should_not be_nil - # (/wget found/ =~ response_s).should_not be_nil - # (/curl found/ =~ response_s).should_not be_nil - (/kubectl found/ =~ response_s).should_not be_nil - (/git found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("prereqs verbose") + result[:status].success?.should be_true + (/helm found/ =~ result[:output]).should_not be_nil + + (/kubectl found/ =~ result[:output]).should_not be_nil + (/git found/ =~ result[:output]).should_not be_nil end it "'prereq' with offline option should check the system for prerequisites except git", tags: ["points"] do - response_s = `./cnf-testsuite prereqs verbose offline=1` - LOGGING.info response_s - $?.success?.should be_true - (/helm found/ =~ response_s).should_not be_nil - # (/wget found/ =~ response_s).should_not be_nil - # (/curl found/ =~ response_s).should_not be_nil - (/kubectl found/ =~ response_s).should_not be_nil - (/git found/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("prereqs verbose offline=1") + result[:status].success?.should be_true + (/helm found/ =~ result[:output]).should_not be_nil + + (/kubectl found/ =~ result[:output]).should_not be_nil + (/git found/ =~ result[:output]).should be_nil end end diff --git a/spec/setup_spec.cr b/spec/setup_spec.cr index 42e481474..5b1940790 100644 --- a/spec/setup_spec.cr +++ b/spec/setup_spec.cr @@ -10,123 +10,108 @@ require "sam" describe "Setup" do after_each do - `./cnf-testsuite cleanup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cleanup") + result[:status].success?.should be_true end it "'setup' should completely setup the cnf testsuite environment before installing cnfs", tags: ["setup"] do - response_s = `./cnf-testsuite setup` - LOGGING.info response_s - $?.success?.should be_true - (/Setup complete/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true + (/Setup complete/ =~ result[:output]).should_not be_nil end it "'generate_config' should generate a cnf-testsuite.yml for a helm chart", tags: ["setup-generate"] do - LOGGING.info `./cnf-testsuite setup` - response_s = `./cnf-testsuite generate_config config-src=stable/coredns output-file=./cnf-testsuite-test.yml` - LOGGING.info response_s - $?.success?.should be_true - # (/Setup complete/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("generate_config config-src=stable/coredns output-file=./cnf-testsuite-test.yml") + result[:status].success?.should be_true yaml = File.open("./cnf-testsuite-test.yml") do |file| YAML.parse(file) end - LOGGING.debug "test yaml: #{yaml}" - LOGGING.info `cat ./cnf-testsuite-test.yml` + Log.debug { "test yaml: #{yaml}" } + result = ShellCmd.run("cat ./cnf-testsuite-test.yml", force_output: true) (yaml["helm_chart"] == "stable/coredns").should be_true ensure - `rm ./cnf-testsuite-test.yml` + result = ShellCmd.run("rm ./cnf-testsuite-test.yml", force_output: true) end it "'generate_config' should generate a cnf-testsuite.yml for a helm directory", tags: ["setup-generate"] do - LOGGING.info `./cnf-testsuite setup` - response_s = `./cnf-testsuite generate_config config-src=sample-cnfs/k8s-sidecar-container-pattern/chart output-file=./cnf-testsuite-test.yml` - LOGGING.info response_s - $?.success?.should be_true - # (/Setup complete/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("generate_config config-src=sample-cnfs/k8s-sidecar-container-pattern/chart output-file=./cnf-testsuite-test.yml") + result[:status].success?.should be_true yaml = File.open("./cnf-testsuite-test.yml") do |file| YAML.parse(file) end - LOGGING.debug "test yaml: #{yaml}" - LOGGING.info `cat ./cnf-testsuite-test.yml` + Log.debug { "test yaml: #{yaml}" } + result = ShellCmd.run("cat ./cnf-testsuite-test.yml", force_output: true) (yaml["helm_directory"] == "sample-cnfs/k8s-sidecar-container-pattern/chart").should be_true ensure - `rm ./cnf-testsuite-test.yml` + result = ShellCmd.run("rm ./cnf-testsuite-test.yml", force_output: true) end it "'generate_config' should generate a cnf-testsuite.yml for a manifest directory", tags: ["setup-generate"] do - LOGGING.info `./cnf-testsuite setup` - response_s = `./cnf-testsuite generate_config config-src=sample-cnfs/k8s-non-helm/manifests output-file=./cnf-testsuite-test.yml` - LOGGING.info response_s - $?.success?.should be_true - # (/Setup complete/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("generate_config config-src=sample-cnfs/k8s-non-helm/manifests output-file=./cnf-testsuite-test.yml") + result[:status].success?.should be_true yaml = File.open("./cnf-testsuite-test.yml") do |file| YAML.parse(file) end - LOGGING.debug "test yaml: #{yaml}" - LOGGING.info `cat ./cnf-testsuite-test.yml` + Log.debug { "test yaml: #{yaml}" } + result = ShellCmd.run("cat ./cnf-testsuite-test.yml", force_output: true) (yaml["manifest_directory"] == "sample-cnfs/k8s-non-helm/manifests").should be_true ensure - `rm ./cnf-testsuite-test.yml` + result = ShellCmd.run("rm ./cnf-testsuite-test.yml", force_output: true) end it "'cnf_setup/cnf_cleanup' should install/cleanup with cnf-path arg as alias for cnf-config", tags: ["setup"] do begin - response_s = `./cnf-testsuite cnf_setup cnf-path=example-cnfs/coredns/cnf-testsuite.yml` - LOGGING.info response_s - $?.success?.should be_true - (/Successfully setup coredns/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=example-cnfs/coredns/cnf-testsuite.yml") + result[:status].success?.should be_true + (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - response_s = `./cnf-testsuite cnf_cleanup cnf-path=example-cnfs/coredns/cnf-testsuite.yml` - LOGGING.info response_s - $?.success?.should be_true - (/Successfully cleaned up/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=example-cnfs/coredns/cnf-testsuite.yml") + result[:status].success?.should be_true + (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf with a cnf-testsuite.yml", tags: ["setup"] do begin - response_s = `./cnf-testsuite cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml` - LOGGING.info response_s - $?.success?.should be_true - (/Successfully setup coredns/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml") + result[:status].success?.should be_true + (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - response_s = `./cnf-testsuite cnf_cleanup cnf-config=example-cnfs/coredns/cnf-testsuite.yml` - LOGGING.info response_s - $?.success?.should be_true - (/Successfully cleaned up/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=example-cnfs/coredns/cnf-testsuite.yml") + result[:status].success?.should be_true + (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should work with cnf-testsuite.yml that has no directory associated with it", tags: ["setup"] do begin #TODO force cnfs/ to be deployment name and not the directory name - response_s = `./cnf-testsuite cnf_setup cnf-config=spec/fixtures/cnf-testsuite.yml verbose` - LOGGING.info("response_s: #{response_s}") - $?.success?.should be_true - (/Successfully setup coredns/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=spec/fixtures/cnf-testsuite.yml verbose") + result[:status].success?.should be_true + (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - response_s = `./cnf-testsuite cnf_cleanup cnf-path=spec/fixtures/cnf-testsuite.yml verbose` - LOGGING.info("response_s: #{response_s}") - $?.success?.should be_true - (/Successfully cleaned up/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=spec/fixtures/cnf-testsuite.yml verbose") + result[:status].success?.should be_true + (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should install/cleanup with helm_directory that descends multiple directories", tags: ["setup"] do begin - response_s = `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml` - LOGGING.info response_s - $?.success?.should be_true - (/Successfully setup coredns/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") + result[:status].success?.should be_true + (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - response_s = `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml` - LOGGING.info response_s - $?.success?.should be_true - (/Successfully cleaned up/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") + result[:status].success?.should be_true + (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index fb81c9822..23a33cee0 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -1,15 +1,23 @@ require "spec" require "colorize" require "../src/cnf_testsuite" +require "../src/tasks/utils/utils.cr" ENV["CRYSTAL_ENV"] = "TEST" -LOGGING.info "Building ./cnf-testsuite".colorize(:green) -`crystal build --warnings none src/cnf-testsuite.cr` -if $?.success? - LOGGING.info "Build Success!".colorize(:green) +Log.info { "Building ./cnf-testsuite".colorize(:green) } +result = ShellCmd.run("crystal build --warnings none src/cnf-testsuite.cr") +if result[:status].success? + Log.info { "Build Success!".colorize(:green) } else - LOGGING.info "crystal build failed!".colorize(:red) + Log.info { "crystal build failed!".colorize(:red) } raise "crystal build failed in spec_helper" end + +module ShellCmd + def self.run_testsuite(testsuite_cmd, cmd_prefix="") + cmd = "#{cmd_prefix} ./cnf-testsuite #{testsuite_cmd}" + run(cmd, log_prefix: "ShellCmd.run_testsuite", force_output: true, joined_output: true) + end +end diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr index d7f6b1116..7bf037527 100644 --- a/spec/utils/cnf_manager_spec.cr +++ b/spec/utils/cnf_manager_spec.cr @@ -3,29 +3,25 @@ require "../spec_helper" require "colorize" require "../../src/tasks/utils/utils.cr" require "../../src/tasks/helmenv_setup.cr" +require "../../src/tasks/utils/utils.cr" require "kubectl_client" require "file_utils" require "sam" describe "SampleUtils" do before_all do - `./cnf-testsuite helm_local_install` - $?.success?.should be_true - `./cnf-testsuite cleanup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("helm_local_install") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("cleanup") + result[:status].success?.should be_true # Ensure a results file is present to test different scenarios CNFManager::Points::Results.ensure_results_file! end - # after_all do - # LOGGING.debug `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose wait_count=0` - # $?.success?.should be_true - # end - after_each do - `./cnf-testsuite cleanup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cleanup") + result[:status].success?.should be_true end it "'images_from_config_src' should return a list of containers for a cnf", tags: ["cnf-setup"] do @@ -34,17 +30,17 @@ describe "SampleUtils" do end it "'cnf_setup' should pass with a minimal cnf-testsuite.yml", tags: ["cnf-setup"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample-minimal-cnf/ wait_count=0` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-minimal-cnf/ wait_count=0") + result[:status].success?.should be_true ensure - `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true") end it "'cnf_setup' should support cnf-config as an alias for cnf-path", tags: ["cnf-setup"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-minimal-cnf/ wait_count=0` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-minimal-cnf/ wait_count=0") + result[:status].success?.should be_true ensure - `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true") end it "'points_yml' should parse and return the points yaml file", tags: ["points"] do @@ -70,26 +66,12 @@ describe "SampleUtils" do (CNFManager::Points.task_points("liveness", CNFManager::ResultStatus::Skipped)).should eq(0) end - # todo add mock test for testing require tests (privilege mode is no longer required) - # it "'task_required' should return if the passed task is required", tags: ["tasks"] do - # CNFManager::Points.clean_results_yml - # (CNFManager::Points.task_required("privileged")).should be_true - # end - - # todo add mock test for testing require tests (privilege mode is no longer required) - # it "'failed_required_tasks' should return a list of failed required tasks", tags: ["tasks"] do - # CNFManager::Points.clean_results_yml - # CNFManager::Points.failed_task("privileged", "FAILED: Privileged container found") - # (CNFManager::Points.failed_required_tasks).should eq(["privileged"]) - # end - it "'upsert_task' insert task in the results file", tags: ["tasks"] do CNFManager::Points.clean_results_yml CNFManager::Points.upsert_task("liveness", PASSED, CNFManager::Points.task_points("liveness"), Time.utc) yaml = File.open("#{CNFManager::Points::Results.file}") do |file| YAML.parse(file) end - # LOGGING.debug yaml["items"].as_a.inspect (yaml["items"].as_a.find {|x| x["name"] == "liveness" && x["points"] == CNFManager::Points.task_points("liveness")}).should be_truthy end @@ -100,7 +82,6 @@ describe "SampleUtils" do yaml = File.open("#{CNFManager::Points::Results.file}") do |file| YAML.parse(file) end - # LOGGING.debug yaml["items"].as_a.inspect (yaml["items"].as_a.find {|x| x["name"] == "liveness" && x["points"] == CNFManager::Points.task_points("liveness")}).should be_truthy (CNFManager::Points.total_points).should eq(100) end @@ -118,7 +99,7 @@ describe "SampleUtils" do upsert_na_task("readiness", "✔️ NA", Time.utc) resp2 = CNFManager::Points.total_max_points - LOGGING.info "readiness points: #{CNFManager::Points.task_points("readiness").not_nil!.to_i}" + Log.info { "readiness points: #{CNFManager::Points.task_points("readiness").not_nil!.to_i}" } (resp2).should eq((resp1 - CNFManager::Points.task_points("readiness").not_nil!.to_i)) end @@ -197,8 +178,6 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - # args = Sam::Args.new - # CNFManager.sample_setup(config_file: config_file, release_name: "", deployment_name: "coredns-coredns", helm_chart: "stable/coredns", helm_directory: "", git_clone_url: "https://github.com/coredns/coredns.git", wait_count: 0, verbose: true) # check if directory exists config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) release_name = config.cnf_config[:release_name] @@ -215,14 +194,11 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - # args = Sam::Args.new - # CNFManager.sample_setup_args(sample_dir: config_file, args: args, verbose: true, wait_count: 0 ) # check if directory exists config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) release_name = config.cnf_config[:release_name] (Dir.exists? "cnfs/#{release_name}").should be_true (File.exists?("cnfs/#{release_name}/cnf-testsuite.yml")).should be_true - # (File.exists?("cnfs/#{release_name}/helm_chart/Chart.yaml")).should be_true CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-generic-cnf", verbose: true) (Dir.exists? "cnfs/#{release_name}").should be_false end @@ -232,14 +208,11 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - # args = Sam::Args.new - # CNFManager.sample_setup_args(sample_dir: config_file, args: args, verbose: true, wait_count: 0 ) # check if directory exists config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) release_name = config.cnf_config[:release_name] (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true (File.exists?("cnfs/#{release_name}/cnf-testsuite.yml")).should be_true - # (File.exists?("cnfs/#{release_name}/helm_chart/Chart.yaml")).should be_true CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-generic-cnf", verbose: true) (Dir.exists? "cnfs/#{release_name}").should be_false end @@ -248,8 +221,6 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - # args = Sam::Args.new - # CNFManager.sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 0 ) cleanup = CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-generic-cnf", verbose: true) (cleanup).should be_true (Dir.exists? "cnfs/coredns").should be_false @@ -262,8 +233,6 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - # args = Sam::Args.new - # CNFManager.sample_setup_args(sample_dir: config_file, deploy_with_chart: false, args: args, verbose: true, wait_count: 0 ) config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) release_name = config.cnf_config[:release_name] (Dir.exists? "cnfs/#{release_name}").should be_true @@ -279,12 +248,9 @@ describe "SampleUtils" do config_file = "sample-cnfs/k8s-non-helm" args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) - Log.info {"Running Setup"} + Log.info { "Running Setup" } CNFManager.sample_setup(cli_hash) - # args = Sam::Args.new - # config_file = "sample-cnfs/k8s-non-helm" - # CNFManager.sample_setup_args(sample_dir: config_file, deploy_with_chart: false, args: args, verbose: true, install_from_manifest: true, wait_count: 0 ) - Log.info {"Parse Config"} + Log.info { "Parse Config" } config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) release_name = config.cnf_config[:release_name] (Dir.exists? "cnfs/#{release_name}").should be_true @@ -307,13 +273,9 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - # args = Sam::Args.new - # config_file = "sample-cnfs/sample-generic-cnf" - # CNFManager.sample_setup_args(sample_dir: config_file, args: args, verbose: true, wait_count: 1 ) args = Sam::Args.new(["cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml", "verbose"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - # CNFManager.sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", args: args, verbose: true ) config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) release_name = config.cnf_config[:release_name] CNFManager.cnf_config_list()[0].should contain("#{release_name}/#{CONFIG_FILE}") @@ -323,9 +285,6 @@ describe "SampleUtils" do config_file = "sample-cnfs/sample-generic-cnf" args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) cli_hash = CNFManager.sample_setup_cli_args(args) - # args = Sam::Args.new - # CNFManager.sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: args, verbose: true, wait_count: 1 ) - # CNFManager.helm_repo_add.should eq(true) args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml"]) CNFManager.helm_repo_add(args: args).should eq(true) end @@ -338,7 +297,7 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml"]) yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - LOGGING.info yml.inspect + Log.info { yml.inspect } ("#{yml.get("release_name").as_s?}").should eq("coredns") valid, command_output = CNFManager.validate_cnf_testsuite_yml(yml) @@ -348,9 +307,9 @@ describe "SampleUtils" do end it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, when a cnf has a valid config file yml", tags: ["validate_config"] do - response_s = `./cnf-testsuite validate_config cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - (/CNF configuration validated/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("validate_config cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + (/CNF configuration validated/ =~ result[:output]).should_not be_nil end @@ -358,12 +317,12 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./spec/fixtures/cnf-testsuite-unmapped-keys-and-subkeys.yml"]) yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - LOGGING.info yml.inspect + Log.info { yml.inspect } ("#{yml.get("release_name").as_s?}").should eq("coredns") status, warning_output = CNFManager.validate_cnf_testsuite_yml(yml) - LOGGING.warn "WARNING: #{warning_output}" + Log.warn { "WARNING: #{warning_output}" } (status).should eq(true) (warning_output).should_not be_nil @@ -371,12 +330,10 @@ describe "SampleUtils" do it "'CNFManager.validate_cnf_testsuite_yml' (command) should warn, but be valid when a cnf config file yml has fields that are not a part of the validation type", tags: ["validate_config"] do - response_s = `./cnf-testsuite validate_config cnf-config=spec/fixtures/cnf-testsuite-unmapped-keys-and-subkeys.yml` - $?.success?.should be_true - LOGGING.debug "validate_config resp: #{response_s}" - # (/WARNING: Unmapped cnf_testsuite.yml keys. Please add them to the validator/ =~ response_s).should_not be_nil - # (/WARNING: helm_repository is unset or has unmapped subkeys. Please update your cnf_testsuite.yml/ =~ response_s).should_not be_nil - (/CNF configuration validated/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("validate_config cnf-config=spec/fixtures/cnf-testsuite-unmapped-keys-and-subkeys.yml") + result[:status].success?.should be_true + Log.debug { "validate_config resp: #{result[:output]}" } + (/CNF configuration validated/ =~ result[:output]).should_not be_nil end @@ -386,9 +343,8 @@ describe "SampleUtils" do dir_list = get_dirs - [".", ".."] dir_list.each do |dir| testsuite_yml = "sample-cnfs/#{dir}/cnf-testsuite.yml" - response_s = `./cnf-testsuite validate_config cnf-config=#{testsuite_yml}` - Log.info { "\n #{testsuite_yml}: #{response_s}" } - (/CNF configuration validated/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("validate_config cnf-config=#{testsuite_yml}") + (/CNF configuration validated/ =~ result[:output]).should_not be_nil end end @@ -398,11 +354,11 @@ describe "SampleUtils" do dir_list = get_dirs - [".", ".."] dir_list.each do |dir| testsuite_yml = "example-cnfs/#{dir}/cnf-testsuite.yml" - response_s = `./cnf-testsuite validate_config cnf-config=#{testsuite_yml}` - if (/Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file/ =~ response_s) - LOGGING.info "\n #{testsuite_yml}: #{response_s}" + result = ShellCmd.run_testsuite("validate_config cnf-config=#{testsuite_yml}") + if (/Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file/ =~ result[:output]) + Log.info { "\n #{testsuite_yml}: #{result[:output]}" } end - (/CNF configuration validated/ =~ response_s).should_not be_nil + (/CNF configuration validated/ =~ result[:output]).should_not be_nil end end @@ -417,17 +373,16 @@ describe "SampleUtils" do it "'CNFManager.workload_resource_test' should accept an args and cnf-config argument, populate a deployment, container, and intialized argument, and then apply a test to a cnf", tags: ["cnf-config"] do args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml"]) - # check_cnf_config_then_deploy(args) cli_hash = CNFManager.sample_setup_cli_args(args, false) CNFManager.sample_setup(cli_hash) if cli_hash["config_file"] config = CNFManager::Config.parse_config_yml("./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml") task_response = CNFManager.workload_resource_test(args, config) do |resource, container, initialized| test_passed = true begin - VERBOSE_LOGGING.debug container.as_h["name"].as_s if check_verbose(args) + Log.for("verbose").debug { container.as_h["name"].as_s } if check_verbose(args) container.as_h["livenessProbe"].as_h rescue ex - VERBOSE_LOGGING.error ex.message if check_verbose(args) + Log.for("verbose").error { ex.message } if check_verbose(args) test_passed = false puts "No livenessProbe found for resource: #{resource} and container: #{container.as_h["name"].as_s}".colorize(:red) end @@ -446,29 +401,24 @@ describe "SampleUtils" do it "bonus tests should not be includded in the maximum points when a failure occurs", tags: ["cnf-config"] do begin # fails because doesn't have a service - install = `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample-ndn-privileged` - Log.info { install } - response_s = `./cnf-testsuite cert_microservice` - Log.info { response_s} - (/of 6 tests passed/ =~ response_s).should_not be_nil - Log.info { response_s } + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-ndn-privileged") + result = ShellCmd.run_testsuite("cert_microservice") + (/of 6 tests passed/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged` } + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged") end end it "Helm_values should be used during the installation of a cnf", tags: ["cnf-config"] do begin # fails because doesn't have a service - install = `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample_coredns_values` - Log.info { install } + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_coredns_values") deployment_containers = KubectlClient::Get.deployment_containers("coredns-coredns") image_tags = KubectlClient::Get.container_image_tags(deployment_containers) - Log.info { "image_tags: #{image_tags}"} + Log.info { "image_tags: #{image_tags}" } (/1.6.9/ =~ image_tags[0][:tag]).should_not be_nil - # Log.info { response_s } ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_coredns_values` } + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_coredns_values") end end diff --git a/spec/utils/k8s_instrumentation_spec.cr b/spec/utils/k8s_instrumentation_spec.cr index 44fab1668..4557a79d8 100644 --- a/spec/utils/k8s_instrumentation_spec.cr +++ b/spec/utils/k8s_instrumentation_spec.cr @@ -7,8 +7,8 @@ require "sam" describe "K8sInstrumentation" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'#disk_speed' should return all responses for the sysbench disk speed call on a pod", tags: ["k8s-instrumentation"] do diff --git a/spec/utils/kubescape_spec.cr b/spec/utils/kubescape_spec.cr index d8dbf1a79..e75c1b827 100644 --- a/spec/utils/kubescape_spec.cr +++ b/spec/utils/kubescape_spec.cr @@ -1,19 +1,20 @@ require "../spec_helper" require "../../src/tasks/utils/kubescape.cr" +require "../../src/tasks/utils/utils.cr" describe "K8sInstrumentation" do before_all do - LOGGING.info `./cnf-testsuite install_kubescape` + result = ShellCmd.run_testsuite("install_kubescape") end it "'#scan and #test_by_test_name' should return the results of a kubescape scan", tags: ["kubescape"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") Kubescape.scan results_json = Kubescape.parse test_json = Kubescape.test_by_test_name(results_json, "Network policies") (test_json).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.ymll") end end diff --git a/spec/utils/utils_spec.cr b/spec/utils/utils_spec.cr index 9e85609ff..1f7d47ae2 100644 --- a/spec/utils/utils_spec.cr +++ b/spec/utils/utils_spec.cr @@ -8,18 +8,18 @@ require "sam" describe "Utils" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true # Ensure a results file is present to test different scenarios CNFManager::Points::Results.ensure_results_file! end before_each do - `./cnf-testsuite results_yml_cleanup` + result = ShellCmd.run_testsuite("results_yml_cleanup") end after_each do - `./cnf-testsuite results_yml_cleanup` + result = ShellCmd.run_testsuite("results_yml_cleanup") end it "'toggle' should return a boolean for a toggle in the config.yml", tags: ["args"] do @@ -39,7 +39,6 @@ describe "Utils" do end it "'check_' should return the feature level for an argument variable", tags: ["args"] do - # (check_ga).should be_false (check_alpha).should be_false (check_beta).should be_false (check_wip).should be_false @@ -81,10 +80,10 @@ describe "Utils" do task_response = CNFManager::Task.single_task_runner(args) do |args, config| - Log.info {"single_task_runner spec args #{args.inspect}"} + Log.info { "single_task_runner spec args #{args.inspect}" } white_list_container_names = config.cnf_config[:white_list_container_names] - Log.info {"white_list_container_names #{white_list_container_names.inspect}"} + Log.info { "white_list_container_names #{white_list_container_names.inspect}" } violation_list = [] of String resource_response = CNFManager.workload_resource_test(args, config) do |resource, container, initialized| @@ -150,15 +149,13 @@ describe "Utils" do it "'all_cnfs_task_runner' should run a test against all cnfs in the cnfs directory if there is not cnf-config argument passed to it", tags: ["task_runner"] do my_args = Sam::Args.new - # CNFManager.sample_setup_args(sample_dir: "sample-cnfs/sample-generic-cnf", args: my_args) - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-generic-cnf` - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_privileged_cnf` - # CNFManager.sample_setup_args(sample_dir: "sample-cnfs/sample_privileged_cnf", args: my_args ) + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-generic-cnf") + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_privileged_cnf") task_response = CNFManager::Task.all_cnfs_task_runner(my_args) do |args, config| - LOGGING.info("all_cnfs_task_runner spec args #{args.inspect}") - VERBOSE_LOGGING.info "privileged" if check_verbose(args) + Log.info { "all_cnfs_task_runner spec args #{args.inspect}" } + Log.for("verbose").info { "privileged" } if check_verbose(args) white_list_container_names = config.cnf_config[:white_list_container_names] - VERBOSE_LOGGING.info "white_list_container_names #{white_list_container_names.inspect}" if check_verbose(args) + Log.for("verbose").info { "white_list_container_names #{white_list_container_names.inspect}" } if check_verbose(args) violation_list = [] of String resource_response = CNFManager.workload_resource_test(args, config) do |resource, container, initialized| @@ -174,7 +171,7 @@ describe "Utils" do true end end - LOGGING.debug "violator list: #{violation_list.flatten}" + Log.debug { "violator list: #{violation_list.flatten}" } emoji_security="" if resource_response resp = upsert_passed_task("privileged", "✔️ PASSED: No privileged containers", Time.utc) @@ -191,78 +188,71 @@ describe "Utils" do end it "'task_runner' should run a test against a single cnf if passed a cnf-config argument even if there are multiple cnfs installed", tags: ["task_runner"] do - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml` - Log.info {response_s} - response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml` - Log.info {response_s} - resp = `./cnf-testsuite privileged` - Log.info { resp } - (resp).includes?("FAILED: Found 1 privileged containers").should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("privileged") + (result[:output]).includes?("FAILED: Found 1 privileged containers").should be_true ensure - response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml` - Log.info { response_s } - $?.success?.should be_true - response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml` - Log.info { response_s } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml") + result[:status].success?.should be_true end it "'logger' command line logger level setting via config.yml", tags: ["logger"] do # NOTE: the config.yml file is in the root of the repo directory. # as written this test depends on they key loglevel being set to 'info' in that config.yml - response_s = `./cnf-testsuite test` - $?.success?.should be_true - (/DEBUG -- cnf-testsuite: debug test/ =~ response_s).should be_nil - (/INFO -- cnf-testsuite: info test/ =~ response_s).should_not be_nil - (/WARN -- cnf-testsuite: warn test/ =~ response_s).should_not be_nil - (/ERROR -- cnf-testsuite: error test/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("test") + result[:status].success?.should be_true + (/DEBUG -- cnf-testsuite: debug test/ =~ result[:output]).should be_nil + (/INFO -- cnf-testsuite: info test/ =~ result[:output]).should_not be_nil + (/WARN -- cnf-testsuite: warn test/ =~ result[:output]).should_not be_nil + (/ERROR -- cnf-testsuite: error test/ =~ result[:output]).should_not be_nil end it "'logger' command line logger level setting works", tags: ["logger"] do # Note: implicitly tests the override of config.yml if it exist in repo root - response_s = `./cnf-testsuite -l debug test` - LOGGING.info response_s - $?.success?.should be_true - (/DEBUG -- cnf-testsuite: debug test/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("-l debug test") + result[:status].success?.should be_true + (/DEBUG -- cnf-testsuite: debug test/ =~ result[:output]).should_not be_nil end it "'logger' LOGLEVEL NO underscore environment variable level setting works", tags: ["logger"] do # Note: implicitly tests the override of config.yml if it exist in repo root - response_s = `unset LOG_LEVEL; LOGLEVEL=DEBUG ./cnf-testsuite test` - $?.success?.should be_true - (/DEBUG -- cnf-testsuite: debug test/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("test", cmd_prefix: "unset LOG_LEVEL; LOGLEVEL=DEBUG") + result[:status].success?.should be_true + (/DEBUG -- cnf-testsuite: debug test/ =~ result[:output]).should_not be_nil end it "'logger' LOG_LEVEL WITH underscore environment variable level setting works", tags: ["logger"] do # Note: implicitly tests the override of config.yml if it exist in repo root - response_s = `LOG_LEVEL=DEBUG ./cnf-testsuite test` - $?.success?.should be_true - (/DEBUG -- cnf-testsuite: debug test/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("test", cmd_prefix: "LOG_LEVEL=DEBUG") + result[:status].success?.should be_true + (/DEBUG -- cnf-testsuite: debug test/ =~ result[:output]).should_not be_nil end it "'logger' command line level setting overrides environment variable", tags: ["logger"] do - response_s = `LOGLEVEL=DEBUG ./cnf-testsuite -l error test` - $?.success?.should be_true - (/DEBUG -- cnf-testsuite: debug test/ =~ response_s).should be_nil - (/INFO -- cnf-testsuite: info test/ =~ response_s).should be_nil - (/WARN -- cnf-testsuite: warn test/ =~ response_s).should be_nil - (/ERROR -- cnf-testsuite: error test/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("-l error test", cmd_prefix: "LOG_LEVEL=DEBUG") + result[:status].success?.should be_true + (/DEBUG -- cnf-testsuite: debug test/ =~ result[:output]).should be_nil + (/INFO -- cnf-testsuite: info test/ =~ result[:output]).should be_nil + (/WARN -- cnf-testsuite: warn test/ =~ result[:output]).should be_nil + (/ERROR -- cnf-testsuite: error test/ =~ result[:output]).should_not be_nil end it "'logger' defaults to error when level set is missplled", tags: ["logger"] do # Note: implicitly tests the override of config.yml if it exist in repo root - response_s = `unset LOG_LEVEL; LOGLEVEL=DEGUB ./cnf-testsuite test` - $?.success?.should be_true - (/ERROR -- cnf-testsuite: Invalid logging level set. defaulting to ERROR/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("test", cmd_prefix: "unset LOG_LEVEL; LOG_LEVEL=DEGUB") + result[:status].success?.should be_true + (/ERROR -- cnf-testsuite: Invalid logging level set. defaulting to ERROR/ =~ result[:output]).should_not be_nil end it "'logger' or verbose output should be shown when verbose flag is set", tags: ["logger"] do - `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-coredns-cnf` - response_s = `./cnf-testsuite -l info helm_deploy verbose` - Log.info { response_s } - puts response_s - $?.success?.should be_true - (/helm_deploy args/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-coredns-cnf") + result = ShellCmd.run_testsuite("-l info helm_deploy verbose") + puts result[:output] + result[:status].success?.should be_true + (/helm_deploy args/ =~ result[:output]).should_not be_nil ensure CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-coredns-cnf", verbose: true) end @@ -284,7 +274,7 @@ describe "Utils" do Process.run("grep -r -I -P '^ *it \"(?!.*tags(.*\"))' ./spec", shell: true) do |proc| while line = proc.output.gets response << line - LOGGING.info "#{line}" + Log.info { "#{line}" } end end (response.to_s.size > 0).should be_false diff --git a/spec/workload/compatibility_spec.cr b/spec/workload/compatibility_spec.cr index 97720d895..0d22cb7dd 100644 --- a/spec/workload/compatibility_spec.cr +++ b/spec/workload/compatibility_spec.cr @@ -8,28 +8,27 @@ require "sam" describe "Compatibility" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'cni_compatible' should pass if the cnf works with calico and flannel", tags: ["compatibility"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true retry_limit = 5 retries = 1 - response_s = "" - until (/PASSED/ =~ response_s) || retries > retry_limit - Log.info {"cni_compatible spec retry: #{retries}"} + until (/PASSED/ =~ result[:output]) || retries > retry_limit + Log.info { "cni_compatible spec retry: #{retries}" } sleep 1.0 - response_s = `./cnf-testsuite cni_compatible verbose` + result = ShellCmd.run_testsuite("cni_compatible verbose") retries = retries + 1 end - Log.info {"Status: #{response_s}"} - (/PASSED: CNF compatible with both Calico and Cilium/ =~ response_s).should_not be_nil + Log.info { "Status: #{result[:output]}" } + (/PASSED: CNF compatible with both Calico and Cilium/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true end end diff --git a/spec/workload/configuration_spec.cr b/spec/workload/configuration_spec.cr index f105fbbad..f2678c8e0 100644 --- a/spec/workload/configuration_spec.cr +++ b/spec/workload/configuration_spec.cr @@ -1,214 +1,171 @@ require "../spec_helper" require "kubectl_client" require "colorize" +require "../../src/tasks/utils/utils.cr" describe CnfTestSuite do before_all do - LOGGING.debug `pwd` - LOGGING.debug `echo $KUBECONFIG` - - `./cnf-testsuite setup` - `./cnf-testsuite samples_cleanup` - $?.success?.should be_true - `./cnf-testsuite configuration_file_setup` - - # `./cnf-testsuite setup` - # $?.success?.should be_true - end - - - # it "'versioned_tag' should pass when a cnf has image tags that are all versioned", tags: ["versioned_tag"] do - # begin - # LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` - # $?.success?.should be_true - # response_s = `LOG_LEVEL=info ./cnf-testsuite versioned_tag verbose` - # LOGGING.info response_s - # $?.success?.should be_true - # (/PASSED: Container images use versioned tags/ =~ response_s).should_not be_nil - # ensure - # LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` - # LOGGING.info `./cnf-testsuite uninstall_opa` - # end - # end - - # The failure scenario spec test for versioned_tag is a hard thing to pass on the GitHub Actions builds. - # Commenting the tests for versioned_tag since it will be replaced with latest_tag soon. - # - # it "'versioned_tag' should fail when a cnf has image tags that are not versioned", tags: ["versioned_tag"] do - # begin - # LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/k8s-sidecar-container-pattern/cnf-testsuite.yml` - # $?.success?.should be_true - # response_s = `LOG_LEVEL=info ./cnf-testsuite versioned_tag verbose` - # LOGGING.info response_s - # $?.success?.should be_true - # (/FAILED: Container images do not use versioned tags/ =~ response_s).should_not be_nil - # ensure - # LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-sidecar-container-pattern/cnf-testsuite.yml` - # LOGGING.info `./cnf-testsuite uninstall_opa` - # end - # end + result = ShellCmd.run("pwd") + Log.debug { result[:output] } + result = ShellCmd.run("echo $KUBECONFIG") + Log.debug { result[:output] } + + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("samples_cleanup") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("configuration_file_setup") + + end it "'increase_decrease_capacity' should pass ", tags: ["increase_decrease_capacity"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite increase_decrease_capacity verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Replicas increased to/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("increase_decrease_capacity verbose") + result[:status].success?.should be_true + (/PASSED: Replicas increased to/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") end end it "'liveness' should pass when livenessProbe is set", tags: ["liveness"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true - response_s = `LOG_LEVEL=debug ./cnf-testsuite liveness verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("liveness verbose", cmd_prefix:"LOG_LEVEL=debug") + result[:status].success?.should be_true + (/PASSED: Helm liveness probe/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ") end end it "'liveness' should fail when livenessProbe is not set", tags: ["liveness"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite liveness verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: No livenessProbe found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("liveness verbose") + result[:status].success?.should be_true + (/FAILED: No livenessProbe found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite sample_coredns_bad_liveness_cleanup` + result = ShellCmd.run_testsuite("sample_coredns_bad_liveness_cleanup") end end it "'readiness' should pass when readinessProbe is set", tags: ["readiness"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true - response_s = `LOG_LEVEL=debug ./cnf-testsuite readiness verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Helm readiness probe/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("readiness verbose", cmd_prefix: "LOG_LEVEL=debug") + result[:status].success?.should be_true + (/PASSED: Helm readiness probe/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ") end end it "'readiness' should fail when readinessProbe is not set", tags: ["readiness"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite readiness verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: No readinessProbe found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("readiness verbose") + result[:status].success?.should be_true + (/FAILED: No readinessProbe found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite sample_coredns_bad_liveness_cleanup` + result = ShellCmd.run_testsuite("sample_coredns_bad_liveness_cleanup") end end it "'rolling_update' should pass when valid version is given", tags: ["rolling_update"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose` - $?.success?.should be_true - response_s = `./cnf-testsuite rolling_update verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("rolling_update verbose") + result[:status].success?.should be_true + (/Passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") end end it "'rolling_update' should fail when invalid version is given", tags: ["rolling_update"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite rolling_update verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Failed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("rolling_update verbose") + result[:status].success?.should be_true + (/Failed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") end end it "'rolling_downgrade' should pass when valid version is given", tags: ["rolling_downgrade"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true retry_limit = 5 retries = 1 - response_s = "" - until (/Passed/ =~ response_s) || retries > retry_limit - LOGGING.info "rolling_downgrade retry: #{retries}" + until (/Passed/ =~ result[:output]) || retries > retry_limit + Log.info { "rolling_downgrade retry: #{retries}" } sleep 1.0 - response_s = `./cnf-testsuite rolling_downgrade verbose` + result = ShellCmd.run_testsuite("rolling_downgrade verbose") retries = retries + 1 end - LOGGING.info response_s - $?.success?.should be_true - (/Passed/ =~ response_s).should_not be_nil + Log.info { result[:output] } + result[:status].success?.should be_true + (/Passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") end end it "'rolling_downgrade' should fail when invalid version is given", tags: ["rolling_downgrade"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite rolling_downgrade verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Failed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("rolling_downgrade verbose") + result[:status].success?.should be_true + (/Failed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") end end it "'rolling_version_change' should pass when valid version is given", tags: ["rolling_version_change"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite rolling_version_change verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("rolling_version_change verbose") + result[:status].success?.should be_true + (/Passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") end end it "'rolling_version_change' should fail when invalid version is given", tags: ["rolling_version_change"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite rolling_version_change verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Failed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("rolling_version_change verbose") + result[:status].success?.should be_true + (/Failed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") end end it "'rollback' should pass ", tags: ["rollback"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite rollback verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("rollback verbose") + result[:status].success?.should be_true + (/Passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") end end @@ -216,292 +173,254 @@ describe CnfTestSuite do it "'nodeport_not_used' should fail when a node port is being used", tags: ["nodeport_not_used"] do begin - `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite nodeport_not_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: NodePort is being used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("nodeport_not_used verbose") + result[:status].success?.should be_true + (/FAILED: NodePort is being used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false") end end it "'nodeport_not_used' should pass when a node port is not being used", tags: ["nodeport_not_used"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite nodeport_not_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: NodePort is not used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("nodeport_not_used verbose") + result[:status].success?.should be_true + (/PASSED: NodePort is not used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") end end it "'hostport_not_used' should fail when a node port is being used", tags: ["hostport_not_used"] do begin - `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite hostport_not_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: HostPort is being used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("hostport_not_used verbose") + result[:status].success?.should be_true + (/FAILED: HostPort is being used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false") end end it "'hostport_not_used' should pass when a node port is not being used", tags: ["hostport_not_used"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite hostport_not_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: HostPort is not used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("hostport_not_used verbose") + result[:status].success?.should be_true + (/PASSED: HostPort is not used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") end end it "'ip_addresses' should pass when no uncommented ip addresses are found in helm chart source", tags: ["ip_addresses"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf-source/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite ip_addresses verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No IP addresses found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf-source/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("ip_addresses verbose") + result[:status].success?.should be_true + (/PASSED: No IP addresses found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite sample_coredns_source_cleanup verbose` + result = ShellCmd.run_testsuite("sample_coredns_source_cleanup verbose") end end it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should fail when a hardcoded ip is found in the K8s configuration", tags: ["ip_addresses"] do begin - `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false` - $?.success?.should be_true - response_s = `LOG_LEVEL=info ./cnf-testsuite hardcoded_ip_addresses_in_k8s_runtime_configuration verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Hard-coded IP addresses found in the runtime K8s configuration/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose", cmd_prefix: "LOG_LEVEL=info") + result[:status].success?.should be_true + (/FAILED: Hard-coded IP addresses found in the runtime K8s configuration/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false") end end it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should pass when no ip addresses are found in the K8s configuration", tags: ["ip_addresses"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite hardcoded_ip_addresses_in_k8s_runtime_configuration verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No hard-coded IP addresses found in the runtime K8s configuration/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose") + result[:status].success?.should be_true + (/PASSED: No hard-coded IP addresses found in the runtime K8s configuration/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") end end it "'secrets_used' should pass when secrets are provided as volumes and used by a container", tags: ["secrets"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose ` - $?.success?.should be_true - response_s = `./cnf-testsuite secrets_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Secrets defined and used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose ") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("secrets_used verbose") + result[:status].success?.should be_true + (/PASSED: Secrets defined and used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_secret_volume verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_secret_volume verbose") end end it "'secrets_used' should be skipped when secrets are provided as volumes and not mounted by a container", tags: ["secrets"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose wait_count=0 ` - $?.success?.should be_true - response_s = `./cnf-testsuite secrets_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/SKIPPED: Secrets not used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose wait_count=0 ") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("secrets_used verbose") + result[:status].success?.should be_true + (/SKIPPED: Secrets not used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_unmounted_secret_volume verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_unmounted_secret_volume verbose") end end it "'secrets_used' should pass when secrets are provided as environment variables and used by a container", tags: ["secrets"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose ` - $?.success?.should be_true - response_s = `./cnf-testsuite secrets_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Secrets defined and used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose ") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("secrets_used verbose") + result[:status].success?.should be_true + (/PASSED: Secrets defined and used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose") end end it "'secrets_used' should skip when secrets are not referenced as environment variables by a container", tags: ["secrets"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml wait_count=2 verbose` - $?.success?.should be_true - response_s = `./cnf-testsuite secrets_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/SKIPPED: Secrets not used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml wait_count=2 verbose") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("secrets_used verbose") + result[:status].success?.should be_true + (/SKIPPED: Secrets not used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose") end end it "'secrets_used' should be skipped when no secret volumes are mounted or no container secrets are provided (secrets ignored)`", tags: ["secrets"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0 ` - $?.success?.should be_true - response_s = `./cnf-testsuite secrets_used verbose` - LOGGING.info response_s - $?.success?.should be_true - (/SKIPPED: Secrets not used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0 ") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("secrets_used verbose") + result[:status].success?.should be_true + (/SKIPPED: Secrets not used/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns verbose") end end it "'immutable_configmap' fail with some mutable configmaps in container env or volume mount", tags: ["immutable_configmap"] do begin - `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite immutable_configmap verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found mutable configmap/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("immutable_configmap verbose") + result[:status].success?.should be_true + (/FAILED: Found mutable configmap/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false") end end it "'immutable_configmap' pass with all immutable configmaps in container env or volume mounts", tags: ["immutable_configmap"] do begin - `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite immutable_configmap verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: All volume or container mounted configmaps immutable/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("immutable_configmap verbose") + result[:status].success?.should be_true + (/PASSED: All volume or container mounted configmaps immutable/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false") end end it "'require_labels' should fail if a cnf does not have the app.kubernetes.io/name label", tags: ["require_labels"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite require_labels verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Pods should have the app.kubernetes.io\/name label/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("require_labels verbose") + result[:status].success?.should be_true + (/FAILED: Pods should have the app.kubernetes.io\/name label/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") end end it "'require_labels' should pass if a cnf has the app.kubernetes.io/name label", tags: ["require_labels"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite require_labels verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Pods have the app.kubernetes.io\/name label/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("require_labels verbose") + result[:status].success?.should be_true + (/PASSED: Pods have the app.kubernetes.io\/name label/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") end end it "'default_namespace' should fail if a cnf creates resources in the default namespace", tags: ["default_namespace"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns` - $?.success?.should be_true - response_s = `./cnf-testsuite default_namespace verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Resources are created in the default namespace/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("default_namespace verbose") + result[:status].success?.should be_true + (/FAILED: Resources are created in the default namespace/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns") KubectlClient::Utils.wait_for_terminations() end end it "'default_namespace' should pass if a cnf does not create resources in the default namespace", tags: ["default_namespace"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_latest_tag` - $?.success?.should be_true - response_s = `./cnf-testsuite default_namespace verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: default namespace is not being used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_latest_tag") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("default_namespace verbose") + result[:status].success?.should be_true + (/PASSED: default namespace is not being used/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag") KubectlClient::Utils.wait_for_terminations() end end it "'latest_tag' should fail if a cnf has containers that use images with the latest tag", tags: ["latest_tag"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_latest_tag` - $?.success?.should be_true - response_s = `./cnf-testsuite latest_tag verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Container images are using the latest tag/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_latest_tag") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("latest_tag verbose") + result[:status].success?.should be_true + (/FAILED: Container images are using the latest tag/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag") end end it "'latest_tag' should pass if a cnf does not have containers that use images with the latest tag", tags: ["latest_tag"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_nonroot` - $?.success?.should be_true - response_s = `./cnf-testsuite latest_tag verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Container images are not using the latest tag/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("latest_tag verbose") + result[:status].success?.should be_true + (/PASSED: Container images are not using the latest tag/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") end end it "'latest_tag' should require a cnf be installed to run", tags: ["latest_tag"] do begin - # LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_nonroot` # NOTE: Purposefully not installing a CNF to test - response_s = `./cnf-testsuite latest_tag verbose` - LOGGING.info response_s - $?.success?.should be_false - (/You must install a CNF first./ =~ response_s).should_not be_nil - ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` - end - end - - # Commenting because the test has now been marked as a POC - # - # it "'alpha_k8s_apis' should pass with a CNF that does not make use of alpha k8s APIs", tags: ["apisnoop"] do - # begin - # Log.info { `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample_coredns` } - # $?.success?.should be_true - # response_s = `./cnf-testsuite alpha_k8s_apis verbose` - # Log.info { response_s } - # $?.success?.should be_true - # (/PASSED: CNF does not use Kubernetes alpha APIs/ =~ response_s).should_not be_nil - # ensure - # Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_coredns` } - # end - # end - + result = ShellCmd.run_testsuite("latest_tag verbose") + result[:status].success?.should be_false + (/You must install a CNF first./ =~ result[:output]).should_not be_nil + ensure + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") + end + end end diff --git a/spec/workload/cpu_hog_spec.cr b/spec/workload/cpu_hog_spec.cr index ae3f1f577..32c4d73b8 100644 --- a/spec/workload/cpu_hog_spec.cr +++ b/spec/workload/cpu_hog_spec.cr @@ -1,27 +1,9 @@ -# require "../spec_helper" -# require "colorize" -# require "../../src/tasks/utils/utils.cr" -# require "helm" -# require "file_utils" -# require "sam" +require "../spec_helper" +require "colorize" +require "../../src/tasks/utils/utils.cr" +require "helm" +require "file_utils" +require "sam" -# describe "CPU_hog" do -# before_all do -# `./cnf-testsuite configuration_file_setup` -# $?.success?.should be_true -# end - -# it "'chaos_cpu_hog' A 'Good' CNF should not crash at 100% cpu usage" do -# begin -# `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` -# $?.success?.should be_true -# response_s = `./cnf-testsuite chaos_cpu_hog verbose` -# LOGGING.info response_s -# $?.success?.should be_true -# (/PASSED: Application pod is healthy after high CPU consumption/ =~ response_s).should_not be_nil -# ensure -# `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` -# $?.success?.should be_true -# end -# end -# end +describe "CPU_hog" do +end diff --git a/spec/workload/installability_spec.cr b/spec/workload/installability_spec.cr index 0671a8d03..50940a37d 100644 --- a/spec/workload/installability_spec.cr +++ b/spec/workload/installability_spec.cr @@ -3,82 +3,72 @@ require "colorize" describe CnfTestSuite do before_all do - `./cnf-testsuite samples_cleanup` - $?.success?.should be_true - LOGGING.info `./cnf-testsuite setup` + result = ShellCmd.run_testsuite("samples_cleanup") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("setup") end it "'helm_deploy' should fail on a bad helm chart", tags: ["helm"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample-bad-helm-deploy-repo verbose` - response_s = `./cnf-testsuite helm_deploy verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Helm deploy failed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-bad-helm-deploy-repo verbose") + result = ShellCmd.run_testsuite("helm_deploy verbose") + result[:status].success?.should be_true + (/FAILED: Helm deploy failed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-bad-helm-deploy-repo verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-bad-helm-deploy-repo verbose") end it "'helm_deploy' should fail if command is not supplied cnf-config argument", tags: ["helm"] do - response_s = `./cnf-testsuite helm_deploy` - LOGGING.info response_s - $?.success?.should be_true - (/No cnf_testsuite.yml found! Did you run the setup task/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("helm_deploy") + result[:status].success?.should be_true + (/No cnf_testsuite.yml found! Did you run the setup task/ =~ result[:output]).should_not be_nil end it "'helm_chart_valid' should pass on a good helm chart", tags: ["helm"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose` - $?.success?.should be_true - response_s = `./cnf-testsuite helm_chart_valid verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Lint Passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("helm_chart_valid verbose") + result[:status].success?.should be_true + (/Lint Passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") end it "'helm_chart_valid' should fail on a bad helm chart", tags: ["helm"] do - # LOGGING.debug `pwd` - # LOGGING.debug `echo $KUBECONFIG` begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite helm_chart_valid` - LOGGING.info response_s - $?.success?.should be_true - (/Lint Failed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("helm_chart_valid") + result[:status].success?.should be_true + (/Lint Failed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") end end it "'helm_chart_published' should pass on a good helm chart repo", tags: ["helm_chart_published"] do begin - `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite helm_chart_published` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Published Helm Chart Found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("helm_chart_published") + result[:status].success?.should be_true + (/PASSED: Published Helm Chart Found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-coredns-cnf") end end it "'helm_chart_published' should fail on a bad helm chart repo", tags: ["helm_chart_published"] do begin - LOGGING.info "search command: #{`helm search repo stable/coredns`}" - # LOGGING.info `#{Helm::BinarySingleton.helm} repo remove stable` - # LOGGING.info "search command: #{`helm search repo stable/coredns`}" - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-bad-helm-repo wait_count=0` - $?.success?.should be_false - LOGGING.info "search command: #{`helm search repo stable/coredns`}" - response_s = `./cnf-testsuite helm_chart_published verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Published Helm Chart Not Found/ =~ response_s).should_not be_nil + result = ShellCmd.run("helm search repo stable/coredns", force_output: true) + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-bad-helm-repo wait_count=0") + result[:status].success?.should be_false + result = ShellCmd.run("helm search repo stable/coredns", force_output: true) + result = ShellCmd.run_testsuite("helm_chart_published verbose") + result[:status].success?.should be_true + (/FAILED: Published Helm Chart Not Found/ =~ result[:output]).should_not be_nil ensure - `#{Helm::BinarySingleton.helm} repo remove badrepo` - `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-bad-helm-repo` + result = ShellCmd.run("#{Helm::BinarySingleton.helm} repo remove badrepo") + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-bad-helm-repo") end end end diff --git a/spec/workload/microservice_spec.cr b/spec/workload/microservice_spec.cr index 5c74f5316..4d186c852 100644 --- a/spec/workload/microservice_spec.cr +++ b/spec/workload/microservice_spec.cr @@ -10,68 +10,64 @@ require "sam" describe "Microservice" do before_all do Log.info { "Running testsuite setup" } - Log.info { `./cnf-testsuite setup` } - Log.info { `./cnf-testsuite setup` } - process_result = $?.success? + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("setup") + process_result = result[:status].success? Log.info(&.emit("Testsuite setup process result", process_result: process_result)) process_result.should be_true end it "'shared_database' should be skipped no MariaDB containers are found", tags: ["shared_database"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml` - response_s = `./cnf-testsuite shared_database` - LOGGING.info response_s - $?.success?.should be_true - (/N\/A: \[shared_database\] No MariaDB containers were found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("shared_database") + result[:status].success?.should be_true + (/N\/A: \[shared_database\] No MariaDB containers were found/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'shared_database' should pass if no database is used by two microservices", tags: ["shared_database"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml` - response_s = `./cnf-testsuite shared_database` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No shared database found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("shared_database") + result[:status].success?.should be_true + (/PASSED: No shared database found/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'shared_database' should pass if one service connects to a database but other non-service connections are made to the database", tags: ["shared_database"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml` - response_s = `./cnf-testsuite shared_database` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No shared database found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("shared_database") + result[:status].success?.should be_true + (/PASSED: No shared database found/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'shared_database' should fail if two services on the cluster connect to the same database", tags: ["shared_database2"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml` - response_s = `./cnf-testsuite shared_database` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found a shared database/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("shared_database") + result[:status].success?.should be_true + (/FAILED: Found a shared database/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'shared_database' should pass if two services on the cluster connect to the same database but they are not in the helm chart of the cnf", tags: ["shared_database"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_coredns` + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") Helm.install("multi-db sample-cnfs/ndn-multi-db-connections-fail/wordpress/") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress", wait_count: 180, namespace: "default") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress2", wait_count: 180, namespace: "default") @@ -80,80 +76,74 @@ describe "Microservice" do # todo run shared_database (should pass) # todo kubectl delete on ndn resourcws # toto cnf_cleanup on coredns - response_s = `./cnf-testsuite shared_database` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No shared database found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("shared_database") + result[:status].success?.should be_true + (/PASSED: No shared database found/ =~ result[:output]).should_not be_nil ensure Helm.delete("multi-db") KubectlClient::Delete.command("pvc data-multi-db-mariadb-0") - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'single_process_type' should pass if the containers in the cnf have only one process type", tags: ["process_check"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_coredns` - response_s = `./cnf-testsuite single_process_type verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Only one process type used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + result = ShellCmd.run_testsuite("single_process_type verbose") + result[:status].success?.should be_true + (/PASSED: Only one process type used/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") + result[:status].success?.should be_true end end it "'single_process_type' should fail if the containers in the cnf have more than one process type", tags: ["process_check"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/k8s-multiple-processes` - response_s = `./cnf-testsuite single_process_type verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: More than one process type used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/k8s-multiple-processes") + result = ShellCmd.run_testsuite("single_process_type verbose") + result[:status].success?.should be_true + (/FAILED: More than one process type used/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/k8s-multiple-processes` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/k8s-multiple-processes") + result[:status].success?.should be_true end end it "'single_process_type' should fail if the containers in the cnf have more than one process type and in a pod", tags: ["process_check"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-multiple-processes` - response_s = `./cnf-testsuite single_process_type verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: More than one process type used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-multiple-processes") + result = ShellCmd.run_testsuite("single_process_type verbose") + result[:status].success?.should be_true + (/FAILED: More than one process type used/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-multiple-processes` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-multiple-processes") + result[:status].success?.should be_true end end it "'reasonable_startup_time' should pass if the cnf has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do begin - Log.info {`./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_coredns`} - response_s = `./cnf-testsuite reasonable_startup_time verbose` - Log.info { response_s } + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + result = ShellCmd.run_testsuite("reasonable_startup_time verbose") - (/PASSED: CNF had a reasonable startup time/ =~ response_s).should_not be_nil + (/PASSED: CNF had a reasonable startup time/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") + result[:status].success?.should be_true end end it "'reasonable_startup_time' should fail if the cnf doesn't has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") begin - response_s = `./cnf-testsuite reasonable_startup_time verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: CNF had a startup time of/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("reasonable_startup_time verbose") + result[:status].success?.should be_true + (/FAILED: CNF had a startup time of/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") + result[:status].success?.should be_true end end @@ -163,68 +153,62 @@ describe "Microservice" do else cnf="./sample-cnfs/sample-coredns-cnf" end - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=#{cnf}` - response_s = `./cnf-testsuite reasonable_image_size verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Image size is good/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + result = ShellCmd.run_testsuite("reasonable_image_size verbose") + result[:status].success?.should be_true + (/Image size is good/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=#{cnf}` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf}") end it "'reasonable_image_size' should fail if image is larger than 5gb", tags: ["reasonable_image_size"] do - `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/ndn-reasonable-image-size wait_count=0` - response_s = `./cnf-testsuite reasonable_image_size verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Image size too large/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/ndn-reasonable-image-size wait_count=0") + result = ShellCmd.run_testsuite("reasonable_image_size verbose") + result[:status].success?.should be_true + (/Image size too large/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/ndn-reasonable-image-size force=true` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/ndn-reasonable-image-size force=true") end it "'specialized_init_system' should fail if pods do not use specialized init systems", tags: ["specialized_init_system"] do - `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample-coredns-cnf` - response_s = `./cnf-testsuite specialized_init_system` - LOGGING.info response_s - $?.success?.should be_true - (/Containers do not use specialized init systems/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-coredns-cnf") + result = ShellCmd.run_testsuite("specialized_init_system") + result[:status].success?.should be_true + (/Containers do not use specialized init systems/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-coredns-cnf force=true` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-coredns-cnf force=true") end it "'specialized_init_system' should pass if pods use specialized init systems", tags: ["specialized_init_system"] do - `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample-init-systems` - response_s = `./cnf-testsuite specialized_init_system` - LOGGING.info response_s - $?.success?.should be_true - (/Containers use specialized init systems/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-init-systems") + result = ShellCmd.run_testsuite("specialized_init_system") + result[:status].success?.should be_true + (/Containers use specialized init systems/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-init-systems force=true` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-init-systems force=true") end it "'service_discovery' should pass if any containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - Log.info { `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_coredns` } - response_s = `./cnf-testsuite service_discovery verbose` - Log.info { response_s } - $?.success?.should be_true - (/PASSED: Some containers exposed as a service/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + result = ShellCmd.run_testsuite("service_discovery verbose") + result[:status].success?.should be_true + (/PASSED: Some containers exposed as a service/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") + result[:status].success?.should be_true end end it "'service_discovery' should fail if no containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - Log.info { `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample-ndn-privileged` } - response_s = `./cnf-testsuite service_discovery verbose` - Log.info { response_s } - $?.success?.should be_true - (/FAILED: No containers exposed as a service/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-ndn-privileged") + result = ShellCmd.run_testsuite("service_discovery verbose") + result[:status].success?.should be_true + (/FAILED: No containers exposed as a service/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged") + result[:status].success?.should be_true end end @@ -235,14 +219,13 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - Log.info { `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample_good_signal_handling/` } - response_s = `./cnf-testsuite sig_term_handled verbose` - Log.info { response_s } - $?.success?.should be_true - (/PASSED: Sig Term handled/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_good_signal_handling/") + result = ShellCmd.run_testsuite("sig_term_handled verbose") + result[:status].success?.should be_true + (/PASSED: Sig Term handled/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling/` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling/") + result[:status].success?.should be_true end end @@ -253,14 +236,13 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - Log.info { `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample_bad_signal_handling/` } - response_s = `./cnf-testsuite sig_term_handled verbose` - Log.info { response_s } - $?.success?.should be_true - (/FAILED: Sig Term not handled/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_bad_signal_handling/") + result = ShellCmd.run_testsuite("sig_term_handled verbose") + result[:status].success?.should be_true + (/FAILED: Sig Term not handled/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_bad_signal_handling/` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_bad_signal_handling/") + result[:status].success?.should be_true end end @@ -271,45 +253,42 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - Log.info { `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample_good_signal_handling_tini/` } + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") # Workaround to wait using kubectl because Jenkins pod takes a LONG time to start. - Log.info { `kubectl wait --for=condition=ready=True pod/jenkins-0 -n cnfspace --timeout=500s` } + result = ShellCmd.run("kubectl wait --for=condition=ready=True pod/jenkins-0 -n cnfspace --timeout=500s", force_output: true) - response_s = `./cnf-testsuite sig_term_handled verbose` - Log.info { response_s } - $?.success?.should be_true - (/PASSED: Sig Term handled/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("sig_term_handled verbose") + result[:status].success?.should be_true + (/PASSED: Sig Term handled/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling_tini/` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") + result[:status].success?.should be_true end end it "'zombie_handled' should pass if a zombie is succesfully reaped by PID 1", tags: ["zombie"] do begin - Log.info { `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample_good_zombie_handling/` } - response_s = `./cnf-testsuite zombie_handled verbose` - Log.info { response_s } - $?.success?.should be_true - (/PASSED: Zombie handled/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_good_zombie_handling/") + result = ShellCmd.run_testsuite("zombie_handled verbose") + result[:status].success?.should be_true + (/PASSED: Zombie handled/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_good_zombie_handling/` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_good_zombie_handling/") + result[:status].success?.should be_true end end it "'zombie_handled' should failed if a zombie is not succesfully reaped by PID 1", tags: ["zombie"] do begin - Log.info { `./cnf-testsuite cnf_setup cnf-path=./sample-cnfs/sample-bad-zombie/` } - response_s = `./cnf-testsuite zombie_handled verbose` - Log.info { response_s } - $?.success?.should be_true - (/FAILED: Zombie not handled/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-bad-zombie/") + result = ShellCmd.run_testsuite("zombie_handled verbose") + result[:status].success?.should be_true + (/FAILED: Zombie not handled/ =~ result[:output]).should_not be_nil ensure - Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-bad-zombie/` } - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-bad-zombie/") + result[:status].success?.should be_true end end end diff --git a/spec/workload/observability_spec.cr b/spec/workload/observability_spec.cr index ba105c3f5..01ca84d80 100644 --- a/spec/workload/observability_spec.cr +++ b/spec/workload/observability_spec.cr @@ -6,36 +6,34 @@ require "../../src/tasks/jaeger_setup.cr" describe "Observability" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end it "'log_output' should pass with a cnf that outputs logs to stdout", tags: ["observability"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - response_s = `./cnf-testsuite log_output verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Resources output logs to stdout and stderr/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("log_output verbose") + result[:status].success?.should be_true + (/PASSED: Resources output logs to stdout and stderr/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") end end it "'log_output' should fail with a cnf that does not output logs to stdout", tags: ["observability"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml` - response_s = `./cnf-testsuite log_output verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Resources do not output logs to stdout and stderr/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("log_output verbose") + result[:status].success?.should be_true + (/FAILED: Resources do not output logs to stdout and stderr/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") end end it "'prometheus_traffic' should pass if there is prometheus traffic", tags: ["observability"] do - ShellCmd.run("./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml", "spec_sample_setup", force_output: true) + ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm Log.info { "Add prometheus helm repo" } @@ -48,166 +46,129 @@ describe "Observability" do KubectlClient::Get.wait_for_install("prometheus-server") ShellCmd.run("kubectl describe deployment prometheus-server", "k8s_describe_prometheus", force_output: true) - test_result = ShellCmd.run("./cnf-testsuite prometheus_traffic", "run_test_cmd", force_output: true) + test_result = ShellCmd.run_testsuite("prometheus_traffic") (/PASSED: Your cnf is sending prometheus traffic/ =~ test_result[:output]).should_not be_nil ensure - ShellCmd.run("./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml", "spec_sample_cleaup") + ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") result = ShellCmd.run("#{helm} delete prometheus", "helm_delete_prometheus") result[:status].success?.should be_true end it "'prometheus_traffic' should skip if there is no prometheus installed", tags: ["observability"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm - resp = `#{helm} delete prometheus` - LOGGING.info resp + result = ShellCmd.run("#{helm} delete prometheus", force_output: true) - response_s = `./cnf-testsuite prometheus_traffic` - LOGGING.info response_s - (/SKIPPED: Prometheus server not found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("prometheus_traffic") + (/SKIPPED: Prometheus server not found/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") end it "'prometheus_traffic' should fail if the cnf is not registered with prometheus", tags: ["observability"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm - LOGGING.info `helm repo add prometheus-community https://prometheus-community.github.io/helm-charts` - # resp = `#{helm} install prometheus prometheus-community/prometheus` - resp = `#{helm} install --set alertmanager.persistentVolume.enabled=false --set server.persistentVolume.enabled=false --set pushgateway.persistentVolume.enabled=false prometheus prometheus-community/prometheus` - LOGGING.info resp + result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) + result = ShellCmd.run("#{helm} install --set alertmanager.persistentVolume.enabled=false --set server.persistentVolume.enabled=false --set pushgateway.persistentVolume.enabled=false prometheus prometheus-community/prometheus", force_output: true) KubectlClient::Get.wait_for_install("prometheus-server") - LOGGING.info `kubectl describe deployment prometheus-server` + result = ShellCmd.run("kubectl describe deployment prometheus-server", force_output: true) #todo logging on prometheus pod - response_s = `./cnf-testsuite prometheus_traffic` - LOGGING.info response_s - (/FAILED: Your cnf is not sending prometheus traffic/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("prometheus_traffic") + (/FAILED: Your cnf is not sending prometheus traffic/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - resp = `#{helm} delete prometheus` - LOGGING.info resp - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result = ShellCmd.run("#{helm} delete prometheus", force_output: true) + result[:status].success?.should be_true end it "'open_metrics' should fail if there is not a valid open metrics response from the cnf", tags: ["observability"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml` - LOGGING.info `helm repo add prometheus-community https://prometheus-community.github.io/helm-charts` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm - # resp = `#{helm} install prometheus prometheus-community/prometheus` - resp = `#{helm} install --set alertmanager.persistentVolume.enabled=false --set server.persistentVolume.enabled=false --set pushgateway.persistentVolume.enabled=false prometheus prometheus-community/prometheus` - LOGGING.info resp + result = ShellCmd.run("#{helm} install --set alertmanager.persistentVolume.enabled=false --set server.persistentVolume.enabled=false --set pushgateway.persistentVolume.enabled=false prometheus prometheus-community/prometheus", force_output: true) KubectlClient::Get.wait_for_install("prometheus-server") - LOGGING.info `kubectl describe deployment prometheus-server` + result = ShellCmd.run("kubectl describe deployment prometheus-server", force_output: true) #todo logging on prometheus pod - response_s = `./cnf-testsuite open_metrics` - LOGGING.info response_s - (/FAILED: Your cnf's metrics traffic is not OpenMetrics compatible/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("open_metrics") + (/FAILED: Your cnf's metrics traffic is not OpenMetrics compatible/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml` - resp = `#{helm} delete prometheus` - LOGGING.info resp - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + result = ShellCmd.run("#{helm} delete prometheus", force_output: true) + result[:status].success?.should be_true end it "'open_metrics' should pass if there is a valid open metrics response from the cnf", tags: ["observability"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml` - LOGGING.info `helm repo add prometheus-community https://prometheus-community.github.io/helm-charts` - LOGGING.info "Installing prometheus server" + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") + result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) + Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm - # resp = `#{helm} install prometheus prometheus-community/prometheus` - resp = `#{helm} install --set alertmanager.persistentVolume.enabled=false --set server.persistentVolume.enabled=false --set pushgateway.persistentVolume.enabled=false prometheus prometheus-community/prometheus` - LOGGING.info resp + result = ShellCmd.run("#{helm} install --set alertmanager.persistentVolume.enabled=false --set server.persistentVolume.enabled=false --set pushgateway.persistentVolume.enabled=false prometheus prometheus-community/prometheus", force_output: true) KubectlClient::Get.wait_for_install("prometheus-server") - LOGGING.info `kubectl describe deployment prometheus-server` + result = ShellCmd.run("kubectl describe deployment prometheus-server", force_output: true) #todo logging on prometheus pod - response_s = `./cnf-testsuite open_metrics` - LOGGING.info response_s - (/PASSED: Your cnf's metrics traffic is OpenMetrics compatible/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("open_metrics") + (/PASSED: Your cnf's metrics traffic is OpenMetrics compatible/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml` - resp = `#{helm} delete prometheus` - LOGGING.info resp - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") + result = ShellCmd.run("#{helm} delete prometheus", force_output: true) + result[:status].success?.should be_true end - #09/27/23 fluentd/fluentd seems to be failing upstream. bitnami/fluentd works - # it "'routed_logs' should pass if cnfs logs are captured by fluentd", tags: ["observability"] do - # LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - # resp = `./cnf-testsuite install_fluentd` - # LOGGING.info resp - # response_s = `./cnf-testsuite routed_logs` - # LOGGING.info response_s - # (/PASSED: Your cnf's logs are being captured/ =~ response_s).should_not be_nil - # ensure - # LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - # resp = `./cnf-testsuite uninstall_fluentd` - # LOGGING.info resp - # $?.success?.should be_true - # end - it "'routed_logs' should pass if cnfs logs are captured by fluentd bitnami", tags: ["observability"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - resp = `./cnf-testsuite install_fluentdbitnami` - LOGGING.info resp - response_s = `./cnf-testsuite routed_logs` - LOGGING.info response_s - (/PASSED: Your cnf's logs are being captured/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("install_fluentdbitnami") + result = ShellCmd.run_testsuite("routed_logs") + (/PASSED: Your cnf's logs are being captured/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - resp = `./cnf-testsuite uninstall_fluentdbitnami` - LOGGING.info resp - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("uninstall_fluentdbitnami") + result[:status].success?.should be_true end it "'routed_logs' should pass if cnfs logs are captured by fluentbit", tags: ["observability"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-fluentbit` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-fluentbit") FluentBit.install - response_s = `./cnf-testsuite routed_logs` - LOGGING.info response_s - (/PASSED: Your cnf's logs are being captured/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("routed_logs") + (/PASSED: Your cnf's logs are being captured/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-fluentbit` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-fluentbit") FluentBit.uninstall - $?.success?.should be_true + result[:status].success?.should be_true end it "'routed_logs' should fail if cnfs logs are not captured", tags: ["observability"] do - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - # resp = `./cnf-testsuite install_fluentd` + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Helm.helm_repo_add("bitnami","oci://registry-1.docker.io/bitnamicharts") #todo #helm install --values ./override.yml fluentd ./fluentd Helm.install("--values ./spec/fixtures/fluentd-values-bad.yml -n #{TESTSUITE_NAMESPACE} fluentd bitnami/fluentd") - Log.info {"Installing FluentD daemonset "} + Log.info { "Installing FluentD daemonset " } KubectlClient::Get.resource_wait_for_install("Daemonset", "fluentd", namespace: TESTSUITE_NAMESPACE) - response_s = `./cnf-testsuite routed_logs` - LOGGING.info response_s - (/FAILED: Your cnf's logs are not being captured/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("routed_logs") + (/FAILED: Your cnf's logs are not being captured/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - resp = `./cnf-testsuite uninstall_fluentd` - LOGGING.info resp - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("uninstall_fluentd") + result[:status].success?.should be_true end it "'tracing' should fail if tracing is not used", tags: ["observability_jaeger_fail"] do - Log.info {"Installing Jaeger "} + Log.info { "Installing Jaeger " } JaegerManager.install - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - response_s = `./cnf-testsuite tracing` - LOGGING.info response_s - (/FAILED: Tracing not used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("tracing") + (/FAILED: Tracing not used/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") JaegerManager.uninstall KubectlClient::Get.resource_wait_for_uninstall("Statefulset", "jaeger-cassandra") KubectlClient::Get.resource_wait_for_uninstall("Deployment", "jaeger-collector") @@ -216,15 +177,14 @@ describe "Observability" do end it "'tracing' should pass if tracing is used", tags: ["observability_jaeger_pass"] do - Log.info {"Installing Jaeger "} + Log.info { "Installing Jaeger " } JaegerManager.install - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml` - response_s = `./cnf-testsuite tracing` - LOGGING.info response_s - (/PASSED: Tracing used/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml") + result = ShellCmd.run_testsuite("tracing") + (/PASSED: Tracing used/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml") JaegerManager.uninstall KubectlClient::Get.resource_wait_for_uninstall("Statefulset", "jaeger-cassandra") KubectlClient::Get.resource_wait_for_uninstall("Deployment", "jaeger-collector") diff --git a/spec/workload/operator_spec.cr b/spec/workload/operator_spec.cr index 3ff702e32..5221a61d3 100644 --- a/spec/workload/operator_spec.cr +++ b/spec/workload/operator_spec.cr @@ -17,20 +17,19 @@ describe "Operator" do Log.info { "OLM already installed. Skipping git clone for OLM." } else GitClient.clone("https://github.com/operator-framework/operator-lifecycle-manager.git #{install_dir}") - `cd #{install_dir} && git fetch -a && git checkout tags/v0.22.0 && cd -` + result = ShellCmd.run("cd #{install_dir} && git fetch -a && git checkout tags/v0.22.0 && cd -") end Helm.install("operator --set olm.image.ref=quay.io/operator-framework/olm:v0.22.0 --set catalog.image.ref=quay.io/operator-framework/olm:v0.22.0 --set package.image.ref=quay.io/operator-framework/olm:v0.22.0 #{install_dir}/deploy/chart/") begin - LOGGING.info `./cnf-testsuite -l info cnf_setup cnf-path=./sample-cnfs/sample_operator` - $?.success?.should be_true - resp = `./cnf-testsuite -l info operator_installed` - Log.info { "#{resp}" } - (/PASSED: Operator is installed/ =~ resp).should_not be_nil + result = ShellCmd.run_testsuite("-l info cnf_setup cnf-path=./sample-cnfs/sample_operator") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("-l info operator_installed") + (/PASSED: Operator is installed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite -l info cnf_cleanup cnf-path=./sample-cnfs/sample_operator` - $?.success?.should be_true + result = ShellCmd.run_testsuite("-l info cnf_cleanup cnf-path=./sample-cnfs/sample_operator") + result[:status].success?.should be_true pods = KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("catalog-operator", "operator-lifecycle-manager"), "operator-lifecycle-manager") + KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("olm-operator", "operator-lifecycle-manager"), "operator-lifecycle-manager") + KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("packageserver", "operator-lifecycle-manager"), "operator-lifecycle-manager") Helm.uninstall("operator") @@ -81,14 +80,13 @@ describe "Operator" do it "'operator_test' operator should not be found", tags: ["operator_test"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_coredns` - $?.success?.should be_true - resp = `./cnf-testsuite -l info operator_installed` - Log.info { "#{resp}" } - (/N\/A: No Operators Found/ =~ resp).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("-l info operator_installed") + (/N\/A: No Operators Found/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") + result[:status].success?.should be_true end end end diff --git a/spec/workload/registry_spec.cr b/spec/workload/registry_spec.cr index a31ad3ed7..cf9bb5b9a 100644 --- a/spec/workload/registry_spec.cr +++ b/spec/workload/registry_spec.cr @@ -13,7 +13,7 @@ end describe "Private Registry: Image" do before_all do - `./cnf-testsuite setup` + result = ShellCmd.run_testsuite("setup") Dockerd.install install_registry = KubectlClient::Apply.file(registry_manifest_path) KubectlClient::Get.resource_wait_for_install("Pod", "registry") @@ -42,25 +42,23 @@ describe "Private Registry: Image" do it "'reasonable_image_size' should pass if using local registry and a port", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry" - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=#{cnf}` - response_s = `./cnf-testsuite reasonable_image_size verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Image size is good/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + result = ShellCmd.run_testsuite("reasonable_image_size verbose") + result[:status].success?.should be_true + (/Image size is good/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=#{cnf}` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf}") end it "'reasonable_image_size' should pass if using local registry, a port and an org", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry_org_image" - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=#{cnf}` - response_s = `./cnf-testsuite reasonable_image_size verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Image size is good/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + result = ShellCmd.run_testsuite("reasonable_image_size verbose") + result[:status].success?.should be_true + (/Image size is good/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=#{cnf}` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf}") end after_all do @@ -71,7 +69,7 @@ end describe "Private Registry: Rolling" do before_all do - `./cnf-testsuite setup` + result = ShellCmd.run_testsuite("setup") Dockerd.install install_registry = KubectlClient::Apply.file(registry_manifest_path) KubectlClient::Get.resource_wait_for_install("Pod", "registry") @@ -93,13 +91,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=#{cnf}` - response_s = `./cnf-testsuite rolling_update verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + result = ShellCmd.run_testsuite("rolling_update verbose") + result[:status].success?.should be_true + (/Passed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=#{cnf} wait_count=0` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf} wait_count=0") end end @@ -107,13 +104,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=#{cnf}` - response_s = `./cnf-testsuite rolling_update verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + result = ShellCmd.run_testsuite("rolling_update verbose") + result[:status].success?.should be_true + (/Passed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=#{cnf} wait_count=0` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf} wait_count=0") end end @@ -121,13 +117,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - LOGGING.info `./cnf-testsuite cnf_setup cnf-path=#{cnf}` - response_s = `./cnf-testsuite rolling_version_change verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + result = ShellCmd.run_testsuite("rolling_version_change verbose") + result[:status].success?.should be_true + (/Passed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=#{cnf} wait_count=0` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf} wait_count=0") end end diff --git a/spec/workload/resilience/container_chaos_spec.cr b/spec/workload/resilience/container_chaos_spec.cr index c929798e9..a34610e4f 100644 --- a/spec/workload/resilience/container_chaos_spec.cr +++ b/spec/workload/resilience/container_chaos_spec.cr @@ -1,43 +1,9 @@ -# require "../../spec_helper" -# require "colorize" -# require "../../../src/tasks/utils/utils.cr" -# require "helm" -# require "file_utils" -# require "sam" +require "../../spec_helper" +require "colorize" +require "../../../src/tasks/utils/utils.cr" +require "helm" +require "file_utils" +require "sam" -# describe "Resilience Container Chaos" do -# before_all do -# `./cnf-testsuite setup` -# `./cnf-testsuite configuration_file_setup` -# $?.success?.should be_true -# end - -# it "'chaos_container_kill' A 'Good' CNF should recover when its container is killed" do -# begin -# `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` -# $?.success?.should be_true -# response_s = `./cnf-testsuite chaos_container_kill verbose` -# LOGGING.info response_s -# $?.success?.should be_true -# (/PASSED: Replicas available match desired count after container kill test/ =~ response_s).should_not be_nil -# ensure -# `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` -# $?.success?.should be_true -# end -# end - - # TODO upgrade chaos mesh - # it "'chaos_container_kill' A 'Bad' CNF should NOT recover when its container is killed" do - # begin - # `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample-fragile-state deploy_with_chart=false` - # $?.success?.should be_true - # response_s = `./cnf-testsuite chaos_container_kill verbose` - # LOGGING.info response_s - # $?.success?.should be_true - # (/FAILED: Replicas did not return desired count after container kill test/ =~ response_s).should_not be_nil - # ensure - # `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-fragile-state deploy_with_chart=false` - # $?.success?.should be_true - # end - # end -# end +describe "Resilience Container Chaos" do +end diff --git a/spec/workload/resilience/disk_fill_spec.cr b/spec/workload/resilience/disk_fill_spec.cr index 6096fcf2e..a2f1e5a16 100644 --- a/spec/workload/resilience/disk_fill_spec.cr +++ b/spec/workload/resilience/disk_fill_spec.cr @@ -7,24 +7,23 @@ require "sam" describe "Resilience Disk Fill Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'disk_fill' A 'Good' CNF should not crash when disk fill occurs", tags: ["disk_fill"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite disk_fill verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: disk_fill chaos test passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("disk_fill verbose") + result[:status].success?.should be_true + (/PASSED: disk_fill chaos test passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml wait_count=0` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/resilience/network_chaos_spec.cr b/spec/workload/resilience/network_chaos_spec.cr index 5b8df4ea6..84812e28e 100644 --- a/spec/workload/resilience/network_chaos_spec.cr +++ b/spec/workload/resilience/network_chaos_spec.cr @@ -7,38 +7,8 @@ require "sam" describe "Resilience Network Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end - - #TODO Fix Chaos Network Loss Test - # it "'chaos_network_loss' A 'Good' CNF should not crash when network loss occurs", do - # begin - # `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - # $?.success?.should be_true - # response_s = `./cnf-testsuite chaos_network_loss verbose` - # LOGGING.info response_s - # $?.success?.should be_true - # (/PASSED: Replicas available match desired count after network chaos test/ =~ response_s).should_not be_nil - # ensure - # `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - # $?.success?.should be_true - # end - # end - - #TODO upgrade chaos mesh - # it "'chaos_network_loss' A 'Bad' CNF should crash when network loss occurs", do - # begin - # `./cnf-testsuite cnf_setup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false wait_count=60` - # $?.success?.should be_true - # response_s = `./cnf-testsuite chaos_network_loss verbose` - # LOGGING.info response_s - # $?.success?.should be_true - # (/FAILED: Replicas did not return desired count after network chaos test/ =~ response_s).should_not be_nil - # ensure - # `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_network_loss deploy_with_chart=false` - # $?.success?.should be_true - # end - # end end diff --git a/spec/workload/resilience/node_drain_spec.cr b/spec/workload/resilience/node_drain_spec.cr index 24b694399..57aba9fc0 100644 --- a/spec/workload/resilience/node_drain_spec.cr +++ b/spec/workload/resilience/node_drain_spec.cr @@ -7,28 +7,27 @@ require "sam" describe "Resilience Node Drain Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'node_drain' A 'Good' CNF should not crash when node drain occurs", tags: ["node_drain"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite node_drain verbose` - LOGGING.info response_s - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("node_drain verbose") + result[:status].success?.should be_true if KubectlClient::Get.schedulable_nodes_list.size > 1 - (/PASSED: node_drain chaos test passed/ =~ response_s).should_not be_nil + (/PASSED: node_drain chaos test passed/ =~ result[:output]).should_not be_nil else - (/SKIPPED: node_drain chaos test requires the cluster to have atleast two/ =~ response_s).should_not be_nil + (/SKIPPED: node_drain chaos test requires the cluster to have atleast two/ =~ result[:output]).should_not be_nil end ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/resilience/pod_delete_spec.cr b/spec/workload/resilience/pod_delete_spec.cr index b14738664..f0556b1bc 100644 --- a/spec/workload/resilience/pod_delete_spec.cr +++ b/spec/workload/resilience/pod_delete_spec.cr @@ -7,24 +7,23 @@ require "sam" describe "Resilience pod delete Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'pod_delete' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_delete"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite pod_delete verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: pod_delete chaos test passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("pod_delete verbose") + result[:status].success?.should be_true + (/PASSED: pod_delete chaos test passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/resilience/pod_dns_error_spec.cr b/spec/workload/resilience/pod_dns_error_spec.cr index a3054b152..fecbb335a 100644 --- a/spec/workload/resilience/pod_dns_error_spec.cr +++ b/spec/workload/resilience/pod_dns_error_spec.cr @@ -7,29 +7,28 @@ require "sam" describe "Resilience pod dns error Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'pod_dns_error' A 'Good' CNF should not crash when pod dns error occurs", tags: ["pod_dns_error"] do begin - `./cnf-testsuite cnf_setup cnf-config=example-cnfs/envoy/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite pod_dns_error verbose` - Log.info { response_s } - $?.success?.should be_true - ((/SKIPPED: pod_dns_error docker runtime not found/) =~ response_s || - (/PASSED: pod_dns_error chaos test passed/ =~ response_s)).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=example-cnfs/envoy/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("pod_dns_error verbose") + result[:status].success?.should be_true + ((/SKIPPED: pod_dns_error docker runtime not found/) =~ result[:output] || + (/PASSED: pod_dns_error chaos test passed/ =~ result[:output])).should_not be_nil rescue ex # Raise back error to ensure test fails. # The ensure block will cleanup the CNF and the litmus installation. raise "Test failed with #{ex.message}" ensure - `./cnf-testsuite cnf_cleanup cnf-config=example-cnfs/envoy/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=example-cnfs/envoy/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/resilience/pod_io_stress_spec.cr b/spec/workload/resilience/pod_io_stress_spec.cr index da057d46e..d7f6d34e1 100644 --- a/spec/workload/resilience/pod_io_stress_spec.cr +++ b/spec/workload/resilience/pod_io_stress_spec.cr @@ -7,24 +7,23 @@ require "sam" describe "Resilience pod delete Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'pod_io_stress' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_io_stress"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite pod_io_stress verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: pod_io_stress chaos test passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("pod_io_stress verbose") + result[:status].success?.should be_true + (/PASSED: pod_io_stress chaos test passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/resilience/pod_memory_hog_spec.cr b/spec/workload/resilience/pod_memory_hog_spec.cr index 5c2622c3d..0a0152a2b 100644 --- a/spec/workload/resilience/pod_memory_hog_spec.cr +++ b/spec/workload/resilience/pod_memory_hog_spec.cr @@ -7,24 +7,23 @@ require "sam" describe "Resilience pod memory hog Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'pod_memory_hog' A 'Good' CNF should not crash when pod memory hog occurs", tags: ["pod_memory_hog"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite pod_memory_hog verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: pod_memory_hog chaos test passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("pod_memory_hog verbose") + result[:status].success?.should be_true + (/PASSED: pod_memory_hog chaos test passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end \ No newline at end of file diff --git a/spec/workload/resilience/pod_network_corruption_spec.cr b/spec/workload/resilience/pod_network_corruption_spec.cr index 178f6acd8..c6eac74ec 100644 --- a/spec/workload/resilience/pod_network_corruption_spec.cr +++ b/spec/workload/resilience/pod_network_corruption_spec.cr @@ -7,24 +7,23 @@ require "sam" describe "Resilience Pod Network corruption Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'pod_network_corruption' A 'Good' CNF should not crash when network corruption occurs", tags: ["pod_network_corruption"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite pod_network_corruption verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: pod_network_corruption chaos test passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("pod_network_corruption verbose") + result[:status].success?.should be_true + (/PASSED: pod_network_corruption chaos test passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/resilience/pod_network_duplication_spec.cr b/spec/workload/resilience/pod_network_duplication_spec.cr index af2af0a89..07b6501be 100644 --- a/spec/workload/resilience/pod_network_duplication_spec.cr +++ b/spec/workload/resilience/pod_network_duplication_spec.cr @@ -7,24 +7,23 @@ require "sam" describe "Resilience Pod Network duplication Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'pod_network_duplication' A 'Good' CNF should not crash when network duplication occurs", tags: ["pod_network_duplication"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite pod_network_duplication verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: pod_network_duplication chaos test passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("pod_network_duplication verbose") + result[:status].success?.should be_true + (/PASSED: pod_network_duplication chaos test passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/resilience/pod_network_latency_spec.cr b/spec/workload/resilience/pod_network_latency_spec.cr index ebe32b066..4c9eae7c3 100644 --- a/spec/workload/resilience/pod_network_latency_spec.cr +++ b/spec/workload/resilience/pod_network_latency_spec.cr @@ -7,24 +7,23 @@ require "sam" describe "Resilience Pod Network Latency Chaos" do before_all do - `./cnf-testsuite setup` - `./cnf-testsuite configuration_file_setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("configuration_file_setup") + result[:status].success?.should be_true end it "'pod_network_latency' A 'Good' CNF should not crash when network latency occurs", tags: ["pod_network_latency"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite pod_network_latency verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: pod_network_latency chaos test passed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("pod_network_latency verbose") + result[:status].success?.should be_true + (/PASSED: pod_network_latency chaos test passed/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - `./cnf-testsuite uninstall_litmus` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("uninstall_litmus") + result[:status].success?.should be_true end end end diff --git a/spec/workload/scalability_spec.cr b/spec/workload/scalability_spec.cr index 994214766..27f2f2b67 100644 --- a/spec/workload/scalability_spec.cr +++ b/spec/workload/scalability_spec.cr @@ -4,26 +4,23 @@ require "colorize" describe "Scalability" do before_all do - `./cnf-testsuite setup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("setup") + result[:status].success?.should be_true end after_all do - `./cnf-testsuite samples_cleanup` - $?.success?.should be_true + result = ShellCmd.run_testsuite("samples_cleanup") + result[:status].success?.should be_true end it "'scalability' should run all of the scalability tests", tags: "[scalability]" do - `./cnf-testsuite samples_cleanup` - response_s = `./cnf-testsuite setup` - LOGGING.info response_s - # `./cnf-testsuite sample_coredns_with_wait_setup` - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose` - $?.success?.should be_true - response_s = `./cnf-testsuite scalability` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Replicas increased to 3/ =~ response_s).should_not be_nil - (/PASSED: Replicas decreased to 1/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("samples_cleanup") + result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("scalability") + result[:status].success?.should be_true + (/PASSED: Replicas increased to 3/ =~ result[:output]).should_not be_nil + (/PASSED: Replicas decreased to 1/ =~ result[:output]).should_not be_nil end end diff --git a/spec/workload/security_spec.cr b/spec/workload/security_spec.cr index 64d802746..b24089969 100644 --- a/spec/workload/security_spec.cr +++ b/spec/workload/security_spec.cr @@ -6,406 +6,377 @@ describe "Security" do it "'privileged' should pass with a non-privileged cnf", tags: ["privileged"] do begin - LOGGING.debug `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml` - response_s = `./cnf-testsuite privileged verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Found.*privileged containers.*coredns/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + Log.debug { result[:output] } + result = ShellCmd.run_testsuite("privileged verbose") + result[:status].success?.should be_true + (/Found.*privileged containers.*coredns/ =~ result[:output]).should be_nil ensure - LOGGING.debug `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + Log.debug { result[:output] } end end it "'privileged' should fail on a non-whitelisted, privileged cnf", tags: ["privileged"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite privileged verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Found.*privileged containers.*/ =~ response_s).should_not be_nil - (/Privileged container (privileged-coredns) in.*/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("privileged verbose") + result[:status].success?.should be_true + (/Found.*privileged containers.*/ =~ result[:output]).should_not be_nil + (/Privileged container (privileged-coredns) in.*/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite sample_privileged_cnf_non_whitelisted_cleanup` + result = ShellCmd.run_testsuite("sample_privileged_cnf_non_whitelisted_cleanup") end end it "'privileged' should pass on a whitelisted, privileged cnf", tags: ["privileged"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose wait_count=0` - $?.success?.should be_true - response_s = `./cnf-testsuite privileged cnf-config=sample-cnfs/sample_whitelisted_privileged_cnf verbose` - LOGGING.info response_s - $?.success?.should be_true - (/Found.*privileged containers.*/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose wait_count=0") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("privileged cnf-config=sample-cnfs/sample_whitelisted_privileged_cnf verbose") + result[:status].success?.should be_true + (/Found.*privileged containers.*/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite sample_privileged_cnf_whitelisted_cleanup` + result = ShellCmd.run_testsuite("sample_privileged_cnf_whitelisted_cleanup") end end it "'privilege_escalation' should fail on a cnf that has escalated privileges", tags: ["privileged"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite privilege_escalation` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No containers that allow privilege escalation were found/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("privilege_escalation") + result[:status].success?.should be_true + (/PASSED: No containers that allow privilege escalation were found/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") end end it "'privilege_escalation' should pass on a cnf that does not have escalated privileges", tags: ["privileged"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite privilege_escalation` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No containers that allow privilege escalation were found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("privilege_escalation") + result[:status].success?.should be_true + (/PASSED: No containers that allow privilege escalation were found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") end end it "'symlink_file_system' should pass on a cnf that does not allow a symlink attack", tags: ["capabilities"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite symlink_file_system` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No containers allow a symlink attack/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("symlink_file_system") + result[:status].success?.should be_true + (/PASSED: No containers allow a symlink attack/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") end end it "'insecure_capabilities' should pass on a cnf that does not have containers with insecure capabilities", tags: ["capabilities"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite insecure_capabilities` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Containers with insecure capabilities were not found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("insecure_capabilities") + result[:status].success?.should be_true + (/PASSED: Containers with insecure capabilities were not found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") end end it "'insecure_capabilities' should fail on a cnf that has containers with insecure capabilities", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite insecure_capabilities` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Containers with insecure capabilities were not found/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("insecure_capabilities") + result[:status].success?.should be_true + (/PASSED: Containers with insecure capabilities were not found/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") end end it "'linux_hardening' should fail on a cnf that does not make use of security services", tags: ["capabilities"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite linux_hardening` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Security services are being used to harden applications/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("linux_hardening") + result[:status].success?.should be_true + (/PASSED: Security services are being used to harden applications/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") end end it "'application_credentials' should fail on a cnf that allows applications credentials in configuration files", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite application_credentials` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found applications credentials in configuration files/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("application_credentials") + result[:status].success?.should be_true + (/FAILED: Found applications credentials in configuration files/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") end end it "'host_network' should pass on a cnf that does not have a host network attached to pod", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite host_network` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No host network attached to pod/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("host_network") + result[:status].success?.should be_true + (/PASSED: No host network attached to pod/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") end end it "'service_account_mapping' should fail on a cnf that automatically maps the service account", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite service_account_mapping` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Service accounts automatically mapped/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("service_account_mapping") + result[:status].success?.should be_true + (/FAILED: Service accounts automatically mapped/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") end end it "'resource_policies' should pass on a cnf that has containers with resource limits defined", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite resource_policies` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Containers have resource limits defined/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("resource_policies") + result[:status].success?.should be_true + (/PASSED: Containers have resource limits defined/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") end end it "'ingress_egress_blocked' should fail on a cnf that has no ingress and egress traffic policy", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite ingress_egress_blocked` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Ingress and Egress traffic blocked on pods/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("ingress_egress_blocked") + result[:status].success?.should be_true + (/PASSED: Ingress and Egress traffic blocked on pods/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") end end it "'host_pid_ipc_privileges' should pass on a cnf that does not have containers with host PID/IPC privileges", tags: ["capabilities"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite host_pid_ipc_privileges` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found containers with hostPID and hostIPC privileges/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("host_pid_ipc_privileges") + result[:status].success?.should be_true + (/FAILED: Found containers with hostPID and hostIPC privileges/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") end end it "'non_root_containers' should pass on a cnf that does not have containers running with root user or user with root group memberships", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-nonroot-containers` - $?.success?.should be_true - response_s = `./cnf-testsuite non_root_containers` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found containers running with root user or user with root group membership/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-nonroot-containers") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("non_root_containers") + result[:status].success?.should be_true + (/FAILED: Found containers running with root user or user with root group membership/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot-containers` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot-containers") end end it "'non_root_containers' should fail on a cnf that has containers running with root user or user with root group memberships", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite non_root_containers` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Containers are running with non-root user with non-root group membership/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("non_root_containers") + result[:status].success?.should be_true + (/PASSED: Containers are running with non-root user with non-root group membership/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") end end it "'privileged_containers' should pass when the cnf has no privileged containers", tags: ["privileged"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite privileged_containers` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found privileged containers/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("privileged_containers") + result[:status].success?.should be_true + (/FAILED: Found privileged containers/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") end end it "'immutable_file_systems' should fail when the cnf containers with mutable file systems", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true - response_s = `./cnf-testsuite immutable_file_systems` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Containers have immutable file systems/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("immutable_file_systems") + result[:status].success?.should be_true + (/PASSED: Containers have immutable file systems/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") end end it "'immutable_file_systems' should pass when the cnf containers with immutable file systems", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-immutable-fs` - $?.success?.should be_true - response_s = `./cnf-testsuite immutable_file_systems` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Containers have immutable file systems/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-immutable-fs") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("immutable_file_systems") + result[:status].success?.should be_true + (/PASSED: Containers have immutable file systems/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-immutable-fs` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-immutable-fs") end end it "'hostpath_mounts' should pass when the cnf has no containers with hostPath mounts", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") + result[:status].success?.should be_true ClusterTools.uninstall - response_s = `./cnf-testsuite hostpath_mounts` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found containers with hostPath mounts/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("hostpath_mounts") + result[:status].success?.should be_true + (/FAILED: Found containers with hostPath mounts/ =~ result[:output]).should be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") ClusterTools.install end end it "'hostpath_mounts' should fail when the cnf has containers with hostPath mounts", tags: ["security"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-hostpath` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-hostpath") + result[:status].success?.should be_true ClusterTools.uninstall - response_s = `./cnf-testsuite hostpath_mounts` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Found containers with hostPath mounts/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("hostpath_mounts") + result[:status].success?.should be_true + (/FAILED: Found containers with hostPath mounts/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-hostpath` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-hostpath") ClusterTools.install end end it "'container_sock_mounts' should pass if a cnf has no pods that mount container engine socket", tags: ["container_sock_mounts"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite container_sock_mounts verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Container engine daemon sockets are not mounted as volumes/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("container_sock_mounts verbose") + result[:status].success?.should be_true + (/PASSED: Container engine daemon sockets are not mounted as volumes/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") end end it "'container_sock_mounts' should fail if the CNF has pods with container engine sockets mounted", tags: ["container_sock_mounts"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite container_sock_mounts verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Container engine daemon sockets are mounted as volumes/ =~ response_s).should_not be_nil - (/Unix socket is not allowed/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("container_sock_mounts verbose") + result[:status].success?.should be_true + (/FAILED: Container engine daemon sockets are mounted as volumes/ =~ result[:output]).should_not be_nil + (/Unix socket is not allowed/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") end end it "'external_ips' should pass if a cnf has no services with external IPs", tags: ["external_ips"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite external_ips verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Services are not using external IPs/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("external_ips verbose") + result[:status].success?.should be_true + (/PASSED: Services are not using external IPs/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") end end it "'external_ips' should fail if a cnf has services with external IPs", tags: ["external_ips"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite external_ips verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Services are using external IPs/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("external_ips verbose") + result[:status].success?.should be_true + (/FAILED: Services are using external IPs/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") end end it "'selinux_options' should fail if containers have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_latest_tag` - $?.success?.should be_true - response_s = `./cnf-testsuite selinux_options verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Pods are using custom SELinux options that can be used for privilege escalations/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_latest_tag") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("selinux_options verbose") + result[:status].success?.should be_true + (/FAILED: Pods are using custom SELinux options that can be used for privilege escalations/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag") end end it "'selinux_options' should be skipped if containers do not use custom selinux options", tags: ["selinux_options"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_nonroot` - $?.success?.should be_true - response_s = `./cnf-testsuite selinux_options verbose` - LOGGING.info response_s - $?.success?.should be_true - (/N\/A: Pods are not using SELinux/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("selinux_options verbose") + result[:status].success?.should be_true + (/N\/A: Pods are not using SELinux/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") end end it "'selinux_options' should pass if containers do not have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_valid_selinux_options` - $?.success?.should be_true - response_s = `./cnf-testsuite selinux_options verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: Pods are not using custom SELinux options that can be used for privilege escalations/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_valid_selinux_options") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("selinux_options verbose") + result[:status].success?.should be_true + (/PASSED: Pods are not using custom SELinux options that can be used for privilege escalations/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_valid_selinux_options` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_valid_selinux_options") end end it "'sysctls' should fail if Pods have restricted sysctls values", tags: ["sysctls"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_sysctls` - $?.success?.should be_true - response_s = `./cnf-testsuite sysctls verbose` - LOGGING.info response_s - $?.success?.should be_true - (/FAILED: Restricted values for are being used for sysctls/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_sysctls") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("sysctls verbose") + result[:status].success?.should be_true + (/FAILED: Restricted values for are being used for sysctls/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_sysctls` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_sysctls") end end it "'sysctls' should pass if Pods have allowed sysctls values", tags: ["sysctls"] do begin - LOGGING.info `./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample_nonroot` - $?.success?.should be_true - response_s = `./cnf-testsuite sysctls verbose` - LOGGING.info response_s - $?.success?.should be_true - (/PASSED: No restricted values found for sysctls/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("sysctls verbose") + result[:status].success?.should be_true + (/PASSED: No restricted values found for sysctls/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") end end end diff --git a/spec/workload/state_spec.cr b/spec/workload/state_spec.cr index 2c619c9ca..75ee83bf9 100644 --- a/spec/workload/state_spec.cr +++ b/spec/workload/state_spec.cr @@ -9,19 +9,18 @@ require "sam" describe "State" do before_all do - `./cnf-testsuite configuration_file_setup` + result = ShellCmd.run_testsuite("configuration_file_setup") end it "'elastic_volumes' should fail if the cnf does not use volumes that are elastic volume", tags: ["elastic_volume"] do begin - LOGGING.info `./cnf-testsuite -l info cnf_setup cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite -l info elastic_volumes verbose` - LOGGING.info "Status: #{response_s}" - (/PASSED: All used volumes are elastic/ =~ response_s).should be_nil + result = ShellCmd.run_testsuite("-l info cnf_setup cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("-l info elastic_volumes verbose") + (/PASSED: All used volumes are elastic/ =~ result[:output]).should be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml") + result[:status].success?.should be_true end end @@ -29,78 +28,64 @@ describe "State" do # This CNF does not use any volumes except the ones that Kubernetes might mount by default (like the service account token) it "'elastic_volumes' should fail if the cnf does not use any elastic volumes", tags: ["elastic_volume"] do begin - LOGGING.info `./cnf-testsuite -l info cnf_setup cnf-config=./sample-cnfs/sample_nonroot` - $?.success?.should be_true - response_s = `./cnf-testsuite -l info elastic_volumes verbose` - LOGGING.info "Status: #{response_s}" - (/FAILED/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("-l info cnf_setup cnf-config=./sample-cnfs/sample_nonroot") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("-l info elastic_volumes verbose") + (/FAILED/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") + result[:status].success?.should be_true end end it "'database_persistence' should pass if the cnf uses a database that uses an elastic volume with a stateful set", tags: ["elastic_volume"] do begin - Log.info {"Installing Mysql "} - # Mysql.install + Log.info { "Installing Mysql " } # todo make helm directories work with parameters - ShellCmd.run("./cnf-testsuite cnf_setup cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml", "sample_cnf_setup") + ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") KubectlClient::Get.resource_wait_for_install("Pod", "mysql-0") - # KubectlClient::Delete.file("https://raw.githubusercontent.com/mysql/mysql-operator/trunk/samples/sample-cluster.yaml --wait=false") - # temp_pw = Random.rand.to_s - # KubectlClient::Create.command(%(secret generic mypwds --from-literal=rootUser=root --from-literal=rootHost=% --from-literal=rootPassword="#{temp_pw}")) - # KubectlClient::Apply.file("https://raw.githubusercontent.com/mysql/mysql-operator/trunk/samples/sample-cluster.yaml") - # KubectlClient::Get.resource_wait_for_install("Pod", "mycluster-2") - response_s = `LOG_LEVEL=info ./cnf-testsuite database_persistence` - Log.info {"Status: #{response_s}"} - (/PASSED: CNF uses database with cloud-native persistence/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("database_persistence", cmd_prefix: "LOG_LEVEL=info") + (/PASSED: CNF uses database with cloud-native persistence/ =~ result[:output]).should_not be_nil ensure - # Mysql.uninstall - # KubectlClient::Delete.file("https://raw.githubusercontent.com/mysql/mysql-operator/trunk/samples/sample-cluster.yaml --wait=false") - # KubectlClient::Delete.file("https://raw.githubusercontent.com/mysql/mysql-operator/trunk/samples/sample-cluster.yaml") #todo fix cleanup for helm directory with parameters - ShellCmd.run("./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml", "sample_cnf_cleanup") + ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") ShellCmd.run("kubectl delete pvc data-mysql-0", "delete_pvc") end end it "'elastic_volumes' should fail if the cnf doesn't use an elastic volume", tags: ["elastic_volume"] do begin - LOGGING.info `./cnf-testsuite -l info cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite -l info elastic_volumes verbose` - LOGGING.info "Status: #{response_s}" - (/FAILED: Some of the used volumes are not elastic/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("-l info cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("-l info elastic_volumes verbose") + (/FAILED: Some of the used volumes are not elastic/ =~ result[:output]).should_not be_nil ensure - LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'volume_hostpath_not_found' should pass if the cnf doesn't have a hostPath volume", tags: ["volume_hostpath_not_found"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true - response_s = `./cnf-testsuite volume_hostpath_not_found verbose` - LOGGING.info "Status: #{response_s}" - (/PASSED: hostPath volumes not found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("volume_hostpath_not_found verbose") + (/PASSED: hostPath volumes not found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + result[:status].success?.should be_true end end it "'volume_hostpath_not_found' should fail if the cnf has a hostPath volume", tags: ["volume_hostpath_not_found"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-fragile-state/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true - response_s = `./cnf-testsuite volume_hostpath_not_found verbose` - LOGGING.info "Status: #{response_s}" - (/FAILED: hostPath volumes found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-fragile-state/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("volume_hostpath_not_found verbose") + (/FAILED: hostPath volumes found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-fragile-state/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-fragile-state/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true end end @@ -109,28 +94,26 @@ describe "State" do # update the helm parameter with a schedulable node for the pv chart schedulable_nodes = KubectlClient::Get.schedulable_nodes update_yml("sample-cnfs/sample-local-storage/cnf-testsuite.yml", "helm_values", "--set worker_node='#{schedulable_nodes[0]}'") - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose` - $?.success?.should be_true - response_s = `./cnf-testsuite no_local_volume_configuration verbose` - LOGGING.info "Status: #{response_s}" - (/FAILED: local storage configuration volumes found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") + (/FAILED: local storage configuration volumes found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml deploy_with_chart=false` + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml deploy_with_chart=false") update_yml("sample-cnfs/sample-local-storage/cnf-testsuite.yml", "helm_values", "") - $?.success?.should be_true + result[:status].success?.should be_true end end it "'no_local_volume_configuration' should pass if local storage configuration is not found", tags: ["no_local_volume_configuration"] do begin - `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose` - $?.success?.should be_true - response_s = `./cnf-testsuite no_local_volume_configuration verbose` - LOGGING.info "Status: #{response_s}" - (/PASSED: local storage configuration volumes not found/ =~ response_s).should_not be_nil + result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + result[:status].success?.should be_true + result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") + (/PASSED: local storage configuration volumes not found/ =~ result[:output]).should_not be_nil ensure - `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml deploy_with_chart=false` - $?.success?.should be_true + result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml deploy_with_chart=false") + result[:status].success?.should be_true end end end diff --git a/src/tasks/utils/utils.cr b/src/tasks/utils/utils.cr index 977015413..0d7e5e5fb 100644 --- a/src/tasks/utils/utils.cr +++ b/src/tasks/utils/utils.cr @@ -13,23 +13,26 @@ require "./http_helper.cr" require "ecr" module ShellCmd - def self.run(cmd, log_prefix, force_output=false) - Log.info { "#{log_prefix} command: #{cmd}" } + def self.run(cmd, log_prefix="ShellCmd.run", force_output=false, joined_output=false) + log = Log.for(log_prefix) + log.info { "command: #{cmd}" } + output = IO::Memory.new + stderr = joined_output ? output : IO::Memory.new status = Process.run( cmd, shell: true, - output: output = IO::Memory.new, - error: stderr = IO::Memory.new + output: output, + error: stderr ) if force_output == false - Log.debug { "#{log_prefix} output: #{output.to_s}" } + log.debug { "output: #{output.to_s}" } else - Log.info { "#{log_prefix} output: #{output.to_s}" } + log.info { "output: #{output.to_s}" } end # Don't have to output log line if stderr is empty - if stderr.to_s.size > 1 - Log.info { "#{log_prefix} stderr: #{stderr.to_s}" } + if !joined_output && stderr.to_s.size > 1 + log.info { "stderr: #{stderr.to_s}" } end {status: status, output: output.to_s, error: stderr.to_s} end