From f26345d5ad30e1f1c8b54e2ecd8c36ecae2bbeb4 Mon Sep 17 00:00:00 2001 From: Konstantin Yarovoy Date: Mon, 29 Jul 2024 08:18:19 +0000 Subject: [PATCH] cnf_setup: Exit on setup failure or timeout Fix handling unsuccessful cnf setup, if resources are not ready after timeout - then setup should fail with exit 1. Change specs accordingly, replace wait_count=0 with new option "ignore_setup_timeout". Add cnf_setup function to ShellCmd and use it for installation of CNFs during specs. Refs: #1937 #2083 Signed-off-by: Konstantin Yarovoy --- USAGE.md | 11 +++ spec/5g/core_spec.cr | 12 +-- spec/5g/ran_spec.cr | 12 +-- spec/setup_spec.cr | 12 +-- spec/spec_helper.cr | 10 ++ spec/utils/cnf_manager_spec.cr | 28 +++--- spec/utils/kubescape_spec.cr | 2 +- spec/utils/utils_spec.cr | 10 +- spec/workload/compatibility_spec.cr | 7 +- spec/workload/configuration_spec.cr | 94 +++++++----------- spec/workload/installability_spec.cr | 14 +-- spec/workload/microservice_spec.cr | 42 ++++---- spec/workload/observability_spec.cr | 24 ++--- spec/workload/operator_spec.cr | 6 +- spec/workload/registry_spec.cr | 10 +- spec/workload/resilience/disk_fill_spec.cr | 3 +- spec/workload/resilience/node_drain_spec.cr | 3 +- spec/workload/resilience/pod_delete_spec.cr | 3 +- .../workload/resilience/pod_dns_error_spec.cr | 3 +- .../workload/resilience/pod_io_stress_spec.cr | 3 +- .../resilience/pod_memory_hog_spec.cr | 3 +- .../resilience/pod_network_corruption_spec.cr | 3 +- .../pod_network_duplication_spec.cr | 3 +- .../resilience/pod_network_latency_spec.cr | 3 +- spec/workload/security_spec.cr | 96 +++++++------------ spec/workload/state_spec.cr | 18 ++-- src/tasks/utils/cnf_manager.cr | 34 +++++-- src/tasks/utils/utils.cr | 24 +++-- 28 files changed, 220 insertions(+), 273 deletions(-) diff --git a/USAGE.md b/USAGE.md index 1b4476b59..efeaa477a 100644 --- a/USAGE.md +++ b/USAGE.md @@ -59,6 +59,17 @@ crystal build src/cnf-testsuite.cr ./cnf-testsuite validate_config cnf-config=[PATH_TO]/cnf-testsuite.yml ``` +#### Installing a cnf: + +``` +./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml +``` + +##### Skip waiting for resource readiness during installation: +``` +./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml skip_wait_for_install +``` + #### Running all of the platform and workload tests: ``` diff --git a/spec/5g/core_spec.cr b/spec/5g/core_spec.cr index d71e578b3..9a772355c 100644 --- a/spec/5g/core_spec.cr +++ b/spec/5g/core_spec.cr @@ -14,8 +14,7 @@ describe "Core" do it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose") (/(PASSED).*(Chaos service degradation is less than 50%)/ =~ result[:output]).should_not be_nil ensure @@ -26,8 +25,7 @@ describe "Core" do it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose baseline_count=300") (/(FAILED).*(Chaos service degradation is more than 50%)/ =~ result[:output]).should_not be_nil ensure @@ -38,8 +36,7 @@ describe "Core" do it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("suci_enabled verbose") (/(PASSED).*(Core uses SUCI 5g authentication)/ =~ result[:output]).should_not be_nil ensure @@ -50,8 +47,7 @@ describe "Core" do it "'suci_enabled' should fail if the 5G core does not have suci enabled", tags: ["5g"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") result = ShellCmd.run_testsuite("suci_enabled verbose") (/(FAILED).*(Core does not use SUCI 5g authentication)/ =~ result[:output]).should_not be_nil ensure diff --git a/spec/5g/ran_spec.cr b/spec/5g/ran_spec.cr index 944bad69b..b1a0ba23f 100644 --- a/spec/5g/ran_spec.cr +++ b/spec/5g/ran_spec.cr @@ -15,10 +15,8 @@ describe "5g" do it "'oran_e2_connection' should pass if the ORAN enabled RAN connects to the RIC using the e2 standard", tags: ["oran"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") result = ShellCmd.run_testsuite("oran_e2_connection verbose") (/(PASSED).*(RAN connects to a RIC using the e2 standard interface)/ =~ result[:output]).should_not be_nil ensure @@ -31,10 +29,8 @@ describe "5g" do it "'oran_e2_connection' should fail if the ORAN enabled RAN does not connect to the RIC using the e2 standard", tags: ["oran"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") result = ShellCmd.run_testsuite("oran_e2_connection verbose") (/(FAILED).*(RAN does not connect to a RIC using the e2 standard interface)/ =~ result[:output]).should_not be_nil ensure diff --git a/spec/setup_spec.cr b/spec/setup_spec.cr index abfcd200a..4bd4de634 100644 --- a/spec/setup_spec.cr +++ b/spec/setup_spec.cr @@ -66,8 +66,7 @@ describe "Setup" do it "'cnf_setup/cnf_cleanup' should install/cleanup with cnf-path arg as alias for cnf-config", tags: ["setup"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=example-cnfs/coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + result = ShellCmd.cnf_setup("cnf-path=example-cnfs/coredns/cnf-testsuite.yml") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=example-cnfs/coredns/cnf-testsuite.yml") @@ -78,8 +77,7 @@ describe "Setup" do it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf with a cnf-testsuite.yml", tags: ["setup"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + result = ShellCmd.cnf_setup("cnf-config=example-cnfs/coredns/cnf-testsuite.yml") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=example-cnfs/coredns/cnf-testsuite.yml") @@ -91,8 +89,7 @@ describe "Setup" do it "'cnf_setup/cnf_cleanup' should work with cnf-testsuite.yml that has no directory associated with it", tags: ["setup"] do begin #TODO force cnfs/ to be deployment name and not the directory name - result = ShellCmd.run_testsuite("cnf_setup cnf-config=spec/fixtures/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + result = ShellCmd.cnf_setup("cnf-config=spec/fixtures/cnf-testsuite.yml verbose") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure @@ -104,8 +101,7 @@ describe "Setup" do it "'cnf_setup/cnf_cleanup' should install/cleanup with helm_directory that descends multiple directories", tags: ["setup"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") - result[:status].success?.should be_true + result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 23a33cee0..7cc84b974 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -20,4 +20,14 @@ module ShellCmd cmd = "#{cmd_prefix} ./cnf-testsuite #{testsuite_cmd}" run(cmd, log_prefix: "ShellCmd.run_testsuite", force_output: true, joined_output: true) end + + def self.cnf_setup(setup_params, cmd_prefix="", expect_failure=false) + result = run_testsuite("cnf_setup #{setup_params} wait_count=300", cmd_prefix) + if !expect_failure + result[:status].success?.should be_true + else + result[:status].success?.should be_false + end + result + end end diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr index 8d23dd49a..84b5c60f1 100644 --- a/spec/utils/cnf_manager_spec.cr +++ b/spec/utils/cnf_manager_spec.cr @@ -30,15 +30,13 @@ describe "SampleUtils" do end it "'cnf_setup' should pass with a minimal cnf-testsuite.yml", tags: ["cnf-setup"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-minimal-cnf/ wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install") ensure result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true") end it "'cnf_setup' should support cnf-config as an alias for cnf-path", tags: ["cnf-setup"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-minimal-cnf/ wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install") ensure result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true") end @@ -160,7 +158,7 @@ describe "SampleUtils" do it "'CNFManager.sample_setup_cli_args(args) and CNFManager.sample_setup(cli_args)' should set up a sample cnf", tags: ["cnf-setup"] do - args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose", "wait_count=180"]) + args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(cli_hash[:config_file])) @@ -175,7 +173,7 @@ describe "SampleUtils" do it "'CNFManager.sample_setup' should set up a sample cnf", tags: ["cnf-setup"] do config_file = "sample-cnfs/sample-generic-cnf" - args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) # check if directory exists @@ -191,7 +189,7 @@ describe "SampleUtils" do it "'CNFManager.sample_setup_args' should set up a sample cnf from a argument", tags: ["cnf-setup"] do config_file = "sample-cnfs/sample-generic-cnf" - args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) # check if directory exists @@ -205,7 +203,7 @@ describe "SampleUtils" do it "'CNFManager.sample_setup_args' should set up a sample cnf from a config file", tags: ["cnf-setup"] do config_file = "sample-cnfs/sample-generic-cnf" - args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) # check if directory exists @@ -218,7 +216,7 @@ describe "SampleUtils" do end it "'CNFManager.sample_cleanup' should clean up a sample cnf from a argument", tags: ["cnf-setup"] do - args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) cleanup = CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-generic-cnf", verbose: true) @@ -230,7 +228,7 @@ describe "SampleUtils" do it "'CNFManager.sample_setup_args' should be able to deploy using a helm_directory", tags: ["cnf-setup"] do config_file = "sample-cnfs/sample_privileged_cnf" - args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) @@ -246,7 +244,7 @@ describe "SampleUtils" do it "'CNFManager.sample_setup_args and CNFManager.sample_cleanup' should be able to deploy and cleanup using a manifest_directory", tags: ["cnf-setup"] do config_file = "sample-cnfs/k8s-non-helm" - args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) Log.info { "Running Setup" } CNFManager.sample_setup(cli_hash) @@ -270,7 +268,7 @@ describe "SampleUtils" do it "'CNFManager.cnf_config_list' should return a list of all of the config files from the cnf directory", tags: ["cnf-setup"] do config_file = "sample-cnfs/sample-generic-cnf" - args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) args = Sam::Args.new(["cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml", "verbose"]) @@ -283,7 +281,7 @@ describe "SampleUtils" do it "'CNFManager.helm_repo_add' should add a helm repo if the helm repo is valid", tags: ["helm-repo"] do config_file = "sample-cnfs/sample-generic-cnf" - args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"]) + args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose"]) cli_hash = CNFManager.sample_setup_cli_args(args) args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml"]) CNFManager.helm_repo_add(args: args).should eq(true) @@ -395,7 +393,7 @@ describe "SampleUtils" do it "bonus tests should not be includded in the maximum points when a failure occurs", tags: ["cnf-config"] do begin # fails because doesn't have a service - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-ndn-privileged") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-ndn-privileged") result = ShellCmd.run_testsuite("cert_microservice") (/of 6 tests passed/ =~ result[:output]).should_not be_nil ensure @@ -406,7 +404,7 @@ describe "SampleUtils" do it "Helm_values should be used during the installation of a cnf", tags: ["cnf-config"] do begin # fails because doesn't have a service - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_coredns_values") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_coredns_values") deployment_containers = KubectlClient::Get.deployment_containers("coredns-coredns") image_tags = KubectlClient::Get.container_image_tags(deployment_containers) Log.info { "image_tags: #{image_tags}" } diff --git a/spec/utils/kubescape_spec.cr b/spec/utils/kubescape_spec.cr index e75c1b827..916751a92 100644 --- a/spec/utils/kubescape_spec.cr +++ b/spec/utils/kubescape_spec.cr @@ -8,7 +8,7 @@ describe "K8sInstrumentation" do end it "'#scan and #test_by_test_name' should return the results of a kubescape scan", tags: ["kubescape"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") Kubescape.scan results_json = Kubescape.parse test_json = Kubescape.test_by_test_name(results_json, "Network policies") diff --git a/spec/utils/utils_spec.cr b/spec/utils/utils_spec.cr index 910ec283f..b5e3b5b74 100644 --- a/spec/utils/utils_spec.cr +++ b/spec/utils/utils_spec.cr @@ -152,8 +152,8 @@ describe "Utils" do it "'all_cnfs_task_runner' should run a test against all cnfs in the cnfs directory if there is not cnf-config argument passed to it", tags: ["task_runner"] do my_args = Sam::Args.new - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-generic-cnf") - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_privileged_cnf") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-generic-cnf") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_privileged_cnf") task_response = CNFManager::Task.all_cnfs_task_runner(my_args) do |args, config| Log.info { "all_cnfs_task_runner spec args #{args.inspect}" } Log.for("verbose").info { "privileged" } if check_verbose(args) @@ -192,8 +192,8 @@ describe "Utils" do end it "'task_runner' should run a test against a single cnf if passed a cnf-config argument even if there are multiple cnfs installed", tags: ["task_runner"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml") - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privileged") (/(FAILED).*(Found 1 privileged containers)/ =~ result[:output]).should_not be_nil ensure @@ -252,7 +252,7 @@ describe "Utils" do end it "'logger' or verbose output should be shown when verbose flag is set", tags: ["logger"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("helm_deploy verbose", cmd_prefix: "LOG_LEVEL=info") puts result[:output] result[:status].success?.should be_true diff --git a/spec/workload/compatibility_spec.cr b/spec/workload/compatibility_spec.cr index 9c24a243e..563c43864 100644 --- a/spec/workload/compatibility_spec.cr +++ b/spec/workload/compatibility_spec.cr @@ -14,10 +14,10 @@ describe "Compatibility" do it "'cni_compatible' should pass if the cnf works with calico and flannel", tags: ["compatibility"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") retry_limit = 5 retries = 1 + result = ShellCmd.run_testsuite("cni_compatible verbose") until (/PASSED/ =~ result[:output]) || retries > retry_limit Log.info { "cni_compatible spec retry: #{retries}" } sleep 1.0 @@ -34,8 +34,7 @@ describe "Compatibility" do it "'increase_decrease_capacity' should pass ", tags: ["increase_decrease_capacity"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("increase_decrease_capacity verbose") result[:status].success?.should be_true (/(PASSED).*(Replicas increased to)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/configuration_spec.cr b/spec/workload/configuration_spec.cr index 3e9d4ad5c..51b239105 100644 --- a/spec/workload/configuration_spec.cr +++ b/spec/workload/configuration_spec.cr @@ -19,8 +19,7 @@ describe CnfTestSuite do it "'liveness' should pass when livenessProbe is set", tags: ["liveness"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") result = ShellCmd.run_testsuite("liveness verbose", cmd_prefix:"LOG_LEVEL=debug") result[:status].success?.should be_true (/(PASSED).*(Helm liveness probe)/ =~ result[:output]).should_not be_nil @@ -31,8 +30,7 @@ describe CnfTestSuite do it "'liveness' should fail when livenessProbe is not set", tags: ["liveness"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("liveness verbose") result[:status].success?.should be_true (/(FAILED).*(No livenessProbe found)/ =~ result[:output]).should_not be_nil @@ -43,8 +41,7 @@ describe CnfTestSuite do it "'readiness' should pass when readinessProbe is set", tags: ["readiness"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") result = ShellCmd.run_testsuite("readiness verbose", cmd_prefix: "LOG_LEVEL=debug") result[:status].success?.should be_true (/(PASSED).*(Helm readiness probe)/ =~ result[:output]).should_not be_nil @@ -55,8 +52,7 @@ describe CnfTestSuite do it "'readiness' should fail when readinessProbe is not set", tags: ["readiness"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("readiness verbose") result[:status].success?.should be_true (/(FAILED).*(No readinessProbe found)/ =~ result[:output]).should_not be_nil @@ -67,8 +63,7 @@ describe CnfTestSuite do it "'rolling_update' should pass when valid version is given", tags: ["rolling_update"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil @@ -79,8 +74,7 @@ describe CnfTestSuite do it "'rolling_update' should fail when invalid version is given", tags: ["rolling_update"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil @@ -91,10 +85,10 @@ describe CnfTestSuite do it "'rolling_downgrade' should pass when valid version is given", tags: ["rolling_downgrade"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") retry_limit = 5 retries = 1 + result = ShellCmd.run_testsuite("rolling_downgrade verbose") until (/Passed/ =~ result[:output]) || retries > retry_limit Log.info { "rolling_downgrade retry: #{retries}" } sleep 1.0 @@ -111,8 +105,7 @@ describe CnfTestSuite do it "'rolling_downgrade' should fail when invalid version is given", tags: ["rolling_downgrade"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") result = ShellCmd.run_testsuite("rolling_downgrade verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil @@ -123,8 +116,7 @@ describe CnfTestSuite do it "'rolling_version_change' should pass when valid version is given", tags: ["rolling_version_change"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil @@ -135,8 +127,7 @@ describe CnfTestSuite do it "'rolling_version_change' should fail when invalid version is given", tags: ["rolling_version_change"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil @@ -147,8 +138,7 @@ describe CnfTestSuite do it "'rollback' should pass ", tags: ["rollback"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("rollback verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil @@ -161,8 +151,7 @@ describe CnfTestSuite do it "'nodeport_not_used' should fail when a node port is being used", tags: ["nodeport_not_used"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false") result = ShellCmd.run_testsuite("nodeport_not_used verbose") result[:status].success?.should be_true (/(FAILED).*(NodePort is being used)/ =~ result[:output]).should_not be_nil @@ -173,8 +162,7 @@ describe CnfTestSuite do it "'nodeport_not_used' should pass when a node port is not being used", tags: ["nodeport_not_used"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("nodeport_not_used verbose") result[:status].success?.should be_true (/(PASSED).*(NodePort is not used)/ =~ result[:output]).should_not be_nil @@ -185,8 +173,7 @@ describe CnfTestSuite do it "'hostport_not_used' should fail when a node port is being used", tags: ["hostport_not_used"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false") result = ShellCmd.run_testsuite("hostport_not_used verbose") result[:status].success?.should be_true (/(FAILED).*(HostPort is being used)/ =~ result[:output]).should_not be_nil @@ -197,8 +184,7 @@ describe CnfTestSuite do it "'hostport_not_used' should pass when a node port is not being used", tags: ["hostport_not_used"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("hostport_not_used verbose") result[:status].success?.should be_true (/(PASSED).*(HostPort is not used)/ =~ result[:output]).should_not be_nil @@ -209,8 +195,7 @@ describe CnfTestSuite do it "'ip_addresses' should pass when no uncommented ip addresses are found in helm chart source", tags: ["ip_addresses"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf-source/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf-source/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("ip_addresses verbose") result[:status].success?.should be_true (/(PASSED).*(No IP addresses found)/ =~ result[:output]).should_not be_nil @@ -221,8 +206,7 @@ describe CnfTestSuite do it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should fail when a hardcoded ip is found in the K8s configuration", tags: ["ip_addresses"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false") result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose", cmd_prefix: "LOG_LEVEL=info") result[:status].success?.should be_true (/(FAILED).*(Hard-coded IP addresses found in the runtime K8s configuration)/ =~ result[:output]).should_not be_nil @@ -233,8 +217,7 @@ describe CnfTestSuite do it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should pass when no ip addresses are found in the K8s configuration", tags: ["ip_addresses"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose") result[:status].success?.should be_true (/(PASSED).*(No hard-coded IP addresses found in the runtime K8s configuration)/ =~ result[:output]).should_not be_nil @@ -245,8 +228,7 @@ describe CnfTestSuite do it "'secrets_used' should pass when secrets are provided as volumes and used by a container", tags: ["secrets"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(PASSED).*(Secrets defined and used)/ =~ result[:output]).should_not be_nil @@ -257,8 +239,7 @@ describe CnfTestSuite do it "'secrets_used' should be skipped when secrets are provided as volumes and not mounted by a container", tags: ["secrets"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil @@ -269,8 +250,7 @@ describe CnfTestSuite do it "'secrets_used' should pass when secrets are provided as environment variables and used by a container", tags: ["secrets"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(PASSED).*(Secrets defined and used)/ =~ result[:output]).should_not be_nil @@ -281,8 +261,7 @@ describe CnfTestSuite do it "'secrets_used' should skip when secrets are not referenced as environment variables by a container", tags: ["secrets"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml wait_count=2 verbose") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml skip_wait_for_install verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil @@ -293,8 +272,7 @@ describe CnfTestSuite do it "'secrets_used' should be skipped when no secret volumes are mounted or no container secrets are provided (secrets ignored)`", tags: ["secrets"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil @@ -305,8 +283,7 @@ describe CnfTestSuite do it "'immutable_configmap' fail with some mutable configmaps in container env or volume mount", tags: ["immutable_configmap"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false") result = ShellCmd.run_testsuite("immutable_configmap verbose") result[:status].success?.should be_true (/(FAILED).*(Found mutable configmap)/ =~ result[:output]).should_not be_nil @@ -317,8 +294,7 @@ describe CnfTestSuite do it "'immutable_configmap' pass with all immutable configmaps in container env or volume mounts", tags: ["immutable_configmap"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false") result = ShellCmd.run_testsuite("immutable_configmap verbose") result[:status].success?.should be_true (/(PASSED).*(All volume or container mounted configmaps immutable)/ =~ result[:output]).should_not be_nil @@ -329,8 +305,7 @@ describe CnfTestSuite do it "'require_labels' should fail if a cnf does not have the app.kubernetes.io/name label", tags: ["require_labels"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") result = ShellCmd.run_testsuite("require_labels verbose") result[:status].success?.should be_true (/(FAILED).*(Pods should have the app.kubernetes.io\/name label)/ =~ result[:output]).should_not be_nil @@ -341,8 +316,7 @@ describe CnfTestSuite do it "'require_labels' should pass if a cnf has the app.kubernetes.io/name label", tags: ["require_labels"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("require_labels verbose") result[:status].success?.should be_true (/(PASSED).*(Pods have the app.kubernetes.io\/name label)/ =~ result[:output]).should_not be_nil @@ -353,8 +327,7 @@ describe CnfTestSuite do it "'default_namespace' should fail if a cnf creates resources in the default namespace", tags: ["default_namespace"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns_default_namespace") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_default_namespace") result = ShellCmd.run_testsuite("default_namespace verbose") result[:status].success?.should be_true (/(FAILED).*(Resources are created in the default namespace)/ =~ result[:output]).should_not be_nil @@ -366,8 +339,7 @@ describe CnfTestSuite do it "'default_namespace' should pass if a cnf does not create resources in the default namespace", tags: ["default_namespace"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_latest_tag") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("default_namespace verbose") result[:status].success?.should be_true (/(PASSED).*(default namespace is not being used)/ =~ result[:output]).should_not be_nil @@ -379,8 +351,7 @@ describe CnfTestSuite do it "'latest_tag' should fail if a cnf has containers that use images with the latest tag", tags: ["latest_tag"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_latest_tag") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("latest_tag verbose") result[:status].success?.should be_true (/(FAILED).*(Container images are using the latest tag)/ =~ result[:output]).should_not be_nil @@ -391,8 +362,7 @@ describe CnfTestSuite do it "'latest_tag' should pass if a cnf does not have containers that use images with the latest tag", tags: ["latest_tag"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("latest_tag verbose") result[:status].success?.should be_true (/(PASSED).*(Container images are not using the latest tag)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/installability_spec.cr b/spec/workload/installability_spec.cr index 7fee96822..f8f9299c3 100644 --- a/spec/workload/installability_spec.cr +++ b/spec/workload/installability_spec.cr @@ -9,7 +9,7 @@ describe CnfTestSuite do end it "'helm_deploy' should fail on a bad helm chart", tags: ["helm"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-bad-helm-deploy-repo verbose") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-bad-helm-deploy-repo verbose") result = ShellCmd.run_testsuite("helm_deploy verbose") result[:status].success?.should be_true (/(FAILED).*(Helm deploy failed)/ =~ result[:output]).should_not be_nil @@ -24,8 +24,7 @@ describe CnfTestSuite do end it "'helm_chart_valid' should pass on a good helm chart", tags: ["helm"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("helm_chart_valid verbose") result[:status].success?.should be_true (/Lint Passed/ =~ result[:output]).should_not be_nil @@ -35,8 +34,7 @@ describe CnfTestSuite do it "'helm_chart_valid' should fail on a bad helm chart", tags: ["helm"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("helm_chart_valid") result[:status].success?.should be_true (/Lint Failed/ =~ result[:output]).should_not be_nil @@ -47,8 +45,7 @@ describe CnfTestSuite do it "'helm_chart_published' should pass on a good helm chart repo", tags: ["helm_chart_published"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("helm_chart_published") result[:status].success?.should be_true (/(PASSED).*(Published Helm Chart Found)/ =~ result[:output]).should_not be_nil @@ -60,8 +57,7 @@ describe CnfTestSuite do it "'helm_chart_published' should fail on a bad helm chart repo", tags: ["helm_chart_published"] do begin result = ShellCmd.run("helm search repo stable/coredns", force_output: true) - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-bad-helm-repo wait_count=0") - result[:status].success?.should be_false + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-bad-helm-repo skip_wait_for_install", expect_failure: true) result = ShellCmd.run("helm search repo stable/coredns", force_output: true) result = ShellCmd.run_testsuite("helm_chart_published verbose") result[:status].success?.should be_true diff --git a/spec/workload/microservice_spec.cr b/spec/workload/microservice_spec.cr index 508dc21ca..862f67ae6 100644 --- a/spec/workload/microservice_spec.cr +++ b/spec/workload/microservice_spec.cr @@ -19,7 +19,7 @@ describe "Microservice" do it "'shared_database' should be skipped no MariaDB containers are found", tags: ["shared_database"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(N\/A).*(No MariaDB containers were found)/ =~ result[:output]).should_not be_nil @@ -31,7 +31,7 @@ describe "Microservice" do it "'shared_database' should pass if no database is used by two microservices", tags: ["shared_database"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(PASSED).*(No shared database found)/ =~ result[:output]).should_not be_nil @@ -43,7 +43,7 @@ describe "Microservice" do it "'shared_database' should pass if one service connects to a database but other non-service connections are made to the database", tags: ["shared_database"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(PASSED).*(No shared database found)/ =~ result[:output]).should_not be_nil @@ -55,7 +55,7 @@ describe "Microservice" do it "'shared_database' should fail if two services on the cluster connect to the same database", tags: ["shared_database2"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(FAILED).*(Found a shared database)/ =~ result[:output]).should_not be_nil @@ -68,7 +68,7 @@ describe "Microservice" do it "'shared_database' should pass if two services on the cluster connect to the same database but they are not in the helm chart of the cnf", tags: ["shared_database"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") Helm.install("multi-db sample-cnfs/ndn-multi-db-connections-fail/wordpress/") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress", wait_count: 180, namespace: "default") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress2", wait_count: 180, namespace: "default") @@ -90,7 +90,7 @@ describe "Microservice" do it "'single_process_type' should pass if the containers in the cnf have only one process type", tags: ["process_check"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(PASSED).*(Only one process type used)/ =~ result[:output]).should_not be_nil @@ -102,7 +102,7 @@ describe "Microservice" do it "'single_process_type' should fail if the containers in the cnf have more than one process type", tags: ["process_check"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/k8s-multiple-processes") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/k8s-multiple-processes") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(FAILED).*(More than one process type used)/ =~ result[:output]).should_not be_nil @@ -114,7 +114,7 @@ describe "Microservice" do it "'single_process_type' should fail if the containers in the cnf have more than one process type and in a pod", tags: ["process_check"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-multiple-processes") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-multiple-processes") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(FAILED).*(More than one process type used)/ =~ result[:output]).should_not be_nil @@ -126,7 +126,7 @@ describe "Microservice" do it "'reasonable_startup_time' should pass if the cnf has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("reasonable_startup_time verbose") result[:status].success?.should be_true (/(PASSED).*(CNF had a reasonable startup time)/ =~ result[:output]).should_not be_nil @@ -137,7 +137,7 @@ describe "Microservice" do end it "'reasonable_startup_time' should fail if the cnf doesn't has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") begin result = ShellCmd.run_testsuite("reasonable_startup_time verbose") result[:status].success?.should be_true @@ -154,7 +154,7 @@ describe "Microservice" do else cnf="./sample-cnfs/sample-coredns-cnf" end - result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + ShellCmd.cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil @@ -163,14 +163,14 @@ describe "Microservice" do end it "'reasonable_image_size' should fail if image is larger than 5gb", tags: ["reasonable_image_size"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/ndn-reasonable-image-size wait_count=0") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/ndn-reasonable-image-size skip_wait_for_install") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size too large/ =~ result[:output]).should_not be_nil end it "'specialized_init_system' should fail if pods do not use specialized init systems", tags: ["specialized_init_system"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("specialized_init_system") result[:status].success?.should be_true (/Containers do not use specialized init systems/ =~ result[:output]).should_not be_nil @@ -179,7 +179,7 @@ describe "Microservice" do end it "'specialized_init_system' should pass if pods use specialized init systems", tags: ["specialized_init_system"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-init-systems") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-init-systems") result = ShellCmd.run_testsuite("specialized_init_system") result[:status].success?.should be_true (/Containers use specialized init systems/ =~ result[:output]).should_not be_nil @@ -189,7 +189,7 @@ describe "Microservice" do it "'service_discovery' should pass if any containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("service_discovery verbose") result[:status].success?.should be_true (/(PASSED).*(Some containers exposed as a service)/ =~ result[:output]).should_not be_nil @@ -201,7 +201,7 @@ describe "Microservice" do it "'service_discovery' should fail if no containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-ndn-privileged") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-ndn-privileged") result = ShellCmd.run_testsuite("service_discovery verbose") result[:status].success?.should be_true (/(FAILED).*(No containers exposed as a service)/ =~ result[:output]).should_not be_nil @@ -218,7 +218,7 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_good_signal_handling/") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling/") result = ShellCmd.run_testsuite("sig_term_handled verbose") result[:status].success?.should be_true (/(PASSED).*(Sig Term handled)/ =~ result[:output]).should_not be_nil @@ -235,7 +235,7 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_bad_signal_handling/") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_bad_signal_handling/") result = ShellCmd.run_testsuite("sig_term_handled verbose") result[:status].success?.should be_true (/(FAILED).*(Sig Term not handled)/ =~ result[:output]).should_not be_nil @@ -252,7 +252,7 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") # Workaround to wait using kubectl because Jenkins pod takes a LONG time to start. result = ShellCmd.run("kubectl wait --for=condition=ready=True pod/jenkins-0 -n cnfspace --timeout=500s", force_output: true) @@ -269,7 +269,7 @@ describe "Microservice" do it "'zombie_handled' should pass if a zombie is succesfully reaped by PID 1", tags: ["zombie"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_good_zombie_handling/") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_zombie_handling/") result = ShellCmd.run_testsuite("zombie_handled verbose") result[:status].success?.should be_true (/(PASSED).*(Zombie handled)/ =~ result[:output]).should_not be_nil @@ -281,7 +281,7 @@ describe "Microservice" do it "'zombie_handled' should failed if a zombie is not succesfully reaped by PID 1", tags: ["zombie"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-bad-zombie/") + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-bad-zombie/") result = ShellCmd.run_testsuite("zombie_handled verbose") result[:status].success?.should be_true (/(FAILED).*(Zombie not handled)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/observability_spec.cr b/spec/workload/observability_spec.cr index 5547d45dd..ad7671f83 100644 --- a/spec/workload/observability_spec.cr +++ b/spec/workload/observability_spec.cr @@ -12,7 +12,7 @@ describe "Observability" do it "'log_output' should pass with a cnf that outputs logs to stdout", tags: ["observability"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("log_output verbose") result[:status].success?.should be_true (/(PASSED).*(Resources output logs to stdout and stderr)/ =~ result[:output]).should_not be_nil @@ -23,7 +23,7 @@ describe "Observability" do it "'log_output' should fail with a cnf that does not output logs to stdout", tags: ["observability"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("log_output verbose") result[:status].success?.should be_true (/(FAILED).*(Resources do not output logs to stdout and stderr)/ =~ result[:output]).should_not be_nil @@ -33,7 +33,7 @@ describe "Observability" do end it "'prometheus_traffic' should pass if there is prometheus traffic", tags: ["observability"] do - ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm Log.info { "Add prometheus helm repo" } @@ -56,7 +56,7 @@ describe "Observability" do it "'prometheus_traffic' should skip if there is no prometheus installed", tags: ["observability"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm result = ShellCmd.run("#{helm} delete prometheus", force_output: true) @@ -68,7 +68,7 @@ describe "Observability" do it "'prometheus_traffic' should fail if the cnf is not registered with prometheus", tags: ["observability"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) @@ -86,7 +86,7 @@ describe "Observability" do end it "'open_metrics' should fail if there is not a valid open metrics response from the cnf", tags: ["observability"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm @@ -104,7 +104,7 @@ describe "Observability" do end it "'open_metrics' should pass if there is a valid open metrics response from the cnf", tags: ["observability"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm @@ -122,7 +122,7 @@ describe "Observability" do end it "'routed_logs' should pass if cnfs logs are captured by fluentd bitnami", tags: ["observability"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("install_fluentdbitnami") result = ShellCmd.run_testsuite("routed_logs") (/(PASSED).*(Your CNF's logs are being captured)/ =~ result[:output]).should_not be_nil @@ -133,7 +133,7 @@ describe "Observability" do end it "'routed_logs' should pass if cnfs logs are captured by fluentbit", tags: ["observability"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-fluentbit") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-fluentbit") result = ShellCmd.run_testsuite("install_fluentbit") result = ShellCmd.run_testsuite("routed_logs") (/(PASSED).*(Your CNF's logs are being captured)/ =~ result[:output]).should_not be_nil @@ -145,7 +145,7 @@ describe "Observability" do it "'routed_logs' should fail if cnfs logs are not captured", tags: ["observability"] do - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Helm.helm_repo_add("bitnami","https://charts.bitnami.com/bitnami") #todo #helm install --values ./override.yml fluentd ./fluentd Helm.install("--values ./spec/fixtures/fluentd-values-bad.yml -n #{TESTSUITE_NAMESPACE} fluentd bitnami/fluentd") @@ -164,7 +164,7 @@ describe "Observability" do Log.info { "Installing Jaeger " } JaegerManager.install - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("tracing") (/(FAILED).*(Tracing not used)/ =~ result[:output]).should_not be_nil ensure @@ -180,7 +180,7 @@ describe "Observability" do Log.info { "Installing Jaeger " } JaegerManager.install - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml") + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml") result = ShellCmd.run_testsuite("tracing") (/(PASSED).*(Tracing used)/ =~ result[:output]).should_not be_nil ensure diff --git a/spec/workload/operator_spec.cr b/spec/workload/operator_spec.cr index b3d74be98..8e57c6d3e 100644 --- a/spec/workload/operator_spec.cr +++ b/spec/workload/operator_spec.cr @@ -23,8 +23,7 @@ describe "Operator" do Helm.install("operator --set olm.image.ref=quay.io/operator-framework/olm:v0.22.0 --set catalog.image.ref=quay.io/operator-framework/olm:v0.22.0 --set package.image.ref=quay.io/operator-framework/olm:v0.22.0 #{install_dir}/deploy/chart/") begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_operator", cmd_prefix: "LOG_LEVEL=info") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_operator", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("operator_installed", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(Operator is installed)/ =~ result[:output]).should_not be_nil ensure @@ -68,8 +67,7 @@ describe "Operator" do it "'operator_test' operator should not be found", tags: ["operator_test"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_coredns") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("operator_installed", cmd_prefix: "LOG_LEVEL=info") (/(N\/A).*(No Operators Found)/ =~ result[:output]).should_not be_nil ensure diff --git a/spec/workload/registry_spec.cr b/spec/workload/registry_spec.cr index cf9bb5b9a..a88fa9616 100644 --- a/spec/workload/registry_spec.cr +++ b/spec/workload/registry_spec.cr @@ -42,7 +42,7 @@ describe "Private Registry: Image" do it "'reasonable_image_size' should pass if using local registry and a port", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry" - result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + ShellCmd.cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil @@ -53,7 +53,7 @@ describe "Private Registry: Image" do it "'reasonable_image_size' should pass if using local registry, a port and an org", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry_org_image" - result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + ShellCmd.cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil @@ -91,7 +91,7 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + ShellCmd.cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil @@ -104,7 +104,7 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + ShellCmd.cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil @@ -117,7 +117,7 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - result = ShellCmd.run_testsuite("cnf_setup cnf-path=#{cnf}") + ShellCmd.cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/resilience/disk_fill_spec.cr b/spec/workload/resilience/disk_fill_spec.cr index b09e36ad2..42fe68ce8 100644 --- a/spec/workload/resilience/disk_fill_spec.cr +++ b/spec/workload/resilience/disk_fill_spec.cr @@ -14,8 +14,7 @@ describe "Resilience Disk Fill Chaos" do it "'disk_fill' A 'Good' CNF should not crash when disk fill occurs", tags: ["disk_fill"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml skip_wait_for_install") result = ShellCmd.run_testsuite("disk_fill verbose") result[:status].success?.should be_true (/(PASSED).*(disk_fill chaos test passed)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/resilience/node_drain_spec.cr b/spec/workload/resilience/node_drain_spec.cr index 4bc5e6d26..29d9fa5e1 100644 --- a/spec/workload/resilience/node_drain_spec.cr +++ b/spec/workload/resilience/node_drain_spec.cr @@ -14,8 +14,7 @@ describe "Resilience Node Drain Chaos" do it "'node_drain' A 'Good' CNF should not crash when node drain occurs", tags: ["node_drain"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("node_drain verbose") result[:status].success?.should be_true if KubectlClient::Get.schedulable_nodes_list.size > 1 diff --git a/spec/workload/resilience/pod_delete_spec.cr b/spec/workload/resilience/pod_delete_spec.cr index 5ff105c3f..80fdb13e2 100644 --- a/spec/workload/resilience/pod_delete_spec.cr +++ b/spec/workload/resilience/pod_delete_spec.cr @@ -14,8 +14,7 @@ describe "Resilience pod delete Chaos" do it "'pod_delete' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_delete"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_delete verbose") result[:status].success?.should be_true (/(PASSED).*(pod_delete chaos test passed)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/resilience/pod_dns_error_spec.cr b/spec/workload/resilience/pod_dns_error_spec.cr index 912150cb7..f7902182a 100644 --- a/spec/workload/resilience/pod_dns_error_spec.cr +++ b/spec/workload/resilience/pod_dns_error_spec.cr @@ -14,8 +14,7 @@ describe "Resilience pod dns error Chaos" do it "'pod_dns_error' A 'Good' CNF should not crash when pod dns error occurs", tags: ["pod_dns_error"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=example-cnfs/envoy/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=example-cnfs/envoy/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_dns_error verbose") result[:status].success?.should be_true ((/(SKIPPED).*(pod_dns_error docker runtime not found)/) =~ result[:output] || diff --git a/spec/workload/resilience/pod_io_stress_spec.cr b/spec/workload/resilience/pod_io_stress_spec.cr index d7f0e6c6d..f768409ce 100644 --- a/spec/workload/resilience/pod_io_stress_spec.cr +++ b/spec/workload/resilience/pod_io_stress_spec.cr @@ -14,8 +14,7 @@ describe "Resilience pod delete Chaos" do it "'pod_io_stress' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_io_stress"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_io_stress verbose") result[:status].success?.should be_true (/(PASSED).*(pod_io_stress chaos test passed)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/resilience/pod_memory_hog_spec.cr b/spec/workload/resilience/pod_memory_hog_spec.cr index f0c92378d..80ec62d0f 100644 --- a/spec/workload/resilience/pod_memory_hog_spec.cr +++ b/spec/workload/resilience/pod_memory_hog_spec.cr @@ -14,8 +14,7 @@ describe "Resilience pod memory hog Chaos" do it "'pod_memory_hog' A 'Good' CNF should not crash when pod memory hog occurs", tags: ["pod_memory_hog"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_memory_hog verbose") result[:status].success?.should be_true (/(PASSED).*(pod_memory_hog chaos test passed)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/resilience/pod_network_corruption_spec.cr b/spec/workload/resilience/pod_network_corruption_spec.cr index bac85178b..fbf8c23f3 100644 --- a/spec/workload/resilience/pod_network_corruption_spec.cr +++ b/spec/workload/resilience/pod_network_corruption_spec.cr @@ -14,8 +14,7 @@ describe "Resilience Pod Network corruption Chaos" do it "'pod_network_corruption' A 'Good' CNF should not crash when network corruption occurs", tags: ["pod_network_corruption"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_corruption verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_corruption chaos test passed)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/resilience/pod_network_duplication_spec.cr b/spec/workload/resilience/pod_network_duplication_spec.cr index b64017206..de60e895a 100644 --- a/spec/workload/resilience/pod_network_duplication_spec.cr +++ b/spec/workload/resilience/pod_network_duplication_spec.cr @@ -14,8 +14,7 @@ describe "Resilience Pod Network duplication Chaos" do it "'pod_network_duplication' A 'Good' CNF should not crash when network duplication occurs", tags: ["pod_network_duplication"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_duplication verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_duplication chaos test passed)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/resilience/pod_network_latency_spec.cr b/spec/workload/resilience/pod_network_latency_spec.cr index e6cde7244..24b455323 100644 --- a/spec/workload/resilience/pod_network_latency_spec.cr +++ b/spec/workload/resilience/pod_network_latency_spec.cr @@ -14,8 +14,7 @@ describe "Resilience Pod Network Latency Chaos" do it "'pod_network_latency' A 'Good' CNF should not crash when network latency occurs", tags: ["pod_network_latency"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_latency verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_latency chaos test passed)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/security_spec.cr b/spec/workload/security_spec.cr index 8d61aebab..bdff0e11c 100644 --- a/spec/workload/security_spec.cr +++ b/spec/workload/security_spec.cr @@ -6,8 +6,7 @@ describe "Security" do it "'privileged' should pass with a non-privileged cnf", tags: ["privileged"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") - Log.debug { result[:output] } + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privileged verbose") result[:status].success?.should be_true (/Found.*privileged containers.*coredns/ =~ result[:output]).should be_nil @@ -18,8 +17,7 @@ describe "Security" do end it "'privileged' should fail on a non-whitelisted, privileged cnf", tags: ["privileged"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("privileged verbose") result[:status].success?.should be_true (/Found.*privileged containers.*/ =~ result[:output]).should_not be_nil @@ -30,8 +28,7 @@ describe "Security" do end it "'privileged' should pass on a whitelisted, privileged cnf", tags: ["privileged"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose wait_count=0") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("privileged cnf-config=sample-cnfs/sample_whitelisted_privileged_cnf verbose") result[:status].success?.should be_true (/Found.*privileged containers.*/ =~ result[:output]).should be_nil @@ -41,8 +38,7 @@ describe "Security" do end it "'privilege_escalation' should fail on a cnf that has escalated privileges", tags: ["privileged"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privilege_escalation") result[:status].success?.should be_true (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ result[:output]).should be_nil @@ -53,8 +49,7 @@ describe "Security" do it "'privilege_escalation' should pass on a cnf that does not have escalated privileges", tags: ["privileged"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privilege_escalation") result[:status].success?.should be_true (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ result[:output]).should_not be_nil @@ -65,8 +60,7 @@ describe "Security" do it "'symlink_file_system' should pass on a cnf that does not allow a symlink attack", tags: ["capabilities"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("symlink_file_system") result[:status].success?.should be_true (/(PASSED).*(No containers allow a symlink attack)/ =~ result[:output]).should_not be_nil @@ -77,8 +71,7 @@ describe "Security" do it "'insecure_capabilities' should pass on a cnf that does not have containers with insecure capabilities", tags: ["capabilities"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("insecure_capabilities") result[:status].success?.should be_true (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ result[:output]).should_not be_nil @@ -89,8 +82,7 @@ describe "Security" do it "'insecure_capabilities' should fail on a cnf that has containers with insecure capabilities", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") result = ShellCmd.run_testsuite("insecure_capabilities") result[:status].success?.should be_true (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ result[:output]).should be_nil @@ -101,8 +93,7 @@ describe "Security" do it "'linux_hardening' should fail on a cnf that does not make use of security services", tags: ["capabilities"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("linux_hardening") result[:status].success?.should be_true (/(PASSED).*(Security services are being used to harden applications)/ =~ result[:output]).should be_nil @@ -113,8 +104,7 @@ describe "Security" do it "'application_credentials' should fail on a cnf that allows applications credentials in configuration files", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") result = ShellCmd.run_testsuite("application_credentials") result[:status].success?.should be_true (/(FAILED).*(Found applications credentials in configuration files)/ =~ result[:output]).should_not be_nil @@ -125,8 +115,7 @@ describe "Security" do it "'host_network' should pass on a cnf that does not have a host network attached to pod", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("host_network") result[:status].success?.should be_true (/(PASSED).*(No host network attached to pod)/ =~ result[:output]).should_not be_nil @@ -137,8 +126,7 @@ describe "Security" do it "'service_account_mapping' should fail on a cnf that automatically maps the service account", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") result = ShellCmd.run_testsuite("service_account_mapping") result[:status].success?.should be_true (/(FAILED).*(Service accounts automatically mapped)/ =~ result[:output]).should_not be_nil @@ -149,8 +137,7 @@ describe "Security" do it "'cpu_limits' should pass on a cnf that has containers with cpu limits set", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("cpu_limits") result[:status].success?.should be_true (/(PASSED).*(Containers have CPU limits set)/ =~ result[:output]).should_not be_nil @@ -161,8 +148,7 @@ describe "Security" do it "'memory_limits' should pass on a cnf that has containers with memory limits set", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("memory_limits") result[:status].success?.should be_true (/(PASSED).*(Containers have memory limits set)/ =~ result[:output]).should_not be_nil @@ -173,8 +159,7 @@ describe "Security" do it "'ingress_egress_blocked' should fail on a cnf that has no ingress and egress traffic policy", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("ingress_egress_blocked") result[:status].success?.should be_true (/(PASSED).*(Ingress and Egress traffic blocked on pods)/ =~ result[:output]).should be_nil @@ -185,8 +170,7 @@ describe "Security" do it "'host_pid_ipc_privileges' should pass on a cnf that does not have containers with host PID/IPC privileges", tags: ["capabilities"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("host_pid_ipc_privileges") result[:status].success?.should be_true (/(FAILED).*(Found containers with hostPID and hostIPC privileges)/ =~ result[:output]).should be_nil @@ -197,8 +181,7 @@ describe "Security" do it "'non_root_containers' should pass on a cnf that does not have containers running with root user or user with root group memberships", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-nonroot") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-nonroot") result = ShellCmd.run_testsuite("non_root_containers") result[:status].success?.should be_true (/(FAILED).*(Found containers running with root user or user with root group membership)/ =~ result[:output]).should be_nil @@ -209,8 +192,7 @@ describe "Security" do it "'non_root_containers' should fail on a cnf that has containers running with root user or user with root group memberships", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("non_root_containers") result[:status].success?.should be_true (/(PASSED).*(Containers are running with non-root user with non-root group membership)/ =~ result[:output]).should be_nil @@ -221,8 +203,7 @@ describe "Security" do it "'privileged_containers' should pass when the cnf has no privileged containers", tags: ["privileged"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("privileged_containers") result[:status].success?.should be_true (/(FAILED).*(Found privileged containers)/ =~ result[:output]).should be_nil @@ -233,8 +214,7 @@ describe "Security" do it "'immutable_file_systems' should fail when the cnf containers with mutable file systems", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("immutable_file_systems") result[:status].success?.should be_true (/(PASSED).*(Containers have immutable file systems)/ =~ result[:output]).should be_nil @@ -245,8 +225,7 @@ describe "Security" do it "'immutable_file_systems' should pass when the cnf containers with immutable file systems", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-immutable-fs") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-immutable-fs") result = ShellCmd.run_testsuite("immutable_file_systems") result[:status].success?.should be_true (/(PASSED).*(Containers have immutable file systems)/ =~ result[:output]).should_not be_nil @@ -257,8 +236,7 @@ describe "Security" do it "'hostpath_mounts' should pass when the cnf has no containers with hostPath mounts", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") ClusterTools.uninstall result = ShellCmd.run_testsuite("hostpath_mounts") result[:status].success?.should be_true @@ -271,8 +249,7 @@ describe "Security" do it "'hostpath_mounts' should fail when the cnf has containers with hostPath mounts", tags: ["security"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-hostpath") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-hostpath") ClusterTools.uninstall result = ShellCmd.run_testsuite("hostpath_mounts") result[:status].success?.should be_true @@ -285,8 +262,7 @@ describe "Security" do it "'container_sock_mounts' should pass if a cnf has no pods that mount container engine socket", tags: ["container_sock_mounts"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("container_sock_mounts verbose") result[:status].success?.should be_true (/(PASSED).*(Container engine daemon sockets are not mounted as volumes)/ =~ result[:output]).should_not be_nil @@ -297,8 +273,7 @@ describe "Security" do it "'container_sock_mounts' should fail if the CNF has pods with container engine sockets mounted", tags: ["container_sock_mounts"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") result = ShellCmd.run_testsuite("container_sock_mounts verbose") result[:status].success?.should be_true (/(FAILED).*(Container engine daemon sockets are mounted as volumes)/ =~ result[:output]).should_not be_nil @@ -310,8 +285,7 @@ describe "Security" do it "'external_ips' should pass if a cnf has no services with external IPs", tags: ["external_ips"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("external_ips verbose") result[:status].success?.should be_true (/(PASSED).*(Services are not using external IPs)/ =~ result[:output]).should_not be_nil @@ -322,8 +296,7 @@ describe "Security" do it "'external_ips' should fail if a cnf has services with external IPs", tags: ["external_ips"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") result = ShellCmd.run_testsuite("external_ips verbose") result[:status].success?.should be_true (/(FAILED).*(Services are using external IPs)/ =~ result[:output]).should_not be_nil @@ -334,8 +307,7 @@ describe "Security" do it "'selinux_options' should fail if containers have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_latest_tag") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(FAILED).*(Pods are using custom SELinux options that can be used for privilege escalations)/ =~ result[:output]).should_not be_nil @@ -346,8 +318,7 @@ describe "Security" do it "'selinux_options' should be skipped if containers do not use custom selinux options", tags: ["selinux_options"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(N\/A).*(Pods are not using SELinux)/ =~ result[:output]).should_not be_nil @@ -358,8 +329,7 @@ describe "Security" do it "'selinux_options' should pass if containers do not have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_valid_selinux_options") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_valid_selinux_options") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(PASSED).*(Pods are not using custom SELinux options that can be used for privilege escalations)/ =~ result[:output]).should_not be_nil @@ -370,8 +340,7 @@ describe "Security" do it "'sysctls' should fail if Pods have restricted sysctls values", tags: ["sysctls"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_sysctls") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_sysctls") result = ShellCmd.run_testsuite("sysctls verbose") result[:status].success?.should be_true (/(FAILED).*(Restricted values for are being used for sysctls)/ =~ result[:output]).should_not be_nil @@ -382,8 +351,7 @@ describe "Security" do it "'sysctls' should pass if Pods have allowed sysctls values", tags: ["sysctls"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("sysctls verbose") result[:status].success?.should be_true (/(PASSED).*(No restricted values found for sysctls)/ =~ result[:output]).should_not be_nil diff --git a/spec/workload/state_spec.cr b/spec/workload/state_spec.cr index d5171a6a8..8f623db12 100644 --- a/spec/workload/state_spec.cr +++ b/spec/workload/state_spec.cr @@ -14,8 +14,7 @@ describe "State" do it "'elastic_volumes' should fail if the cnf does not use volumes that are elastic volume", tags: ["elastic_volume"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(All used volumes are elastic)/ =~ result[:output]).should be_nil ensure @@ -28,8 +27,7 @@ describe "State" do # This CNF does not use any volumes except the ones that Kubernetes might mount by default (like the service account token) it "'elastic_volumes' should fail if the cnf does not use any elastic volumes", tags: ["elastic_volume"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_nonroot", cmd_prefix: "LOG_LEVEL=info") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/FAILED/ =~ result[:output]).should_not be_nil ensure @@ -42,8 +40,7 @@ describe "State" do begin Log.info { "Installing Mysql " } # todo make helm directories work with parameters - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") result = ShellCmd.run_testsuite("database_persistence", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(CNF uses database with cloud-native persistence)/ =~ result[:output]).should_not be_nil ensure @@ -55,8 +52,7 @@ describe "State" do it "'elastic_volumes' should fail if the cnf doesn't use an elastic volume", tags: ["elastic_volume"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/(FAILED).*(Some of the used volumes are not elastic)/ =~ result[:output]).should_not be_nil ensure @@ -70,8 +66,7 @@ describe "State" do # update the helm parameter with a schedulable node for the pv chart schedulable_nodes = KubectlClient::Get.schedulable_nodes update_yml("sample-cnfs/sample-local-storage/cnf-testsuite.yml", "helm_values", "--set worker_node='#{schedulable_nodes[0]}'") - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") (/(FAILED).*(local storage configuration volumes found)/ =~ result[:output]).should_not be_nil ensure @@ -83,8 +78,7 @@ describe "State" do it "'no_local_volume_configuration' should pass if local storage configuration is not found", tags: ["no_local_volume_configuration"] do begin - result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") (/(PASSED).*(local storage configuration volumes not found)/ =~ result[:output]).should_not be_nil ensure diff --git a/src/tasks/utils/cnf_manager.cr b/src/tasks/utils/cnf_manager.cr index c2d5dd0d7..4b332c644 100644 --- a/src/tasks/utils/cnf_manager.cr +++ b/src/tasks/utils/cnf_manager.cr @@ -482,9 +482,12 @@ module CNFManager elsif args.named.keys.includes? "wait-count" wait_count = args.named["wait-count"].to_i else - wait_count = 180 + wait_count = 1800 end - cli_args = {config_file: cnf_path, wait_count: wait_count, verbose: check_verbose(args)} + + skip_wait_for_install = args.named.keys.includes? "skip_wait_for_install" + + cli_args = {config_file: cnf_path, wait_count: wait_count, skip_wait_for_install: skip_wait_for_install, verbose: check_verbose(args)} Log.debug { "cli_args: #{cli_args}" } cli_args end @@ -618,6 +621,7 @@ module CNFManager Log.info { "sample_setup cli_args: #{cli_args}" } config_file = cli_args[:config_file] wait_count = cli_args[:wait_count] + skip_wait_for_install = cli_args[:skip_wait_for_install] verbose = cli_args[:verbose] config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) Log.debug { "config in sample_setup: #{config.cnf_config}" } @@ -802,12 +806,24 @@ module CNFManager end end end - - resource_names.each do | resource | - case resource[:kind].downcase - when "replicaset", "deployment", "statefulset", "pod", "daemonset" - KubectlClient::Get.resource_wait_for_install(resource[:kind], resource[:name], wait_count: wait_count, namespace: resource[:namespace]) + if !skip_wait_for_install + stdout_success "Waiting for resource availability, timeout for each resource is #{wait_count} seconds\n" + workload_resource_names = resource_names.select { |resource| + ["replicaset", "deployment", "statefulset", "pod", "daemonset"].includes?(resource[:kind].downcase) + } + total_resource_count = workload_resource_names.size() + current_resource_number = 1 + workload_resource_names.each do | resource | + stdout_success "Waiting for resource (#{current_resource_number}/#{total_resource_count}): [#{resource[:kind]}] #{resource[:name]}", same_line: true + ready = KubectlClient::Get.resource_wait_for_install(resource[:kind], resource[:name], wait_count: wait_count, namespace: resource[:namespace]) + if !ready + stdout_failure "CNF setup has timed-out, [#{resource[:kind]}] #{resource[:name]} is not ready after #{wait_count} seconds.", same_line: true + stdout_failure "Recommented course of actions would be to investigate the resource in cluster, then call cnf_cleanup and try to reinstall the CNF." + exit 1 + end + current_resource_number += 1 end + stdout_success "All CNF resources are up!", same_line: true end end @@ -836,13 +852,13 @@ module CNFManager Log.info { "helm_install[:error].to_s: #{helm_install[:error].to_s}" } Log.info { "helm_install[:error].to_s.size: #{helm_install[:error].to_s.size}" } helm_used = false - if helm_install && helm_error == false # fails on warnings ... && helm_install[:error].to_s.size == 0 # && helm_pull.to_s.size > 0 + if helm_install && !helm_error # fails on warnings ... && helm_install[:error].to_s.size == 0 # && helm_pull.to_s.size > 0 helm_used = true stdout_success "Successfully setup #{release_name}" end # Not required to write elapsed time configmap if the cnf already exists due to a previous Helm install - return true if fresh_install == false + return true if !fresh_install # Immutable config maps are only supported in Kubernetes 1.19+ immutable_configmap = true diff --git a/src/tasks/utils/utils.cr b/src/tasks/utils/utils.cr index abef8512f..657fb5187 100644 --- a/src/tasks/utils/utils.cr +++ b/src/tasks/utils/utils.cr @@ -407,20 +407,28 @@ def stdout_info(msg) puts msg end -def stdout_success(msg) - puts msg.colorize(:green) +# \e[1A\e[K is a sequence that allows to clear current line and place cursor at its beginning +def stdout_colored(msg, color, same_line=false) + if same_line + msg = "#{"\e[1A\e[K"}#{msg}" + end + puts msg.colorize(color) +end + +def stdout_success(msg, same_line=false) + stdout_colored(msg, :green, same_line) end -def stdout_warning(msg) - puts msg.colorize(:yellow) +def stdout_warning(msg, same_line=false) + stdout_colored(msg, :yellow, same_line) end -def stdout_failure(msg) - puts msg.colorize(:red) +def stdout_failure(msg, same_line=false) + stdout_colored(msg, :red, same_line) end -def stdout_error(msg) - puts msg.colorize(Colorize::Color256.new(208)) +def stdout_error(msg, same_line=false) + stdout_colored(msg, Colorize::Color256.new(208), same_line) end def stdout_score(test_name)