cnf_setup: Exit on setup failure or timeout

Fix handling unsuccessful cnf setup, if resources are not ready
after timeout - then setup should fail with exit 1.
Change specs accordingly, replace wait_count=0 with new option
"ignore_setup_timeout".
Add cnf_setup function to ShellCmd and use it for
installation of CNFs during specs.

Refs: cnti-testcatalog#1937 cnti-testcatalog#2083
Signed-off-by: Konstantin Yarovoy <[email protected]>

USAGE.md

@@ -59,6 +59,17 @@ crystal build src/cnf-testsuite.cr
 ./cnf-testsuite validate_config cnf-config=[PATH_TO]/cnf-testsuite.yml
 ```
 
+#### Installing a cnf:
+
+```
+./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml
+```
+
+##### Skip waiting for resource readiness during installation:
+```
+./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml skip_wait_for_install
+```
+
 #### Running all of the platform and workload tests:
 
 ```

spec/5g/core_spec.cr

@@ -14,8 +14,7 @@ describe "Core" do
 
   it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose")
       (/(PASSED).*(Chaos service degradation is less than 50%)/ =~ result[:output]).should_not be_nil
     ensure
@@ -26,8 +25,7 @@ describe "Core" do
 
   it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose baseline_count=300")
       (/(FAILED).*(Chaos service degradation is more than 50%)/ =~ result[:output]).should_not be_nil
     ensure
@@ -38,8 +36,7 @@ describe "Core" do
 
   it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("suci_enabled verbose")
       (/(PASSED).*(Core uses SUCI 5g authentication)/ =~ result[:output]).should_not be_nil
     ensure
@@ -50,8 +47,7 @@ describe "Core" do
 
   it "'suci_enabled' should fail if the 5G core does not have suci enabled", tags: ["5g"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("suci_enabled verbose")
       (/(FAILED).*(Core does not use SUCI 5g authentication)/ =~ result[:output]).should_not be_nil
     ensure

spec/5g/ran_spec.cr

@@ -15,10 +15,8 @@ describe "5g" do
 
   it "'oran_e2_connection' should pass if the ORAN enabled RAN connects to the RIC using the e2 standard", tags: ["oran"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml")
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("oran_e2_connection verbose")
       (/(PASSED).*(RAN connects to a RIC using the e2 standard interface)/ =~ result[:output]).should_not be_nil
     ensure
@@ -31,10 +29,8 @@ describe "5g" do
 
   it "'oran_e2_connection' should fail if the ORAN enabled RAN does not connect to the RIC using the e2 standard", tags: ["oran"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") 
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml")
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml")
       result = ShellCmd.run_testsuite("oran_e2_connection verbose")
       (/(FAILED).*(RAN does not connect to a RIC using the e2 standard interface)/ =~ result[:output]).should_not be_nil
     ensure

spec/setup_spec.cr

@@ -66,8 +66,7 @@ describe "Setup" do
 
   it "'cnf_setup/cnf_cleanup' should install/cleanup with cnf-path arg as alias for cnf-config", tags: ["setup"] do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-path=example-cnfs/coredns/cnf-testsuite.yml")
-      result[:status].success?.should be_true
+      result = ShellCmd.cnf_setup("cnf-path=example-cnfs/coredns/cnf-testsuite.yml")
       (/Successfully setup coredns/ =~ result[:output]).should_not be_nil
     ensure
       result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=example-cnfs/coredns/cnf-testsuite.yml")
@@ -78,8 +77,7 @@ describe "Setup" do
 
   it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf with a cnf-testsuite.yml", tags: ["setup"] do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml")
-      result[:status].success?.should be_true
+      result = ShellCmd.cnf_setup("cnf-config=example-cnfs/coredns/cnf-testsuite.yml")
       (/Successfully setup coredns/ =~ result[:output]).should_not be_nil
     ensure
       result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=example-cnfs/coredns/cnf-testsuite.yml")
@@ -91,8 +89,7 @@ describe "Setup" do
   it "'cnf_setup/cnf_cleanup' should work with cnf-testsuite.yml that has no directory associated with it", tags: ["setup"] do
     begin
       #TODO force cnfs/<name> to be deployment name and not the directory name
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=spec/fixtures/cnf-testsuite.yml verbose")
-      result[:status].success?.should be_true
+      result = ShellCmd.cnf_setup("cnf-config=spec/fixtures/cnf-testsuite.yml verbose")
       (/Successfully setup coredns/ =~ result[:output]).should_not be_nil
     ensure
 
@@ -104,8 +101,7 @@ describe "Setup" do
 
   it "'cnf_setup/cnf_cleanup' should install/cleanup with helm_directory that descends multiple directories", tags: ["setup"] do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml")
-      result[:status].success?.should be_true
+      result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml")
       (/Successfully setup coredns/ =~ result[:output]).should_not be_nil
     ensure
       result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml")

spec/spec_helper.cr

@@ -20,4 +20,14 @@ module ShellCmd
     cmd = "#{cmd_prefix} ./cnf-testsuite #{testsuite_cmd}"
     run(cmd, log_prefix: "ShellCmd.run_testsuite", force_output: true, joined_output: true)
   end
+
+  def self.cnf_setup(setup_params, cmd_prefix="", expect_failure=false)
+    result = run_testsuite("cnf_setup #{setup_params} wait_count=300", cmd_prefix)
+    if !expect_failure
+      result[:status].success?.should be_true
+    else
+      result[:status].success?.should be_false
+    end
+    result
+  end
 end

spec/utils/cnf_manager_spec.cr

@@ -30,15 +30,13 @@ describe "SampleUtils" do
   end
 
   it "'cnf_setup' should pass with a minimal cnf-testsuite.yml", tags: ["cnf-setup"] do
-    result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-minimal-cnf/ wait_count=0")
-    result[:status].success?.should be_true
+    ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install")
   ensure
     result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true")
   end
 
   it "'cnf_setup' should support cnf-config as an alias for cnf-path", tags: ["cnf-setup"] do
-    result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample-minimal-cnf/ wait_count=0")
-    result[:status].success?.should be_true
+    ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install")
   ensure
     result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true")
   end
@@ -160,7 +158,7 @@ describe "SampleUtils" do
 
 
   it "'CNFManager.sample_setup_cli_args(args) and CNFManager.sample_setup(cli_args)' should set up a sample cnf", tags: ["cnf-setup"]  do
-    args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose", "wait_count=180"])
+    args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     CNFManager.sample_setup(cli_hash)
     config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(cli_hash[:config_file]))    
@@ -175,7 +173,7 @@ describe "SampleUtils" do
 
   it "'CNFManager.sample_setup' should set up a sample cnf", tags: ["cnf-setup"]  do
     config_file = "sample-cnfs/sample-generic-cnf"
-    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     CNFManager.sample_setup(cli_hash)
     # check if directory exists
@@ -191,7 +189,7 @@ describe "SampleUtils" do
 
   it "'CNFManager.sample_setup_args' should set up a sample cnf from a argument", tags: ["cnf-setup"]  do
     config_file = "sample-cnfs/sample-generic-cnf"
-    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     CNFManager.sample_setup(cli_hash)
     # check if directory exists
@@ -205,7 +203,7 @@ describe "SampleUtils" do
 
   it "'CNFManager.sample_setup_args' should set up a sample cnf from a config file", tags: ["cnf-setup"]  do
     config_file = "sample-cnfs/sample-generic-cnf"
-    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     CNFManager.sample_setup(cli_hash)
     # check if directory exists
@@ -218,7 +216,7 @@ describe "SampleUtils" do
   end
 
   it "'CNFManager.sample_cleanup' should clean up a sample cnf from a argument", tags: ["cnf-setup"]  do
-    args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose", "skip_wait_for_install"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     CNFManager.sample_setup(cli_hash)
     cleanup = CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-generic-cnf", verbose: true)
@@ -230,7 +228,7 @@ describe "SampleUtils" do
 
   it "'CNFManager.sample_setup_args' should be able to deploy using a helm_directory", tags: ["cnf-setup"]  do
     config_file = "sample-cnfs/sample_privileged_cnf"
-    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     CNFManager.sample_setup(cli_hash)
     config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file))    
@@ -246,7 +244,7 @@ describe "SampleUtils" do
 
   it "'CNFManager.sample_setup_args and CNFManager.sample_cleanup' should be able to deploy and cleanup using a manifest_directory", tags: ["cnf-setup"]  do
     config_file = "sample-cnfs/k8s-non-helm"
-    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     Log.info { "Running Setup" }
     CNFManager.sample_setup(cli_hash)
@@ -270,7 +268,7 @@ describe "SampleUtils" do
 
   it "'CNFManager.cnf_config_list' should return a list of all of the config files from the cnf directory", tags: ["cnf-setup"]  do
     config_file = "sample-cnfs/sample-generic-cnf"
-    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     CNFManager.sample_setup(cli_hash)
     args = Sam::Args.new(["cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml", "verbose"])
@@ -283,7 +281,7 @@ describe "SampleUtils" do
 
   it "'CNFManager.helm_repo_add' should add a helm repo if the helm repo is valid", tags: ["helm-repo"] do
     config_file = "sample-cnfs/sample-generic-cnf"
-    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "wait_count=0"])
+    args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose"])
     cli_hash = CNFManager.sample_setup_cli_args(args)
     args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml"])
     CNFManager.helm_repo_add(args: args).should eq(true)
@@ -395,7 +393,7 @@ describe "SampleUtils" do
   it "bonus tests should not be includded in the maximum points when a failure occurs", tags: ["cnf-config"]  do
     begin
       # fails because doesn't have a service
-      result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample-ndn-privileged")
+      ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-ndn-privileged")
       result = ShellCmd.run_testsuite("cert_microservice")
       (/of 6 tests passed/ =~ result[:output]).should_not be_nil
     ensure
@@ -406,7 +404,7 @@ describe "SampleUtils" do
   it "Helm_values should be used during the installation of a cnf", tags: ["cnf-config"]  do
     begin
       # fails because doesn't have a service
-      result = ShellCmd.run_testsuite("cnf_setup cnf-path=./sample-cnfs/sample_coredns_values") 
+      ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_coredns_values")
       deployment_containers = KubectlClient::Get.deployment_containers("coredns-coredns")
       image_tags = KubectlClient::Get.container_image_tags(deployment_containers) 
       Log.info { "image_tags: #{image_tags}" }

spec/utils/kubescape_spec.cr

@@ -8,7 +8,7 @@ describe "K8sInstrumentation" do
   end
 
   it "'#scan and #test_by_test_name' should return the results of a kubescape scan", tags: ["kubescape"]  do
-    result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml")
+    ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml")
     Kubescape.scan
     results_json = Kubescape.parse
     test_json = Kubescape.test_by_test_name(results_json, "Network policies")

spec/utils/utils_spec.cr

@@ -152,8 +152,8 @@ describe "Utils" do
 
   it "'all_cnfs_task_runner' should run a test against all cnfs in the cnfs directory if there is not cnf-config argument passed to it", tags: ["task_runner"]  do
     my_args = Sam::Args.new
-      result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-generic-cnf")
-      result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample_privileged_cnf")
+      ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-generic-cnf")
+      ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_privileged_cnf")
     task_response = CNFManager::Task.all_cnfs_task_runner(my_args) do |args, config|
       Log.info { "all_cnfs_task_runner spec args #{args.inspect}" }
       Log.for("verbose").info { "privileged" } if check_verbose(args)
@@ -192,8 +192,8 @@ describe "Utils" do
   end
 
   it "'task_runner' should run a test against a single cnf if passed a cnf-config argument even if there are multiple cnfs installed", tags: ["task_runner"]  do
-    result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml")
-    result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml")
+    ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml")
+    ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml")
     result = ShellCmd.run_testsuite("privileged")
     (/(FAILED).*(Found 1 privileged containers)/ =~ result[:output]).should_not be_nil
   ensure
@@ -252,7 +252,7 @@ describe "Utils" do
   end
 
   it "'logger' or verbose output should be shown when verbose flag is set", tags: ["logger"] do
-    result = ShellCmd.run_testsuite("cnf_setup cnf-path=sample-cnfs/sample-coredns-cnf")
+    ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf")
     result = ShellCmd.run_testsuite("helm_deploy verbose", cmd_prefix: "LOG_LEVEL=info")
     puts result[:output]
     result[:status].success?.should be_true

spec/workload/compatibility_spec.cr

@@ -14,10 +14,10 @@ describe "Compatibility" do
 
   it "'cni_compatible' should pass if the cnf works with calico and flannel", tags: ["compatibility"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml")
       retry_limit = 5 
       retries = 1
+      result = ShellCmd.run_testsuite("cni_compatible verbose")
       until (/PASSED/ =~ result[:output]) || retries > retry_limit
         Log.info { "cni_compatible spec retry: #{retries}" }
         sleep 1.0
@@ -34,8 +34,7 @@ describe "Compatibility" do
 
   it "'increase_decrease_capacity' should pass ", tags: ["increase_decrease_capacity"]  do
     begin
-      result = ShellCmd.run_testsuite("cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose wait_count=0")
-      result[:status].success?.should be_true
+      ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install")
       result = ShellCmd.run_testsuite("increase_decrease_capacity verbose")
       result[:status].success?.should be_true
       (/(PASSED).*(Replicas increased to)/ =~ result[:output]).should_not be_nil