spec: remove backticks and use new logging

REF: cnti-testcatalog#1495 Replace all backtick command execution with ShellCmd module. Use Log instead of LOGGING module. Fix edgecase issues caused by logging refactor. Remove commented-out code. Signed-off-by: Konstantin Yarovoy <[email protected]>
kosstennbl · Apr 22, 2024 · af7b839 · af7b839
1 parent 8114e5d
commit af7b839
Show file tree

Hide file tree

Showing 46 changed files with 1,261 additions and 1,814 deletions.
diff --git a/spec/5g/core_spec.cr b/spec/5g/core_spec.cr
@@ -8,58 +8,55 @@ require "sam"
 describe "Core" do
 
   before_all do
-    `./cnf-testsuite setup`
-    $?.success?.should be_true end
+    result = ShellCmd.run_testsuite("setup")
+    result[:status].success?.should be_true 
+  end
 
   it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"]  do
     begin
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      response_s = `./cnf-testsuite smf_upf_heartbeat verbose`
-      Log.info {"response: #{response_s}"}
-      (/PASSED: Chaos service degradation is less than 50%/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose")
+      (/PASSED: Chaos service degradation is less than 50%/ =~ result[:output]).should_not be_nil
     ensure
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
     end
   end
 
   it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"]  do
     begin
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      response_s = `./cnf-testsuite smf_upf_heartbeat verbose baseline_count=300`
-      Log.info {"response: #{response_s}"}
-      (/FAILED: Chaos service degradation is more than 50%/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose baseline_count=300")
+      (/FAILED: Chaos service degradation is more than 50%/ =~ result[:output]).should_not be_nil
     ensure
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
     end
   end
 
   it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"]  do
     begin
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      response_s = `./cnf-testsuite suci_enabled verbose`
-      Log.info {"response: #{response_s}"}
-      (/PASSED: Core uses SUCI 5g authentication/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("suci_enabled verbose")
+      (/PASSED: Core uses SUCI 5g authentication/ =~ result[:output]).should_not be_nil
     ensure
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
     end
   end
 
   it "'suci_enabled' should fail if the 5G core does not have suci enabled", tags: ["5g"]  do
     begin
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      response_s = `./cnf-testsuite suci_enabled verbose`
-      Log.info {"response: #{response_s}"}
-      (/FAILED: Core does not use SUCI 5g authentication/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("suci_enabled verbose")
+      (/FAILED: Core does not use SUCI 5g authentication/ =~ result[:output]).should_not be_nil
     ensure
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml`}
-      $?.success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
     end
   end
 

diff --git a/spec/5g/ran_spec.cr b/spec/5g/ran_spec.cr
@@ -8,42 +8,40 @@ require "sam"
 describe "5g" do
 
   before_all do
-    `./cnf-testsuite setup`
-    $?.success?.should be_true
+    result = ShellCmd.run_testsuite("setup")
+    result[:status].success?.should be_true
   end
 
 
   it "'oran_e2_connection' should pass if the ORAN enabled RAN connects to the RIC using the e2 standard", tags: ["oran"]  do
     begin
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      response_s = `./cnf-testsuite oran_e2_connection verbose`
-      Log.info {"response: #{response_s}"}
-      (/PASSED: RAN connects to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("oran_e2_connection verbose")
+      (/PASSED: RAN connects to a RIC using the e2 standard interface/ =~ result[:output]).should_not be_nil
     ensure
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
     end
   end
 
   it "'oran_e2_connection' should fail if the ORAN enabled RAN does not connect to the RIC using the e2 standard", tags: ["oran"]  do
     begin
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      response_s = `./cnf-testsuite oran_e2_connection verbose`
-      Log.info {"response: #{response_s}"}
-      (/FAILED: RAN does not connect to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("oran_e2_connection verbose")
+      (/FAILED: RAN does not connect to a RIC using the e2 standard interface/ =~ result[:output]).should_not be_nil
     ensure
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
-      $?.success?.should be_true
-      Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml`}
-      $?.success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") 
+      result[:status].success?.should be_true
     end
   end
 

diff --git a/spec/airgap_task_spec.cr b/spec/airgap_task_spec.cr
@@ -1,15 +1,14 @@
 require "./spec_helper"
 require "colorize"
-# require "../src/tasks/utils/utils.cr"
+require "../src/tasks/utils/utils.cr"
 require "file_utils"
 require "sam"
 
 describe "AirGap" do
 
   before_all do
     Helm.helm_repo_add("chaos-mesh", "https://charts.chaos-mesh.org")
-    # AirGap::LOGGING.info `./cnf-testsuite airgapped output-file=/tmp/airgapped.tar.gz` unless File.exists?("/tmp/airgapped.tar.gz")
-    Log.info { `./cnf-testsuite airgapped output-file=/tmp/airgapped.tar.gz` }
+    result = ShellCmd.run_testsuite("airgapped output-file=/tmp/airgapped.tar.gz")
     (File.exists?("/tmp/airgapped.tar.gz")).should be_true
   end
 
@@ -19,7 +18,7 @@ describe "AirGap" do
   end
 
   it "'setup' task should create an airgapped tarball with the necessary files", tags: ["airgap-setup"] do
-    file_list = `tar -tvf /tmp/airgapped.tar.gz`
+    file_list = ShellCmd.run("tar -tvf /tmp/airgapped.tar.gz")[:output]
     Log.info { "file_list: #{file_list}" }
     (file_list).match(/kubectl.tar/).should_not be_nil
     (file_list).match(/chaos-mesh.tar/).should_not be_nil
@@ -36,9 +35,8 @@ describe "AirGap" do
   end
 
   it "'setup' task should install the necessary cri tools in the cluster", tags: ["airgap-setup"] do
-    response_s = `./cnf-testsuite -l info setup offline=/tmp/airgapped.tar.gz`
-    Log.info { response_s }
-    $?.success?.should be_true
+    result = ShellCmd.run_testsuite("-l info setup offline=/tmp/airgapped.tar.gz")
+    result[:status].success?.should be_true
     pods = KubectlClient::Get.pods_by_nodes(KubectlClient::Get.schedulable_nodes_list)
     pods = KubectlClient::Get.pods_by_label(pods, "name", "cri-tools")
     # Get the generated name of the cri-tools per node
@@ -49,61 +47,51 @@ describe "AirGap" do
       sh = KubectlClient.exec("-ti #{pod_name} -- cat /usr/local/bin/ctr > /dev/null")  
       sh[:status].success?
     end
-    (/All prerequisites found./ =~ response_s).should_not be_nil
-    (/Setup complete/ =~ response_s).should_not be_nil
+    (/All prerequisites found./ =~ result[:output]).should_not be_nil
+    (/Setup complete/ =~ result[:output]).should_not be_nil
   end
 
   it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf helm chart in airgapped mode", tags: ["airgap-repo"]  do
     begin
-      response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz`
-      Log.info { response_s }
-      file_list = `tar -tvf /tmp/airgapped.tar.gz`
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz")
+      file_list = ShellCmd.run("tar -tvf /tmp/airgapped.tar.gz")[:output]
       Log.info { "file_list: #{file_list}" }
       (file_list).match(/coredns_1.8.0.tar/).should_not be_nil
       (file_list).match(/coredns_1.6.7.tar/).should_not be_nil
-      response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz`
-      $?.success?.should be_true
-      Log.info { response_s }
-      (/Successfully setup coredns/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz")
+      result[:status].success?.should be_true
+      (/Successfully setup coredns/ =~ result[:output]).should_not be_nil
     ensure
-      response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml wait_count=0`
-      $?.success?.should be_true
-      Log.info { response_s }
-      (/Successfully cleaned up/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml wait_count=0")
+      result[:status].success?.should be_true
+      (/Successfully cleaned up/ =~ result[:output]).should_not be_nil
     end
   end
 
   it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf helm directory in airgapped mode", tags: ["airgap-directory"]  do
     begin
-      response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz`
-      Log.info { response_s }
-      response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz`
-      Log.info { response_s }
-      $?.success?.should be_true
-      (/Successfully setup coredns/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz")
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz")
+      result[:status].success?.should be_true
+      (/Successfully setup coredns/ =~ result[:output]).should_not be_nil
     ensure
-      response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml wait_count=0`
-      $?.success?.should be_true
-      Log.info { response_s }
-      (/Successfully cleaned up/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml wait_count=0")
+      result[:status].success?.should be_true
+      (/Successfully cleaned up/ =~ result[:output]).should_not be_nil
     end
   end
 
   it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf manifest directory in airgapped mode", tags: ["airgap-manifest"]  do
     begin
-      response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz`
-      Log.info { response_s }
-      response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz`
-      Log.info { response_s }
-      $?.success?.should be_true
-      Log.info { response_s }
-      (/Successfully setup nginx-webapp/ =~ response_s).should_not be_nil
-      (/exported_chart\" not found/ =~ response_s).should be_nil
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz")
+      result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz")
+      result[:status].success?.should be_true
+      (/Successfully setup nginx-webapp/ =~ result[:output]).should_not be_nil
+      (/exported_chart\" not found/ =~ result[:output]).should be_nil
     ensure
-      response_s = `LOG_LEVEL=debug ./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml wait_count=0`
-      $?.success?.should be_true
-      Log.info { response_s }
-      (/Successfully cleaned up/ =~ response_s).should_not be_nil
+      result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml wait_count=0", cmd_prefix: "LOG_LEVEL=debug")
+      result[:status].success?.should be_true
+      (/Successfully cleaned up/ =~ result[:output]).should_not be_nil
     end
   end
 

diff --git a/spec/cluster_setup_spec.cr b/spec/cluster_setup_spec.cr
@@ -7,34 +7,31 @@ require "cluster_tools"
 describe "Cluster Setup" do
 
   before_each do
-    `./cnf-testsuite cleanup`
-    $?.success?.should be_true
+    result = ShellCmd.run_testsuite("cleanup")
+    result[:status].success?.should be_true
   end
 
   it "'install_cluster_tools' should give a message if namespace does not exist", tags: ["cluster_setup"]  do
     KubectlClient::Delete.command("namespace #{ClusterTools.namespace}")
-    response_s = `./cnf-testsuite install_cluster_tools`
-    LOGGING.info response_s
-    $?.success?.should be_false
-    (/please run cnf-testsuite setup/ =~ response_s).should_not be_nil
+    result = ShellCmd.run_testsuite("install_cluster_tools")
+    result[:status].success?.should be_false
+    (/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil
   end
 
   it "'install_cluster_tools' should give a message if namespace does not exist even after setup", tags: ["cluster_setup"]  do
-    `./cnf-testsuite setup`
+    result = ShellCmd.run_testsuite("setup")
 
     KubectlClient::Delete.command("namespace #{ClusterTools.namespace}")
 
-    response_s = `./cnf-testsuite install_cluster_tools`
-    LOGGING.info response_s
-    $?.success?.should be_false
-    (/please run cnf-testsuite setup/ =~ response_s).should_not be_nil
+    result = ShellCmd.run_testsuite("install_cluster_tools")
+    result[:status].success?.should be_false
+    (/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil
   end
 
   it "'uninstall_cluster_tools' should give a message if namespace does not exist", tags: ["cluster_setup"]  do
-    response_s = `./cnf-testsuite uninstall_cluster_tools`
-    LOGGING.info response_s
-    $?.success?.should be_false
-    (/please run cnf-testsuite setup/ =~ response_s).should_not be_nil
+    result = ShellCmd.run_testsuite("uninstall_cluster_tools")
+    result[:status].success?.should be_false
+    (/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil
   end
 
 end
diff --git a/spec/cnf_testsuite_all/cnf_testsuite_config_lifecycle_spec.cr b/spec/cnf_testsuite_all/cnf_testsuite_config_lifecycle_spec.cr
@@ -1,29 +1,7 @@
 #TODO Update Test
-# require "../spec_helper"
-# require "../../src/tasks/utils/utils.cr"
-# require "colorize"
+require "../spec_helper"
+require "../../src/tasks/utils/utils.cr"
+require "colorize"
 
-# describe CnfTestSuite do
-#   before_all do
-#     `./cnf-testsuite setup`
-#     $?.success?.should be_true
-#   end
-
-#   after_all do
-#     `./cnf-testsuite samples_cleanup`
-#     $?.success?.should be_true
-#   end
-
-#  it "'testsuite all' should run the configuration lifecycle tests", tags: ["testsuite-config-lifecycle"] do
-#     `./cnf-testsuite samples_cleanup`
-#     response_s = `./cnf-testsuite all ~reasonable_startup_time ~reasonable_image_size ~disk_fill ~pod_delete ~pod_io_stress ~pod_network_latency ~pod_network_corruption ~pod_network_duplication ~pod_memory_hog ~chaos_network_loss ~node_drain ~pod_dns_error ~chaos_cpu_hog ~chaos_container_kill ~platform ~volume_hostpath_not_found ~privileged ~increase_capacity ~decrease_capacity ~helm_chart_valid ~helm_chart_published "cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml" verbose`
-#     LOGGING.info response_s
-#     (/PASSED: Helm readiness probe found/ =~ response_s).should_not be_nil
-#     (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil
-#     (/Final workload score:/ =~ response_s).should_not be_nil
-#     (/Final score:/ =~ response_s).should_not be_nil
-#     (CNFManager::Points.all_result_test_names(CNFManager::Points.final_cnf_results_yml).sort).should eq(["ip_addresses", "liveness", "readiness", "rolling_update", "rolling_downgrade", "rolling_version_change", "nodeport_not_used", "hostport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "rollback", "secrets_used", "immutable_configmap"].sort)
-#     (/^.*\.cr:[0-9].*/ =~ response_s).should be_nil
-#     $?.success?.should be_true
-#   end
-# end
+describe CnfTestSuite do
+end