Skip to content

Commit

Permalink
spec: remove backticks and use new logging
Browse files Browse the repository at this point in the history
REF: cnti-testcatalog#1495
Replace all backtick command execution with ShellCmd module.
Use Log instead of LOGGING module.
Fix edgecase issues caused by logging refactor.
Remove commented-out code.

Signed-off-by: Konstantin Yarovoy <[email protected]>
  • Loading branch information
Konstantin Yarovoy committed Apr 22, 2024
1 parent 675c9de commit 98d16cb
Show file tree
Hide file tree
Showing 45 changed files with 1,219 additions and 1,864 deletions.
57 changes: 27 additions & 30 deletions spec/5g/core_spec.cr
Original file line number Diff line number Diff line change
Expand Up @@ -8,58 +8,55 @@ require "sam"
describe "Core" do

before_all do
`./cnf-testsuite setup`
$?.success?.should be_true end
result = ShellCmd.run_testsuite("setup")
result[:status].success?.should be_true
end

it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"] do
begin
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
response_s = `./cnf-testsuite smf_upf_heartbeat verbose`
Log.info {"response: #{response_s}"}
(/PASSED: Chaos service degradation is less than 50%/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose")
(/PASSED: Chaos service degradation is less than 50%/ =~ result[:output]).should_not be_nil
ensure
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
end
end

it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"] do
begin
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
response_s = `./cnf-testsuite smf_upf_heartbeat verbose baseline_count=300`
Log.info {"response: #{response_s}"}
(/FAILED: Chaos service degradation is more than 50%/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose baseline_count=300")
(/FAILED: Chaos service degradation is more than 50%/ =~ result[:output]).should_not be_nil
ensure
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
end
end

it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"] do
begin
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
response_s = `./cnf-testsuite suci_enabled verbose`
Log.info {"response: #{response_s}"}
(/PASSED: Core uses SUCI 5g authentication/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("suci_enabled verbose")
(/PASSED: Core uses SUCI 5g authentication/ =~ result[:output]).should_not be_nil
ensure
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
end
end

it "'suci_enabled' should fail if the 5G core does not have suci enabled", tags: ["5g"] do
begin
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml`}
$?.success?.should be_true
response_s = `./cnf-testsuite suci_enabled verbose`
Log.info {"response: #{response_s}"}
(/FAILED: Core does not use SUCI 5g authentication/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("suci_enabled verbose")
(/FAILED: Core does not use SUCI 5g authentication/ =~ result[:output]).should_not be_nil
ensure
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml`}
$?.success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml")
result[:status].success?.should be_true
end
end

Expand Down
46 changes: 22 additions & 24 deletions spec/5g/ran_spec.cr
Original file line number Diff line number Diff line change
Expand Up @@ -8,42 +8,40 @@ require "sam"
describe "5g" do

before_all do
`./cnf-testsuite setup`
$?.success?.should be_true
result = ShellCmd.run_testsuite("setup")
result[:status].success?.should be_true
end


it "'oran_e2_connection' should pass if the ORAN enabled RAN connects to the RIC using the e2 standard", tags: ["oran"] do
begin
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml`}
$?.success?.should be_true
response_s = `./cnf-testsuite oran_e2_connection verbose`
Log.info {"response: #{response_s}"}
(/PASSED: RAN connects to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("oran_e2_connection verbose")
(/PASSED: RAN connects to a RIC using the e2 standard interface/ =~ result[:output]).should_not be_nil
ensure
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml`}
$?.success?.should be_true
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
end
end

it "'oran_e2_connection' should fail if the ORAN enabled RAN does not connect to the RIC using the e2 standard", tags: ["oran"] do
begin
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
Log.info {`./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml`}
$?.success?.should be_true
response_s = `./cnf-testsuite oran_e2_connection verbose`
Log.info {"response: #{response_s}"}
(/FAILED: RAN does not connect to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample_srsran_ueauth_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("cnf_setup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("oran_e2_connection verbose")
(/FAILED: RAN does not connect to a RIC using the e2 standard interface/ =~ result[:output]).should_not be_nil
ensure
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`}
$?.success?.should be_true
Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml`}
$?.success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml")
result[:status].success?.should be_true
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml")
result[:status].success?.should be_true
end
end

Expand Down
104 changes: 1 addition & 103 deletions spec/airgap_task_spec.cr
Original file line number Diff line number Diff line change
@@ -1,110 +1,8 @@
require "./spec_helper"
require "colorize"
# require "../src/tasks/utils/utils.cr"
require "../src/tasks/utils/utils.cr"
require "file_utils"
require "sam"

describe "AirGap" do

# before_all do
# Helm.helm_repo_add("chaos-mesh", "https://charts.chaos-mesh.org")
# # AirGap::LOGGING.info `./cnf-testsuite airgapped output-file=/tmp/airgapped.tar.gz` unless File.exists?("/tmp/airgapped.tar.gz")
# Log.info { `./cnf-testsuite airgapped output-file=/tmp/airgapped.tar.gz` }
# (File.exists?("/tmp/airgapped.tar.gz")).should be_true
# end
#
# after_all do
# AirGap.tmp_cleanup
# (File.exists?("/tmp/airgapped.tar.gz")).should_not be_true
# end

# it "'setup' task should create an airgapped tarball with the necessary files", tags: ["airgap-setup"] do
# file_list = `tar -tvf /tmp/airgapped.tar.gz`
# Log.info { "file_list: #{file_list}" }
# (file_list).match(/kubectl.tar/).should_not be_nil
# (file_list).match(/chaos-mesh.tar/).should_not be_nil
# (file_list).match(/chaos-daemon.tar/).should_not be_nil
# (file_list).match(/chaos-dashboard.tar/).should_not be_nil
# (file_list).match(/chaos-kernel.tar/).should_not be_nil
# (file_list).match(/prometheus.tar/).should_not be_nil
# (file_list).match(/rbac.yaml/).should_not be_nil
# (file_list).match(/disk-fill-rbac.yaml/).should_not be_nil
# (file_list).match(/litmus-operator/).should_not be_nil
# (file_list).match(/download\/sonobuoy.tar.gz/).should_not be_nil
# (file_list).match(/crictl-v1.17.0-linux-amd64.tar.gz/).should_not be_nil
# (file_list).match(/containerd-1.5.0-linux-amd64.tar.gz/).should_not be_nil
# end
#
# it "'setup' task should install the necessary cri tools in the cluster", tags: ["airgap-setup"] do
# response_s = `./cnf-testsuite -l info setup offline=/tmp/airgapped.tar.gz`
# Log.info { response_s }
# $?.success?.should be_true
# pods = KubectlClient::Get.pods_by_nodes(KubectlClient::Get.schedulable_nodes_list)
# pods = KubectlClient::Get.pods_by_label(pods, "name", "cri-tools")
# # Get the generated name of the cri-tools per node
# pods.map do |pod|
# pod_name = pod.dig?("metadata", "name")
# sh = KubectlClient.exec("-ti #{pod_name} -- cat /usr/local/bin/crictl > /dev/null")
# sh[:status].success?
# sh = KubectlClient.exec("-ti #{pod_name} -- cat /usr/local/bin/ctr > /dev/null")
# sh[:status].success?
# end
# (/All prerequisites found./ =~ response_s).should_not be_nil
# (/Setup complete/ =~ response_s).should_not be_nil
# end
#
# it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf helm chart in airgapped mode", tags: ["airgap-repo"] do
# begin
# response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz`
# Log.info { response_s }
# file_list = `tar -tvf /tmp/airgapped.tar.gz`
# Log.info { "file_list: #{file_list}" }
# (file_list).match(/coredns_1.8.0.tar/).should_not be_nil
# (file_list).match(/coredns_1.6.7.tar/).should_not be_nil
# response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz`
# $?.success?.should be_true
# Log.info { response_s }
# (/Successfully setup coredns/ =~ response_s).should_not be_nil
# ensure
# response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_rolling/cnf-testsuite.yml wait_count=0`
# $?.success?.should be_true
# Log.info { response_s }
# (/Successfully cleaned up/ =~ response_s).should_not be_nil
# end
# end
#
# it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf helm directory in airgapped mode", tags: ["airgap-directory"] do
# begin
# response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz`
# Log.info { response_s }
# response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz`
# Log.info { response_s }
# $?.success?.should be_true
# (/Successfully setup coredns/ =~ response_s).should_not be_nil
# ensure
# response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml wait_count=0`
# $?.success?.should be_true
# Log.info { response_s }
# (/Successfully cleaned up/ =~ response_s).should_not be_nil
# end
# end
#
# it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf manifest directory in airgapped mode", tags: ["airgap-manifest"] do
# begin
# response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz`
# Log.info { response_s }
# response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml input-file=/tmp/airgapped.tar.gz`
# Log.info { response_s }
# $?.success?.should be_true
# Log.info { response_s }
# (/Successfully setup nginx-webapp/ =~ response_s).should_not be_nil
# (/exported_chart\" not found/ =~ response_s).should be_nil
# ensure
# response_s = `LOG_LEVEL=debug ./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml wait_count=0`
# $?.success?.should be_true
# Log.info { response_s }
# (/Successfully cleaned up/ =~ response_s).should_not be_nil
# end
# end

end
27 changes: 12 additions & 15 deletions spec/cluster_setup_spec.cr
Original file line number Diff line number Diff line change
Expand Up @@ -7,34 +7,31 @@ require "cluster_tools"
describe "Cluster Setup" do

before_each do
`./cnf-testsuite cleanup`
$?.success?.should be_true
result = ShellCmd.run_testsuite("cleanup")
result[:status].success?.should be_true
end

it "'install_cluster_tools' should give a message if namespace does not exist", tags: ["cluster_setup"] do
KubectlClient::Delete.command("namespace #{ClusterTools.namespace}")
response_s = `./cnf-testsuite install_cluster_tools`
LOGGING.info response_s
$?.success?.should be_false
(/please run cnf-testsuite setup/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("install_cluster_tools")
result[:status].success?.should be_false
(/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil
end

it "'install_cluster_tools' should give a message if namespace does not exist even after setup", tags: ["cluster_setup"] do
`./cnf-testsuite setup`
result = ShellCmd.run_testsuite("setup")

KubectlClient::Delete.command("namespace #{ClusterTools.namespace}")

response_s = `./cnf-testsuite install_cluster_tools`
LOGGING.info response_s
$?.success?.should be_false
(/please run cnf-testsuite setup/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("install_cluster_tools")
result[:status].success?.should be_false
(/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil
end

it "'uninstall_cluster_tools' should give a message if namespace does not exist", tags: ["cluster_setup"] do
response_s = `./cnf-testsuite uninstall_cluster_tools`
LOGGING.info response_s
$?.success?.should be_false
(/please run cnf-testsuite setup/ =~ response_s).should_not be_nil
result = ShellCmd.run_testsuite("uninstall_cluster_tools")
result[:status].success?.should be_false
(/please run cnf-testsuite setup/ =~ result[:output]).should_not be_nil
end

end
32 changes: 5 additions & 27 deletions spec/cnf_testsuite_all/cnf_testsuite_config_lifecycle_spec.cr
Original file line number Diff line number Diff line change
@@ -1,29 +1,7 @@
#TODO Update Test
# require "../spec_helper"
# require "../../src/tasks/utils/utils.cr"
# require "colorize"
require "../spec_helper"
require "../../src/tasks/utils/utils.cr"
require "colorize"

# describe CnfTestSuite do
# before_all do
# `./cnf-testsuite setup`
# $?.success?.should be_true
# end

# after_all do
# `./cnf-testsuite samples_cleanup`
# $?.success?.should be_true
# end

# it "'testsuite all' should run the configuration lifecycle tests", tags: ["testsuite-config-lifecycle"] do
# `./cnf-testsuite samples_cleanup`
# response_s = `./cnf-testsuite all ~reasonable_startup_time ~reasonable_image_size ~disk_fill ~pod_delete ~pod_io_stress ~pod_network_latency ~pod_network_corruption ~pod_network_duplication ~pod_memory_hog ~chaos_network_loss ~node_drain ~pod_dns_error ~chaos_cpu_hog ~chaos_container_kill ~platform ~volume_hostpath_not_found ~privileged ~increase_capacity ~decrease_capacity ~helm_chart_valid ~helm_chart_published "cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml" verbose`
# LOGGING.info response_s
# (/PASSED: Helm readiness probe found/ =~ response_s).should_not be_nil
# (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil
# (/Final workload score:/ =~ response_s).should_not be_nil
# (/Final score:/ =~ response_s).should_not be_nil
# (CNFManager::Points.all_result_test_names(CNFManager::Points.final_cnf_results_yml).sort).should eq(["ip_addresses", "liveness", "readiness", "rolling_update", "rolling_downgrade", "rolling_version_change", "nodeport_not_used", "hostport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", "rollback", "secrets_used", "immutable_configmap"].sort)
# (/^.*\.cr:[0-9].*/ =~ response_s).should be_nil
# $?.success?.should be_true
# end
# end
describe CnfTestSuite do
end
22 changes: 0 additions & 22 deletions spec/cnf_testsuite_all/cnf_testsuite_container_chaos_spec.cr
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,4 @@ require "../../src/tasks/utils/utils.cr"
require "colorize"

describe "CNF Test Suite all Container Chaos" do
# before_all do
# `./cnf-testsuite setup`
# $?.success?.should be_true
# end

# after_all do
# `./cnf-testsuite samples_cleanup`
# $?.success?.should be_true
# end

# it "'all ~platform ~compatibilty ~state ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_and_scheduling ~microservice ~chaos_network_loss' should run the chaos tests" do
# `./cnf-testsuite samples_cleanup`
# response_s = `./cnf-testsuite all ~platform ~compatibilty ~state ~security ~scalability ~configuration_lifecycle ~observability ~installability ~hardware_and_scheduling ~microservice ~chaos_network_loss cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false verbose`
# LOGGING.info response_s
# (/Final workload score:/ =~ response_s).should_not be_nil
# (/Final score:/ =~ response_s).should_not be_nil
# (CNFManager::Points.all_result_test_names(CNFManager.final_cnf_results_yml)).should eq([ "chaos_cpu_hog", "chaos_container_kill"])
# $?.success?.should be_true
# ensure
# LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false `
# end

end
Loading

0 comments on commit 98d16cb

Please sign in to comment.