From 4b43358ed9ec87a6dcb36a5b9b28ffc368c92635 Mon Sep 17 00:00:00 2001 From: Konstantin Yarovoy Date: Tue, 23 Apr 2024 09:04:41 +0000 Subject: [PATCH] output: Add test name before and after the test execution Ref: cnti-testcatalog#1958 Add information of which test is starting to the console output Add test name to test result console output Signed-off-by: Konstantin Yarovoy --- spec/5g/core_spec.cr | 8 +-- spec/5g/ran_spec.cr | 4 +- spec/platform/hardware_and_scheduler_spec.cr | 2 +- spec/platform/observability_spec.cr | 8 +-- spec/platform/platform_spec.cr | 2 +- spec/platform/resilience_spec.cr | 2 +- spec/platform/security_spec.cr | 8 +-- spec/utils/utils_spec.cr | 6 +- spec/workload/compatibility_spec.cr | 4 +- spec/workload/configuration_spec.cr | 48 +++++++-------- spec/workload/installability_spec.cr | 6 +- spec/workload/microservice_spec.cr | 34 +++++------ spec/workload/observability_spec.cr | 24 ++++---- spec/workload/operator_spec.cr | 4 +- spec/workload/resilience/disk_fill_spec.cr | 2 +- spec/workload/resilience/node_drain_spec.cr | 4 +- spec/workload/resilience/pod_delete_spec.cr | 2 +- .../workload/resilience/pod_dns_error_spec.cr | 4 +- .../workload/resilience/pod_io_stress_spec.cr | 2 +- .../resilience/pod_memory_hog_spec.cr | 2 +- .../resilience/pod_network_corruption_spec.cr | 2 +- .../pod_network_duplication_spec.cr | 2 +- .../resilience/pod_network_latency_spec.cr | 2 +- spec/workload/security_spec.cr | 58 +++++++++---------- spec/workload/state_spec.cr | 14 ++--- src/tasks/utils/task.cr | 1 + src/tasks/utils/utils.cr | 8 +-- 27 files changed, 132 insertions(+), 131 deletions(-) diff --git a/spec/5g/core_spec.cr b/spec/5g/core_spec.cr index 946289a24..c1dbc333d 100644 --- a/spec/5g/core_spec.cr +++ b/spec/5g/core_spec.cr @@ -17,7 +17,7 @@ describe "Core" do $?.success?.should be_true response_s = `./cnf-testsuite smf_upf_heartbeat verbose` Log.info {"response: #{response_s}"} - (/PASSED: Chaos service degradation is less than 50%/ =~ response_s).should_not be_nil + (/(PASSED).*(Chaos service degradation is less than 50%)/ =~ response_s).should_not be_nil ensure Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} $?.success?.should be_true @@ -30,7 +30,7 @@ describe "Core" do $?.success?.should be_true response_s = `./cnf-testsuite smf_upf_heartbeat verbose baseline_count=300` Log.info {"response: #{response_s}"} - (/FAILED: Chaos service degradation is more than 50%/ =~ response_s).should_not be_nil + (/(FAILED).*(Chaos service degradation is more than 50%)/ =~ response_s).should_not be_nil ensure Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} $?.success?.should be_true @@ -43,7 +43,7 @@ describe "Core" do $?.success?.should be_true response_s = `./cnf-testsuite suci_enabled verbose` Log.info {"response: #{response_s}"} - (/PASSED: Core uses SUCI 5g authentication/ =~ response_s).should_not be_nil + (/(PASSED).*(Core uses SUCI 5g authentication)/ =~ response_s).should_not be_nil ensure Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} $?.success?.should be_true @@ -56,7 +56,7 @@ describe "Core" do $?.success?.should be_true response_s = `./cnf-testsuite suci_enabled verbose` Log.info {"response: #{response_s}"} - (/FAILED: Core does not use SUCI 5g authentication/ =~ response_s).should_not be_nil + (/(FAILED).*(Core does not use SUCI 5g authentication)/ =~ response_s).should_not be_nil ensure Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml`} $?.success?.should be_true diff --git a/spec/5g/ran_spec.cr b/spec/5g/ran_spec.cr index 9eb48debe..e2a37ea3d 100644 --- a/spec/5g/ran_spec.cr +++ b/spec/5g/ran_spec.cr @@ -21,7 +21,7 @@ describe "5g" do $?.success?.should be_true response_s = `./cnf-testsuite oran_e2_connection verbose` Log.info {"response: #{response_s}"} - (/PASSED: RAN connects to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil + (/(PASSED).*(RAN connects to a RIC using the e2 standard interface)/ =~ response_s).should_not be_nil ensure Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml`} $?.success?.should be_true @@ -38,7 +38,7 @@ describe "5g" do $?.success?.should be_true response_s = `./cnf-testsuite oran_e2_connection verbose` Log.info {"response: #{response_s}"} - (/FAILED: RAN does not connect to a RIC using the e2 standard interface/ =~ response_s).should_not be_nil + (/(FAILED).*(RAN does not connect to a RIC using the e2 standard interface)/ =~ response_s).should_not be_nil ensure Log.info {`./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml`} $?.success?.should be_true diff --git a/spec/platform/hardware_and_scheduler_spec.cr b/spec/platform/hardware_and_scheduler_spec.cr index a565ebf4c..ef54763cb 100644 --- a/spec/platform/hardware_and_scheduler_spec.cr +++ b/spec/platform/hardware_and_scheduler_spec.cr @@ -15,7 +15,7 @@ describe "Platform" do it "'oci_compliant' should pass if all runtimes are oci_compliant", tags: ["platform:oci_compliant"] do response_s = `./cnf-testsuite platform:oci_compliant` LOGGING.info response_s - (/(PASSED){1}.*(which are OCI compliant runtimes){1}/ =~ response_s).should_not be_nil + (/(PASSED).*(which are OCI compliant runtimes)/ =~ response_s).should_not be_nil end end diff --git a/spec/platform/observability_spec.cr b/spec/platform/observability_spec.cr index 23dca1766..bf8612e0b 100644 --- a/spec/platform/observability_spec.cr +++ b/spec/platform/observability_spec.cr @@ -21,7 +21,7 @@ describe "Platform Observability" do response_s = `./cnf-testsuite platform:kube_state_metrics poc` LOGGING.info response_s - (/(PASSED){1}.*(Your platform is using the){1}.*(release for kube state metrics){1}/ =~ response_s).should_not be_nil + (/(PASSED).*(Your platform is using the).*(release for kube state metrics)/ =~ response_s).should_not be_nil ensure resp = `#{helm} delete kube-state-metrics` LOGGING.info resp @@ -47,7 +47,7 @@ describe "Platform Observability" do response_s = `./cnf-testsuite platform:node_exporter poc` LOGGING.info response_s if check_containerd - (/(PASSED){1}.*(Your platform is using the node exporter){1}/ =~ response_s).should_not be_nil + (/(PASSED).*(Your platform is using the node exporter)/ =~ response_s).should_not be_nil else (/skipping node_exporter: This test only supports the Containerd Runtime./ =~ response_s).should_not be_nil end @@ -71,7 +71,7 @@ describe "Platform Observability" do response_s = `./cnf-testsuite platform:prometheus_adapter poc` Log.info { response_s } - (/(PASSED){1}.*(Your platform is using the prometheus adapter){1}/ =~ response_s).should_not be_nil + (/(PASSED).*(Your platform is using the prometheus adapter)/ =~ response_s).should_not be_nil ensure resp = Helm.uninstall("prometheus-adapter") end @@ -90,7 +90,7 @@ describe "Platform Observability" do KubectlClient::Get.wait_for_install(deployment_name: "metrics-server") response_s = `./cnf-testsuite platform:metrics_server poc` LOGGING.info response_s - (/(PASSED){1}.*(Your platform is using the metrics server){1}/ =~ response_s).should_not be_nil + (/(PASSED).*(Your platform is using the metrics server)/ =~ response_s).should_not be_nil ensure resp = Helm.uninstall("metrics-server") LOGGING.info resp diff --git a/spec/platform/platform_spec.cr b/spec/platform/platform_spec.cr index c7762550f..ac792e59c 100644 --- a/spec/platform/platform_spec.cr +++ b/spec/platform/platform_spec.cr @@ -36,7 +36,7 @@ describe "Platform" do it "'k8s_conformance' should pass if the sonobuoy tests pass", tags: ["platform"] do response_s = `./cnf-testsuite k8s_conformance` LOGGING.info response_s - (/PASSED: K8s conformance test has no failures/ =~ response_s).should_not be_nil + (/(PASSED).*(K8s conformance test has no failures)/ =~ response_s).should_not be_nil end it "individual tasks like 'platform:control_plane_hardening' should not require an installed cnf to run", tags: ["platform"] do diff --git a/spec/platform/resilience_spec.cr b/spec/platform/resilience_spec.cr index 71f18a8d8..db3baaca1 100644 --- a/spec/platform/resilience_spec.cr +++ b/spec/platform/resilience_spec.cr @@ -12,7 +12,7 @@ describe "Platform" do puts "Tests running in destructive mode".colorize(:red) response_s = `./cnf-testsuite platform:worker_reboot_recovery destructive` LOGGING.info response_s - (/(PASSED: Node came back online)/ =~ response_s).should_not be_nil + (/(PASSED).*(Node came back online)/ =~ response_s).should_not be_nil else response_s = `./cnf-testsuite platform:worker_reboot_recovery` LOGGING.info response_s diff --git a/spec/platform/security_spec.cr b/spec/platform/security_spec.cr index 13abb89cf..0e03aca02 100644 --- a/spec/platform/security_spec.cr +++ b/spec/platform/security_spec.cr @@ -11,7 +11,7 @@ describe "Platform" do it "'control_plane_hardening' should pass if the control plane has been hardened", tags: ["platform:security"] do response_s = `./cnf-testsuite platform:control_plane_hardening` Log.info { response_s } - (/(PASSED: Insecure port of Kubernetes API server is not enabled)/ =~ response_s).should_not be_nil + (/(PASSED).*(Insecure port of Kubernetes API server is not enabled)/ =~ response_s).should_not be_nil end it "'cluster_admin' should fail on a cnf that uses a cluster admin binding", tags: ["platform:security"] do @@ -21,7 +21,7 @@ describe "Platform" do response_s = `./cnf-testsuite platform:cluster_admin` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Users with cluster-admin RBAC permissions found/ =~ response_s).should_not be_nil + (/(FAILED).*(Users with cluster-admin RBAC permissions found)/ =~ response_s).should_not be_nil # ensure # `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` end @@ -32,7 +32,7 @@ describe "Platform" do KubectlClient::Get.resource_wait_for_install("pod", "tiller") response_s = `./cnf-testsuite platform:helm_tiller` $?.success?.should be_true - (/FAILED: Containers with the Helm Tiller image are running/ =~ response_s).should_not be_nil + (/(FAILED).*(Containers with the Helm Tiller image are running)/ =~ response_s).should_not be_nil ensure KubectlClient::Delete.command("pod/tiller") KubectlClient::Get.resource_wait_for_uninstall("pod", "tiller") @@ -42,6 +42,6 @@ describe "Platform" do # By default we have nothing to setup for this task to pass since Helm v3 does not use Tiller. response_s = `./cnf-testsuite platform:helm_tiller` $?.success?.should be_true - (/PASSED: No Helm Tiller containers are running/ =~ response_s).should_not be_nil + (/(PASSED).*(No Helm Tiller containers are running)/ =~ response_s).should_not be_nil end end diff --git a/spec/utils/utils_spec.cr b/spec/utils/utils_spec.cr index 1dde2ea97..97427b05d 100644 --- a/spec/utils/utils_spec.cr +++ b/spec/utils/utils_spec.cr @@ -196,9 +196,9 @@ describe "Utils" do Log.info {response_s} response_s = `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml` Log.info {response_s} - resp = `./cnf-testsuite privileged` - Log.info { resp } - (resp).includes?("FAILED: Found 1 privileged containers").should be_true + response_s = `./cnf-testsuite privileged` + Log.info { response_s} + (/(FAILED).*(Found 1 privileged containers)/ =~ response_s).should_not be_nil ensure response_s = `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-generic-cnf/cnf-testsuite.yml` Log.info { response_s } diff --git a/spec/workload/compatibility_spec.cr b/spec/workload/compatibility_spec.cr index 9fda346db..9a9bd0ee8 100644 --- a/spec/workload/compatibility_spec.cr +++ b/spec/workload/compatibility_spec.cr @@ -26,7 +26,7 @@ describe "Compatibility" do retries = retries + 1 end Log.info {"Status: #{response_s}"} - (/PASSED: CNF compatible with both Calico and Cilium/ =~ response_s).should_not be_nil + (/(PASSED).*(CNF compatible with both Calico and Cilium)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true @@ -40,7 +40,7 @@ describe "Compatibility" do response_s = `./cnf-testsuite increase_decrease_capacity verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Replicas increased to/ =~ response_s).should_not be_nil + (/(PASSED).*(Replicas increased to)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` end diff --git a/spec/workload/configuration_spec.cr b/spec/workload/configuration_spec.cr index e1db4b210..e67122848 100644 --- a/spec/workload/configuration_spec.cr +++ b/spec/workload/configuration_spec.cr @@ -55,7 +55,7 @@ describe CnfTestSuite do response_s = `LOG_LEVEL=debug ./cnf-testsuite liveness verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Helm liveness probe/ =~ response_s).should_not be_nil + (/(PASSED).*(Helm liveness probe)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ` end @@ -68,7 +68,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite liveness verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: No livenessProbe found/ =~ response_s).should_not be_nil + (/(FAILED).*(No livenessProbe found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite sample_coredns_bad_liveness_cleanup` end @@ -81,7 +81,7 @@ describe CnfTestSuite do response_s = `LOG_LEVEL=debug ./cnf-testsuite readiness verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Helm readiness probe/ =~ response_s).should_not be_nil + (/(PASSED).*(Helm readiness probe)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ` end @@ -94,7 +94,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite readiness verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: No readinessProbe found/ =~ response_s).should_not be_nil + (/(FAILED).*(No readinessProbe found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite sample_coredns_bad_liveness_cleanup` end @@ -208,7 +208,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite nodeport_not_used verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: NodePort is being used/ =~ response_s).should_not be_nil + (/(FAILED).*(NodePort is being used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false` end @@ -221,7 +221,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite nodeport_not_used verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: NodePort is not used/ =~ response_s).should_not be_nil + (/(PASSED).*(NodePort is not used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` end @@ -234,7 +234,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite hostport_not_used verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: HostPort is being used/ =~ response_s).should_not be_nil + (/(FAILED).*(HostPort is being used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false` end @@ -247,7 +247,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite hostport_not_used verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: HostPort is not used/ =~ response_s).should_not be_nil + (/(PASSED).*(HostPort is not used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` end @@ -260,7 +260,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite ip_addresses verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No IP addresses found/ =~ response_s).should_not be_nil + (/(PASSED).*(No IP addresses found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite sample_coredns_source_cleanup verbose` end @@ -273,7 +273,7 @@ describe CnfTestSuite do response_s = `LOG_LEVEL=info ./cnf-testsuite hardcoded_ip_addresses_in_k8s_runtime_configuration verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Hard-coded IP addresses found in the runtime K8s configuration/ =~ response_s).should_not be_nil + (/(FAILED).*(Hard-coded IP addresses found in the runtime K8s configuration)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false` end @@ -286,7 +286,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite hardcoded_ip_addresses_in_k8s_runtime_configuration verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No hard-coded IP addresses found in the runtime K8s configuration/ =~ response_s).should_not be_nil + (/(PASSED).*(No hard-coded IP addresses found in the runtime K8s configuration)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` end @@ -299,7 +299,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite secrets_used verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Secrets defined and used/ =~ response_s).should_not be_nil + (/(PASSED).*(Secrets defined and used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_secret_volume verbose` end @@ -312,7 +312,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite secrets_used verbose` LOGGING.info response_s $?.success?.should be_true - (/SKIPPED: Secrets not used/ =~ response_s).should_not be_nil + (/(SKIPPED).*(Secrets not used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_unmounted_secret_volume verbose` end @@ -325,7 +325,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite secrets_used verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Secrets defined and used/ =~ response_s).should_not be_nil + (/(PASSED).*(Secrets defined and used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose` end @@ -338,7 +338,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite secrets_used verbose` LOGGING.info response_s $?.success?.should be_true - (/SKIPPED: Secrets not used/ =~ response_s).should_not be_nil + (/(SKIPPED).*(Secrets not used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose` end @@ -351,7 +351,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite secrets_used verbose` LOGGING.info response_s $?.success?.should be_true - (/SKIPPED: Secrets not used/ =~ response_s).should_not be_nil + (/(SKIPPED).*(Secrets not used)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns verbose` end @@ -364,7 +364,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite immutable_configmap verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found mutable configmap/ =~ response_s).should_not be_nil + (/(FAILED).*(Found mutable configmap)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false` end @@ -377,7 +377,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite immutable_configmap verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: All volume or container mounted configmaps immutable/ =~ response_s).should_not be_nil + (/(PASSED).*(All volume or container mounted configmaps immutable)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false` end @@ -390,7 +390,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite require_labels verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Pods should have the app.kubernetes.io\/name label/ =~ response_s).should_not be_nil + (/(FAILED).*(Pods should have the app.kubernetes.io\/name label)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml` end @@ -403,7 +403,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite require_labels verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Pods have the app.kubernetes.io\/name label/ =~ response_s).should_not be_nil + (/(PASSED).*(Pods have the app.kubernetes.io\/name label)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` end @@ -416,7 +416,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite default_namespace verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Resources are created in the default namespace/ =~ response_s).should_not be_nil + (/(FAILED).*(Resources are created in the default namespace)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns` KubectlClient::Utils.wait_for_terminations() @@ -430,7 +430,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite default_namespace verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: default namespace is not being used/ =~ response_s).should_not be_nil + (/(PASSED).*(default namespace is not being used)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag` KubectlClient::Utils.wait_for_terminations() @@ -444,7 +444,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite latest_tag verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Container images are using the latest tag/ =~ response_s).should_not be_nil + (/(FAILED).*(Container images are using the latest tag)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag` end @@ -457,7 +457,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite latest_tag verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Container images are not using the latest tag/ =~ response_s).should_not be_nil + (/(PASSED).*(Container images are not using the latest tag)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` end diff --git a/spec/workload/installability_spec.cr b/spec/workload/installability_spec.cr index 0671a8d03..715df2a49 100644 --- a/spec/workload/installability_spec.cr +++ b/spec/workload/installability_spec.cr @@ -13,7 +13,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite helm_deploy verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Helm deploy failed/ =~ response_s).should_not be_nil + (/(FAILED).*(Helm deploy failed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-bad-helm-deploy-repo verbose` end @@ -58,7 +58,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite helm_chart_published` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Published Helm Chart Found/ =~ response_s).should_not be_nil + (/(PASSED).*(Published Helm Chart Found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-coredns-cnf` end @@ -75,7 +75,7 @@ describe CnfTestSuite do response_s = `./cnf-testsuite helm_chart_published verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Published Helm Chart Not Found/ =~ response_s).should_not be_nil + (/(FAILED).*(Published Helm Chart Not Found)/ =~ response_s).should_not be_nil ensure `#{Helm::BinarySingleton.helm} repo remove badrepo` `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-bad-helm-repo` diff --git a/spec/workload/microservice_spec.cr b/spec/workload/microservice_spec.cr index e7cbb59e4..bd47a0ff6 100644 --- a/spec/workload/microservice_spec.cr +++ b/spec/workload/microservice_spec.cr @@ -23,7 +23,7 @@ describe "Microservice" do response_s = `./cnf-testsuite shared_database` LOGGING.info response_s $?.success?.should be_true - (/N\/A: \[shared_database\] No MariaDB containers were found/ =~ response_s).should_not be_nil + (/(N\/A).*(No MariaDB containers were found)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml` $?.success?.should be_true @@ -36,7 +36,7 @@ describe "Microservice" do response_s = `./cnf-testsuite shared_database` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No shared database found/ =~ response_s).should_not be_nil + (/(PASSED).*(No shared database found)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml` $?.success?.should be_true @@ -49,7 +49,7 @@ describe "Microservice" do response_s = `./cnf-testsuite shared_database` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No shared database found/ =~ response_s).should_not be_nil + (/(PASSED).*(No shared database found)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml` $?.success?.should be_true @@ -62,7 +62,7 @@ describe "Microservice" do response_s = `./cnf-testsuite shared_database` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found a shared database/ =~ response_s).should_not be_nil + (/(FAILED).*(Found a shared database)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml` $?.success?.should be_true @@ -83,7 +83,7 @@ describe "Microservice" do response_s = `./cnf-testsuite shared_database` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No shared database found/ =~ response_s).should_not be_nil + (/(PASSED).*(No shared database found)/ =~ response_s).should_not be_nil ensure Helm.delete("multi-db") KubectlClient::Delete.command("pvc data-multi-db-mariadb-0") @@ -98,7 +98,7 @@ describe "Microservice" do response_s = `./cnf-testsuite single_process_type verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Only one process type used/ =~ response_s).should_not be_nil + (/(PASSED).*(Only one process type used)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` $?.success?.should be_true @@ -111,7 +111,7 @@ describe "Microservice" do response_s = `./cnf-testsuite single_process_type verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: More than one process type used/ =~ response_s).should_not be_nil + (/(FAILED).*(More than one process type used)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/k8s-multiple-processes` $?.success?.should be_true @@ -124,7 +124,7 @@ describe "Microservice" do response_s = `./cnf-testsuite single_process_type verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: More than one process type used/ =~ response_s).should_not be_nil + (/(FAILED).*(More than one process type used)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample-multiple-processes` $?.success?.should be_true @@ -137,7 +137,7 @@ describe "Microservice" do response_s = `./cnf-testsuite reasonable_startup_time verbose` Log.info { response_s } - (/PASSED: CNF had a reasonable startup time/ =~ response_s).should_not be_nil + (/(PASSED).*(CNF had a reasonable startup time)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` } $?.success?.should be_true @@ -150,7 +150,7 @@ describe "Microservice" do response_s = `./cnf-testsuite reasonable_startup_time verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: CNF had a startup time of/ =~ response_s).should_not be_nil + (/(FAILED).*(CNF had a startup time of)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true` $?.success?.should be_true @@ -210,7 +210,7 @@ describe "Microservice" do response_s = `./cnf-testsuite service_discovery verbose` Log.info { response_s } $?.success?.should be_true - (/PASSED: Some containers exposed as a service/ =~ response_s).should_not be_nil + (/(PASSED).*(Some containers exposed as a service)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` } $?.success?.should be_true @@ -223,7 +223,7 @@ describe "Microservice" do response_s = `./cnf-testsuite service_discovery verbose` Log.info { response_s } $?.success?.should be_true - (/FAILED: No containers exposed as a service/ =~ response_s).should_not be_nil + (/(FAILED).*(No containers exposed as a service)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged` } $?.success?.should be_true @@ -241,7 +241,7 @@ describe "Microservice" do response_s = `./cnf-testsuite sig_term_handled verbose` Log.info { response_s } $?.success?.should be_true - (/PASSED: Sig Term handled/ =~ response_s).should_not be_nil + (/(PASSED).*(Sig Term handled)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling/` } $?.success?.should be_true @@ -259,7 +259,7 @@ describe "Microservice" do response_s = `./cnf-testsuite sig_term_handled verbose` Log.info { response_s } $?.success?.should be_true - (/FAILED: Sig Term not handled/ =~ response_s).should_not be_nil + (/(FAILED).*(Sig Term not handled)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_bad_signal_handling/` } $?.success?.should be_true @@ -281,7 +281,7 @@ describe "Microservice" do response_s = `./cnf-testsuite sig_term_handled verbose` Log.info { response_s } $?.success?.should be_true - (/PASSED: Sig Term handled/ =~ response_s).should_not be_nil + (/(PASSED).*(Sig Term handled)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling_tini/` } $?.success?.should be_true @@ -295,7 +295,7 @@ describe "Microservice" do response_s = `./cnf-testsuite zombie_handled verbose` Log.info { response_s } $?.success?.should be_true - (/PASSED: Zombie handled/ =~ response_s).should_not be_nil + (/(PASSED).*(Zombie handled)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample_good_zombie_handling/` } $?.success?.should be_true @@ -308,7 +308,7 @@ describe "Microservice" do response_s = `./cnf-testsuite zombie_handled verbose` Log.info { response_s } $?.success?.should be_true - (/FAILED: Zombie not handled/ =~ response_s).should_not be_nil + (/(FAILED).*(Zombie not handled)/ =~ response_s).should_not be_nil ensure Log.info { `./cnf-testsuite cnf_cleanup cnf-path=./sample-cnfs/sample-bad-zombie/` } $?.success?.should be_true diff --git a/spec/workload/observability_spec.cr b/spec/workload/observability_spec.cr index ba105c3f5..3aaf49ce3 100644 --- a/spec/workload/observability_spec.cr +++ b/spec/workload/observability_spec.cr @@ -16,7 +16,7 @@ describe "Observability" do response_s = `./cnf-testsuite log_output verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Resources output logs to stdout and stderr/ =~ response_s).should_not be_nil + (/(PASSED).*(Resources output logs to stdout and stderr)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` end @@ -28,7 +28,7 @@ describe "Observability" do response_s = `./cnf-testsuite log_output verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Resources do not output logs to stdout and stderr/ =~ response_s).should_not be_nil + (/(FAILED).*(Resources do not output logs to stdout and stderr)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml` end @@ -49,7 +49,7 @@ describe "Observability" do ShellCmd.run("kubectl describe deployment prometheus-server", "k8s_describe_prometheus", force_output: true) test_result = ShellCmd.run("./cnf-testsuite prometheus_traffic", "run_test_cmd", force_output: true) - (/PASSED: Your cnf is sending prometheus traffic/ =~ test_result[:output]).should_not be_nil + (/(PASSED).*(Your cnf is sending prometheus traffic)/ =~ test_result[:output]).should_not be_nil ensure ShellCmd.run("./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml", "spec_sample_cleaup") result = ShellCmd.run("#{helm} delete prometheus", "helm_delete_prometheus") @@ -65,7 +65,7 @@ describe "Observability" do response_s = `./cnf-testsuite prometheus_traffic` LOGGING.info response_s - (/SKIPPED: Prometheus server not found/ =~ response_s).should_not be_nil + (/(SKIPPED).*(Prometheus server not found)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` end @@ -85,7 +85,7 @@ describe "Observability" do response_s = `./cnf-testsuite prometheus_traffic` LOGGING.info response_s - (/FAILED: Your cnf is not sending prometheus traffic/ =~ response_s).should_not be_nil + (/(FAILED).*(Your cnf is not sending prometheus traffic)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` resp = `#{helm} delete prometheus` @@ -107,7 +107,7 @@ describe "Observability" do response_s = `./cnf-testsuite open_metrics` LOGGING.info response_s - (/FAILED: Your cnf's metrics traffic is not OpenMetrics compatible/ =~ response_s).should_not be_nil + (/(FAILED).*(Your cnf's metrics traffic is not OpenMetrics compatible)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml` resp = `#{helm} delete prometheus` @@ -129,7 +129,7 @@ describe "Observability" do response_s = `./cnf-testsuite open_metrics` LOGGING.info response_s - (/PASSED: Your cnf's metrics traffic is OpenMetrics compatible/ =~ response_s).should_not be_nil + (/(PASSED).*(Your cnf's metrics traffic is OpenMetrics compatible)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml` resp = `#{helm} delete prometheus` @@ -158,7 +158,7 @@ describe "Observability" do LOGGING.info resp response_s = `./cnf-testsuite routed_logs` LOGGING.info response_s - (/PASSED: Your cnf's logs are being captured/ =~ response_s).should_not be_nil + (/(PASSED).*(Your cnf's logs are being captured)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` resp = `./cnf-testsuite uninstall_fluentdbitnami` @@ -171,7 +171,7 @@ describe "Observability" do FluentBit.install response_s = `./cnf-testsuite routed_logs` LOGGING.info response_s - (/PASSED: Your cnf's logs are being captured/ =~ response_s).should_not be_nil + (/(PASSED).*(Your cnf's logs are being captured)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-fluentbit` FluentBit.uninstall @@ -190,7 +190,7 @@ describe "Observability" do response_s = `./cnf-testsuite routed_logs` LOGGING.info response_s - (/FAILED: Your cnf's logs are not being captured/ =~ response_s).should_not be_nil + (/(FAILED).*(Your cnf's logs are not being captured)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` resp = `./cnf-testsuite uninstall_fluentd` @@ -205,7 +205,7 @@ describe "Observability" do LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` response_s = `./cnf-testsuite tracing` LOGGING.info response_s - (/FAILED: Tracing not used/ =~ response_s).should_not be_nil + (/(FAILED).*(Tracing not used)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` JaegerManager.uninstall @@ -222,7 +222,7 @@ describe "Observability" do LOGGING.info `./cnf-testsuite cnf_setup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml` response_s = `./cnf-testsuite tracing` LOGGING.info response_s - (/PASSED: Tracing used/ =~ response_s).should_not be_nil + (/(PASSED).*(Tracing used)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml` JaegerManager.uninstall diff --git a/spec/workload/operator_spec.cr b/spec/workload/operator_spec.cr index 3ff702e32..2b51a5882 100644 --- a/spec/workload/operator_spec.cr +++ b/spec/workload/operator_spec.cr @@ -27,7 +27,7 @@ describe "Operator" do $?.success?.should be_true resp = `./cnf-testsuite -l info operator_installed` Log.info { "#{resp}" } - (/PASSED: Operator is installed/ =~ resp).should_not be_nil + (/(PASSED).*(Operator is installed)/ =~ resp).should_not be_nil ensure LOGGING.info `./cnf-testsuite -l info cnf_cleanup cnf-path=./sample-cnfs/sample_operator` $?.success?.should be_true @@ -85,7 +85,7 @@ describe "Operator" do $?.success?.should be_true resp = `./cnf-testsuite -l info operator_installed` Log.info { "#{resp}" } - (/N\/A: No Operators Found/ =~ resp).should_not be_nil + (/(N\/A).*(No Operators Found)/ =~ resp).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-path=sample-cnfs/sample_coredns` $?.success?.should be_true diff --git a/spec/workload/resilience/disk_fill_spec.cr b/spec/workload/resilience/disk_fill_spec.cr index 6096fcf2e..2a74fae92 100644 --- a/spec/workload/resilience/disk_fill_spec.cr +++ b/spec/workload/resilience/disk_fill_spec.cr @@ -19,7 +19,7 @@ describe "Resilience Disk Fill Chaos" do response_s = `./cnf-testsuite disk_fill verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: disk_fill chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(disk_fill chaos test passed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml wait_count=0` $?.success?.should be_true diff --git a/spec/workload/resilience/node_drain_spec.cr b/spec/workload/resilience/node_drain_spec.cr index 24b694399..01d556429 100644 --- a/spec/workload/resilience/node_drain_spec.cr +++ b/spec/workload/resilience/node_drain_spec.cr @@ -20,9 +20,9 @@ describe "Resilience Node Drain Chaos" do LOGGING.info response_s $?.success?.should be_true if KubectlClient::Get.schedulable_nodes_list.size > 1 - (/PASSED: node_drain chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(node_drain chaos test passed)/ =~ response_s).should_not be_nil else - (/SKIPPED: node_drain chaos test requires the cluster to have atleast two/ =~ response_s).should_not be_nil + (/(SKIPPED).*(node_drain chaos test requires the cluster to have atleast two)/ =~ response_s).should_not be_nil end ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` diff --git a/spec/workload/resilience/pod_delete_spec.cr b/spec/workload/resilience/pod_delete_spec.cr index b14738664..8c6acc8a6 100644 --- a/spec/workload/resilience/pod_delete_spec.cr +++ b/spec/workload/resilience/pod_delete_spec.cr @@ -19,7 +19,7 @@ describe "Resilience pod delete Chaos" do response_s = `./cnf-testsuite pod_delete verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: pod_delete chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(pod_delete chaos test passed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true diff --git a/spec/workload/resilience/pod_dns_error_spec.cr b/spec/workload/resilience/pod_dns_error_spec.cr index a3054b152..88134d533 100644 --- a/spec/workload/resilience/pod_dns_error_spec.cr +++ b/spec/workload/resilience/pod_dns_error_spec.cr @@ -19,8 +19,8 @@ describe "Resilience pod dns error Chaos" do response_s = `./cnf-testsuite pod_dns_error verbose` Log.info { response_s } $?.success?.should be_true - ((/SKIPPED: pod_dns_error docker runtime not found/) =~ response_s || - (/PASSED: pod_dns_error chaos test passed/ =~ response_s)).should_not be_nil + ((/(SKIPPED).*(pod_dns_error docker runtime not found)/) =~ response_s || + (/(PASSED).*(pod_dns_error chaos test passed)/ =~ response_s)).should_not be_nil rescue ex # Raise back error to ensure test fails. # The ensure block will cleanup the CNF and the litmus installation. diff --git a/spec/workload/resilience/pod_io_stress_spec.cr b/spec/workload/resilience/pod_io_stress_spec.cr index da057d46e..11b7cfcde 100644 --- a/spec/workload/resilience/pod_io_stress_spec.cr +++ b/spec/workload/resilience/pod_io_stress_spec.cr @@ -19,7 +19,7 @@ describe "Resilience pod delete Chaos" do response_s = `./cnf-testsuite pod_io_stress verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: pod_io_stress chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(pod_io_stress chaos test passed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true diff --git a/spec/workload/resilience/pod_memory_hog_spec.cr b/spec/workload/resilience/pod_memory_hog_spec.cr index ee66e1305..3b1f32192 100644 --- a/spec/workload/resilience/pod_memory_hog_spec.cr +++ b/spec/workload/resilience/pod_memory_hog_spec.cr @@ -20,7 +20,7 @@ describe "Resilience pod memory hog Chaos" do response_s = `./cnf-testsuite pod_memory_hog verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: pod_memory_hog chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(pod_memory_hog chaos test passed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true diff --git a/spec/workload/resilience/pod_network_corruption_spec.cr b/spec/workload/resilience/pod_network_corruption_spec.cr index 178f6acd8..11c22ab23 100644 --- a/spec/workload/resilience/pod_network_corruption_spec.cr +++ b/spec/workload/resilience/pod_network_corruption_spec.cr @@ -19,7 +19,7 @@ describe "Resilience Pod Network corruption Chaos" do response_s = `./cnf-testsuite pod_network_corruption verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: pod_network_corruption chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(pod_network_corruption chaos test passed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true diff --git a/spec/workload/resilience/pod_network_duplication_spec.cr b/spec/workload/resilience/pod_network_duplication_spec.cr index af2af0a89..c6feed97f 100644 --- a/spec/workload/resilience/pod_network_duplication_spec.cr +++ b/spec/workload/resilience/pod_network_duplication_spec.cr @@ -19,7 +19,7 @@ describe "Resilience Pod Network duplication Chaos" do response_s = `./cnf-testsuite pod_network_duplication verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: pod_network_duplication chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(pod_network_duplication chaos test passed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true diff --git a/spec/workload/resilience/pod_network_latency_spec.cr b/spec/workload/resilience/pod_network_latency_spec.cr index ebe32b066..f75c3b4c5 100644 --- a/spec/workload/resilience/pod_network_latency_spec.cr +++ b/spec/workload/resilience/pod_network_latency_spec.cr @@ -19,7 +19,7 @@ describe "Resilience Pod Network Latency Chaos" do response_s = `./cnf-testsuite pod_network_latency verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: pod_network_latency chaos test passed/ =~ response_s).should_not be_nil + (/(PASSED).*(pod_network_latency chaos test passed)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true diff --git a/spec/workload/security_spec.cr b/spec/workload/security_spec.cr index ca4d2907f..b9a4559c2 100644 --- a/spec/workload/security_spec.cr +++ b/spec/workload/security_spec.cr @@ -47,7 +47,7 @@ describe "Security" do response_s = `./cnf-testsuite privilege_escalation` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No containers that allow privilege escalation were found/ =~ response_s).should be_nil + (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` end @@ -60,7 +60,7 @@ describe "Security" do response_s = `./cnf-testsuite privilege_escalation` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No containers that allow privilege escalation were found/ =~ response_s).should_not be_nil + (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml` end @@ -73,7 +73,7 @@ describe "Security" do response_s = `./cnf-testsuite symlink_file_system` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No containers allow a symlink attack/ =~ response_s).should_not be_nil + (/(PASSED).*(No containers allow a symlink attack)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` end @@ -86,7 +86,7 @@ describe "Security" do response_s = `./cnf-testsuite insecure_capabilities` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Containers with insecure capabilities were not found/ =~ response_s).should_not be_nil + (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` end @@ -99,7 +99,7 @@ describe "Security" do response_s = `./cnf-testsuite insecure_capabilities` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Containers with insecure capabilities were not found/ =~ response_s).should be_nil + (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml` end @@ -112,7 +112,7 @@ describe "Security" do response_s = `./cnf-testsuite linux_hardening` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Security services are being used to harden applications/ =~ response_s).should be_nil + (/(PASSED).*(Security services are being used to harden applications)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -125,7 +125,7 @@ describe "Security" do response_s = `./cnf-testsuite application_credentials` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found applications credentials in configuration files/ =~ response_s).should_not be_nil + (/(FAILED).*(Found applications credentials in configuration files)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml` end @@ -138,7 +138,7 @@ describe "Security" do response_s = `./cnf-testsuite host_network` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No host network attached to pod/ =~ response_s).should_not be_nil + (/(PASSED).*(No host network attached to pod)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml` end @@ -151,7 +151,7 @@ describe "Security" do response_s = `./cnf-testsuite service_account_mapping` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Service accounts automatically mapped/ =~ response_s).should_not be_nil + (/(FAILED).*(Service accounts automatically mapped)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml` end @@ -164,7 +164,7 @@ describe "Security" do response_s = `./cnf-testsuite cpu_limits` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Containers have CPU limits set/ =~ response_s).should_not be_nil + (/(PASSED).*(Containers have CPU limits set)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -177,7 +177,7 @@ describe "Security" do response_s = `./cnf-testsuite memory_limits` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Containers have memory limits set/ =~ response_s).should_not be_nil + (/(PASSED).*(Containers have memory limits set)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -190,7 +190,7 @@ describe "Security" do response_s = `./cnf-testsuite ingress_egress_blocked` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Ingress and Egress traffic blocked on pods/ =~ response_s).should be_nil + (/(PASSED).*(Ingress and Egress traffic blocked on pods)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -203,7 +203,7 @@ describe "Security" do response_s = `./cnf-testsuite host_pid_ipc_privileges` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found containers with hostPID and hostIPC privileges/ =~ response_s).should be_nil + (/(FAILED).*(Found containers with hostPID and hostIPC privileges)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -216,7 +216,7 @@ describe "Security" do response_s = `./cnf-testsuite non_root_containers` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found containers running with root user or user with root group membership/ =~ response_s).should be_nil + (/(FAILED).*(Found containers running with root user or user with root group membership)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot` end @@ -229,7 +229,7 @@ describe "Security" do response_s = `./cnf-testsuite non_root_containers` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Containers are running with non-root user with non-root group membership/ =~ response_s).should be_nil + (/(PASSED).*(Containers are running with non-root user with non-root group membership)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -242,7 +242,7 @@ describe "Security" do response_s = `./cnf-testsuite privileged_containers` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found privileged containers/ =~ response_s).should be_nil + (/(FAILED).*(Found privileged containers)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -255,7 +255,7 @@ describe "Security" do response_s = `./cnf-testsuite immutable_file_systems` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Containers have immutable file systems/ =~ response_s).should be_nil + (/(PASSED).*(Containers have immutable file systems)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` end @@ -268,7 +268,7 @@ describe "Security" do response_s = `./cnf-testsuite immutable_file_systems` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Containers have immutable file systems/ =~ response_s).should_not be_nil + (/(PASSED).*(Containers have immutable file systems)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-immutable-fs` end @@ -282,7 +282,7 @@ describe "Security" do response_s = `./cnf-testsuite hostpath_mounts` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found containers with hostPath mounts/ =~ response_s).should be_nil + (/(FAILED).*(Found containers with hostPath mounts)/ =~ response_s).should be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf` ClusterTools.install @@ -297,7 +297,7 @@ describe "Security" do response_s = `./cnf-testsuite hostpath_mounts` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Found containers with hostPath mounts/ =~ response_s).should_not be_nil + (/(FAILED).*(Found containers with hostPath mounts)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-hostpath` ClusterTools.install @@ -311,7 +311,7 @@ describe "Security" do response_s = `./cnf-testsuite container_sock_mounts verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Container engine daemon sockets are not mounted as volumes/ =~ response_s).should_not be_nil + (/(PASSED).*(Container engine daemon sockets are not mounted as volumes)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` end @@ -324,7 +324,7 @@ describe "Security" do response_s = `./cnf-testsuite container_sock_mounts verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Container engine daemon sockets are mounted as volumes/ =~ response_s).should_not be_nil + (/(FAILED).*(Container engine daemon sockets are mounted as volumes)/ =~ response_s).should_not be_nil (/Unix socket is not allowed/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml` @@ -338,7 +338,7 @@ describe "Security" do response_s = `./cnf-testsuite external_ips verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Services are not using external IPs/ =~ response_s).should_not be_nil + (/(PASSED).*(Services are not using external IPs)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml` end @@ -351,7 +351,7 @@ describe "Security" do response_s = `./cnf-testsuite external_ips verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Services are using external IPs/ =~ response_s).should_not be_nil + (/(FAILED).*(Services are using external IPs)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml` end @@ -364,7 +364,7 @@ describe "Security" do response_s = `./cnf-testsuite selinux_options verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Pods are using custom SELinux options that can be used for privilege escalations/ =~ response_s).should_not be_nil + (/(FAILED).*(Pods are using custom SELinux options that can be used for privilege escalations)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag` end @@ -377,7 +377,7 @@ describe "Security" do response_s = `./cnf-testsuite selinux_options verbose` LOGGING.info response_s $?.success?.should be_true - (/N\/A: Pods are not using SELinux/ =~ response_s).should_not be_nil + (/(N\/A).*(Pods are not using SELinux)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` end @@ -390,7 +390,7 @@ describe "Security" do response_s = `./cnf-testsuite selinux_options verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: Pods are not using custom SELinux options that can be used for privilege escalations/ =~ response_s).should_not be_nil + (/(PASSED).*(Pods are not using custom SELinux options that can be used for privilege escalations)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_valid_selinux_options` end @@ -403,7 +403,7 @@ describe "Security" do response_s = `./cnf-testsuite sysctls verbose` LOGGING.info response_s $?.success?.should be_true - (/FAILED: Restricted values for are being used for sysctls/ =~ response_s).should_not be_nil + (/(FAILED).*(Restricted values for are being used for sysctls)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_sysctls` end @@ -416,7 +416,7 @@ describe "Security" do response_s = `./cnf-testsuite sysctls verbose` LOGGING.info response_s $?.success?.should be_true - (/PASSED: No restricted values found for sysctls/ =~ response_s).should_not be_nil + (/(PASSED).*(No restricted values found for sysctls)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot` end diff --git a/spec/workload/state_spec.cr b/spec/workload/state_spec.cr index 2c619c9ca..c3ae3ae60 100644 --- a/spec/workload/state_spec.cr +++ b/spec/workload/state_spec.cr @@ -18,7 +18,7 @@ describe "State" do $?.success?.should be_true response_s = `./cnf-testsuite -l info elastic_volumes verbose` LOGGING.info "Status: #{response_s}" - (/PASSED: All used volumes are elastic/ =~ response_s).should be_nil + (/(PASSED).*(All used volumes are elastic)/ =~ response_s).should be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml` $?.success?.should be_true @@ -54,7 +54,7 @@ describe "State" do # KubectlClient::Get.resource_wait_for_install("Pod", "mycluster-2") response_s = `LOG_LEVEL=info ./cnf-testsuite database_persistence` Log.info {"Status: #{response_s}"} - (/PASSED: CNF uses database with cloud-native persistence/ =~ response_s).should_not be_nil + (/(PASSED).*(CNF uses database with cloud-native persistence)/ =~ response_s).should_not be_nil ensure # Mysql.uninstall # KubectlClient::Delete.file("https://raw.githubusercontent.com/mysql/mysql-operator/trunk/samples/sample-cluster.yaml --wait=false") @@ -71,7 +71,7 @@ describe "State" do $?.success?.should be_true response_s = `./cnf-testsuite -l info elastic_volumes verbose` LOGGING.info "Status: #{response_s}" - (/FAILED: Some of the used volumes are not elastic/ =~ response_s).should_not be_nil + (/(FAILED).*(Some of the used volumes are not elastic)/ =~ response_s).should_not be_nil ensure LOGGING.info `./cnf-testsuite cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true @@ -84,7 +84,7 @@ describe "State" do $?.success?.should be_true response_s = `./cnf-testsuite volume_hostpath_not_found verbose` LOGGING.info "Status: #{response_s}" - (/PASSED: hostPath volumes not found/ =~ response_s).should_not be_nil + (/(PASSED).*(hostPath volumes not found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml` $?.success?.should be_true @@ -97,7 +97,7 @@ describe "State" do $?.success?.should be_true response_s = `./cnf-testsuite volume_hostpath_not_found verbose` LOGGING.info "Status: #{response_s}" - (/FAILED: hostPath volumes found/ =~ response_s).should_not be_nil + (/(FAILED).*(hostPath volumes found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-fragile-state/cnf-testsuite.yml deploy_with_chart=false` $?.success?.should be_true @@ -113,7 +113,7 @@ describe "State" do $?.success?.should be_true response_s = `./cnf-testsuite no_local_volume_configuration verbose` LOGGING.info "Status: #{response_s}" - (/FAILED: local storage configuration volumes found/ =~ response_s).should_not be_nil + (/(FAILED).*(local storage configuration volumes found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml deploy_with_chart=false` update_yml("sample-cnfs/sample-local-storage/cnf-testsuite.yml", "helm_values", "") @@ -127,7 +127,7 @@ describe "State" do $?.success?.should be_true response_s = `./cnf-testsuite no_local_volume_configuration verbose` LOGGING.info "Status: #{response_s}" - (/PASSED: local storage configuration volumes not found/ =~ response_s).should_not be_nil + (/(PASSED).*(local storage configuration volumes not found)/ =~ response_s).should_not be_nil ensure `./cnf-testsuite cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml deploy_with_chart=false` $?.success?.should be_true diff --git a/src/tasks/utils/task.cr b/src/tasks/utils/task.cr index d1bea1eed..6856cd148 100644 --- a/src/tasks/utils/task.cr +++ b/src/tasks/utils/task.cr @@ -115,6 +115,7 @@ module CNFManager test_name = task.as(Sam::Task).name.as(String) Log.for(test_name).info { "Starting test" } Log.for(test_name).debug { "cnf_config: #{config}" } + puts "🎬 Testing: [#{test_name}]" end ret = yield args, config if ret.is_a?(CNFManager::TestcaseResult) diff --git a/src/tasks/utils/utils.cr b/src/tasks/utils/utils.cr index 73dbe3321..6f2216f6d 100644 --- a/src/tasks/utils/utils.cr +++ b/src/tasks/utils/utils.cr @@ -305,13 +305,13 @@ def upsert_decorated_task(task, status : CNFManager::ResultStatus, message, star cat_emoji = CNFManager::Points.task_emoji_by_task(task) case status.to_basic when CNFManager::ResultStatus::Passed - upsert_passed_task(task, "✔️ #{cat_emoji}PASSED: #{message} #{tc_emoji}", start_time) + upsert_passed_task(task, "✔️ #{cat_emoji}PASSED: [#{task}] #{message} #{tc_emoji}", start_time) when CNFManager::ResultStatus::Failed - upsert_failed_task(task, "✖️ #{cat_emoji}FAILED: #{message} #{tc_emoji}", start_time) + upsert_failed_task(task, "✖️ #{cat_emoji}FAILED: [#{task}] #{message} #{tc_emoji}", start_time) when CNFManager::ResultStatus::Skipped - upsert_skipped_task(task, "⏭️ #{cat_emoji}SKIPPED: #{message} #{tc_emoji}", start_time) + upsert_skipped_task(task, "⏭️ #{cat_emoji}SKIPPED: [#{task}] #{message} #{tc_emoji}", start_time) when CNFManager::ResultStatus::NA - upsert_na_task(task, "⏭️ #{cat_emoji}N/A: #{message} #{tc_emoji}", start_time) + upsert_na_task(task, "⏭️ #{cat_emoji}N/A: [#{task}] #{message} #{tc_emoji}", start_time) end end