From 9739fb95bdcd2e66619af381cc8e2da46742c1cd Mon Sep 17 00:00:00 2001 From: Konstantin Yarovoy Date: Fri, 29 Nov 2024 09:43:32 +0000 Subject: [PATCH] naming: rename setup and cleanup tasks Rename cnf_setup, cnf_cleanup and setup tasks to cnf_install, cnf_uninstall, install_dependencies Change documentation, specs and code accordingly Refs: #2181 Signed-off-by: Konstantin Yarovoy --- .github/workflows/actions.yml | 14 +- CNF_TESTSUITE_YML_USAGE.md | 4 +- INSTALL.md | 20 +-- README.md | 4 +- SOURCE_INSTALL.md | 31 ++--- USAGE.md | 4 +- example-cnfs/coredns/README.md | 6 +- example-cnfs/envoy/README.md | 8 +- example-cnfs/ip-forwarder/README.md | 8 +- example-cnfs/linkerd2/README.md | 8 +- example-cnfs/nsm/README.md | 6 +- example-cnfs/pantheon-nsm-nat/README.md | 8 +- .../pantheon-nsm-nat/nat-cnf/README.md | 6 +- example-cnfs/vpp-3c2n-csp-use-case/README.md | 8 +- spec/5g/core_spec.cr | 18 +-- spec/5g/ran_spec.cr | 2 +- spec/cluster_setup_spec.cr | 4 +- spec/cnf_testsuite_all/cnf_testsuite_spec.cr | 2 +- spec/platform/cluster_api_spec.cr | 2 +- spec/platform/hardware_and_scheduler_spec.cr | 2 +- spec/platform/observability_spec.cr | 2 +- spec/platform/platform_spec.cr | 2 +- spec/platform/resilience_spec.cr | 2 +- spec/platform/security_spec.cr | 2 +- spec/setup_spec.cr | 94 ++++++------- spec/spec_helper.cr | 12 +- spec/utils/cnf_manager_spec.cr | 8 +- spec/utils/k8s_instrumentation_spec.cr | 2 +- spec/utils/kubescape_spec.cr | 4 +- spec/utils/utils_spec.cr | 16 +-- spec/workload/compatibility_spec.cr | 10 +- spec/workload/configuration_spec.cr | 122 ++++++++--------- spec/workload/installability_spec.cr | 24 ++-- spec/workload/microservice_spec.cr | 92 ++++++------- spec/workload/observability_spec.cr | 42 +++--- spec/workload/operator_spec.cr | 8 +- spec/workload/registry_spec.cr | 24 ++-- spec/workload/resilience/disk_fill_spec.cr | 6 +- .../workload/resilience/network_chaos_spec.cr | 2 +- spec/workload/resilience/node_drain_spec.cr | 6 +- spec/workload/resilience/pod_delete_spec.cr | 6 +- .../workload/resilience/pod_dns_error_spec.cr | 8 +- .../workload/resilience/pod_io_stress_spec.cr | 6 +- .../resilience/pod_memory_hog_spec.cr | 6 +- .../resilience/pod_network_corruption_spec.cr | 6 +- .../pod_network_duplication_spec.cr | 6 +- .../resilience/pod_network_latency_spec.cr | 6 +- spec/workload/security_spec.cr | 124 +++++++++--------- spec/workload/state_spec.cr | 24 ++-- src/tasks/cleanup.cr | 14 +- src/tasks/cluster_api_setup.cr | 6 +- src/tasks/cluster_setup.cr | 4 +- src/tasks/cnf_setup.cr | 8 +- src/tasks/helmenv_setup.cr | 4 +- src/tasks/prereqs.cr | 2 +- src/tasks/setup.cr | 4 +- src/tasks/sonobuoy_setup.cr | 4 +- .../utils/cnf_installation/install_common.cr | 8 +- src/tasks/utils/cnf_manager.cr | 2 +- src/tasks/workload/compatibility.cr | 2 +- src/tasks/workload/observability.cr | 2 +- .../Dockerfile | 2 +- .../README.md | 4 +- 63 files changed, 452 insertions(+), 451 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index f1ca204ce..aa381c769 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -187,7 +187,7 @@ jobs: # sleep 1 #done crystal build src/cnf-testsuite.cr - ./cnf-testsuite setup + ./cnf-testsuite install_dependencies LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.spec }} -v @@ -392,9 +392,9 @@ jobs: mkdir /shared/$DIR mv cnf-testsuite /shared/$DIR cd /shared/$DIR - ./cnf-testsuite setup + ./cnf-testsuite install_dependencies wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/testsuite/${GITHUB_SHA}/example-cnfs/coredns/cnf-testsuite.yml - ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml + ./cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml LOG_LEVEL=info ./cnf-testsuite all ~compatibility ~resilience ~reasonable_startup_time ~reasonable_image_size ~platform ~increase_capacity ~decrease_capacity ~install_script_helm ~helm_chart_valid ~helm_chart_published verbose - name: Delete Cluster if: ${{ always() }} @@ -472,9 +472,9 @@ jobs: mkdir /shared/$DIR mv cnf-testsuite /shared/$DIR cd /shared/$DIR - ./cnf-testsuite setup + ./cnf-testsuite install_dependencies wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/testsuite/${GITHUB_SHA}/example-cnfs/coredns/cnf-testsuite.yml - ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml + ./cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml LOG_LEVEL=info ./cnf-testsuite all ~resilience ~compatibility ~pod_network_latency ~platform ~increase_capacity ~decrease_capacity ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~install_script_helm ~helm_chart_valid ~helm_chart_published ~rollback ~secrets_used ~immutable_configmap verbose - name: Delete Cluster if: ${{ always() }} @@ -552,9 +552,9 @@ jobs: mkdir /shared/$DIR mv cnf-testsuite /shared/$DIR cd /shared/$DIR - ./cnf-testsuite setup + ./cnf-testsuite install_dependencies wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/testsuite/${GITHUB_SHA}/example-cnfs/coredns/cnf-testsuite.yml - ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml + ./cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml LOG_LEVEL=info ./cnf-testsuite all ~resilience ~platform ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~rollback ~secrets_used ~immutable_configmap ~reasonable_startup_time ~reasonable_image_size verbose - name: Delete Cluster if: ${{ always() }} diff --git a/CNF_TESTSUITE_YML_USAGE.md b/CNF_TESTSUITE_YML_USAGE.md index e79195fb6..3c0aec525 100644 --- a/CNF_TESTSUITE_YML_USAGE.md +++ b/CNF_TESTSUITE_YML_USAGE.md @@ -1,7 +1,7 @@ # Test Suite Configuration Usage: cnf-testsuite.yml ### What is the cnf-testsuite.yml and why is it required?: -The cnf-testsuite.yml is used by `cnf_setup` in order to install the CNF to be tested onto an existing K8s cluster. +The cnf-testsuite.yml is used by `cnf_install` in order to install the CNF to be tested onto an existing K8s cluster. The information in the cnf-testsuite.yml is also used for additional configuration of some tests e.g. `white_list_container_names` is used for exculding containers from the [privileged_containers](https://github.com/cnti-testcatalog/testsuite/blob/main/src/tasks/workload/security.cr#L138) container test. @@ -122,7 +122,7 @@ Described below: [link](#5G-parameters) Deployments are defined as three arrays, each for different installation method. Each array element represents one deployment, and they are meant to represent a single CNF together. At least one deployment should exist in CNF config for it to be a proper config. -All info for deployments is used in cnf_setup and cnf_cleanup tasks. +All info for deployments is used in cnf_install and cnf_uninstall tasks. ```yaml deployments: diff --git a/INSTALL.md b/INSTALL.md index d72460ec9..4389a9dca 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -147,10 +147,10 @@ export KUBECONFIG=path/to/mycluster.config *Note: We recommend running cnf-testsuite on a non-production cluster.* -The next step is to run the `setup` which prepares the cnf-testsuite. This runs pre-reqs to verify you have everything needed in order to run the suite, simply run the following: +The next step is to run the `install_dependencies` which prepares the cnf-testsuite. This runs pre-reqs to verify you have everything needed in order to run the suite, simply run the following: ``` -cnf-testsuite setup +cnf-testsuite install_dependencies ``` The test suite by default will pull docker images from https://docker.io. You can set your own username and password with local environment variables by doing the following: @@ -177,7 +177,7 @@ source test.sh ### Configuration -Now cnf-testsuite is setup, we're ready to configure it to point at a CNF to test. +Now cnf-testsuite is prepared, we're ready to configure it to point at a CNF to test. #### Using an Example CNF @@ -188,7 +188,7 @@ wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/tes ``` - The wget gets a working config file, now tell cnf-testsuite to use it by doing the following: ``` -cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml +cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml ``` - There are other examples in the [examples-cnfs](https://github.com/cnti-testcatalog/testsuite/tree/master/example-cnfs) directory that can be used for testing as well. @@ -200,14 +200,14 @@ If you've brought your own CNF to test, review the [CNF_TESTSUITE_YML_USAGE.md]( If you've followed the [CNF_TESTSUITE_YML_USAGE.md](CNF_TESTSUITE_YML_USAGE.md) guide and have your cnf-testsuite.yml ready, you can run the same command we ran for the example CNF to set it up: ``` -cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml +cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml ``` ### Running cnf-testsuite for the first time #### Running Tests -If you want to run all tests, do the following (this is assuming your `cnf_setup` ran without errors in the [configuration](#Configuration) steps:) +If you want to run all tests, do the following (this is assuming your `cnf_install` ran without errors in the [configuration](#Configuration) steps:) _For complete usage, see the [USAGE.md](USAGE.md) doc._ ``` @@ -233,12 +233,12 @@ A test log file, eg. `cnf-testsuite-results-20201216.txt`, will be created which For more details on points, see our [POINTS.md](./POINTS.md) documentation. -#### Cleaning Up +#### Uninstallation Run the following to uninstall the CNF (this is assuming you installed the cnf-testsuite.yml in your present working directory): ``` -cnf-testsuite cnf_cleanup +cnf-testsuite cnf_uninstall ``` -You can also run `cleanall` and cnf-testsuite will attempt to cleanup everything. +You can also run `cleanup` and cnf-testsuite will attempt to cleanup everything. -_NOTE: Cleanup does not handle manually deployed CNFs_ +_NOTE: CNF uninstallation does not handle manually deployed CNFs_ diff --git a/README.md b/README.md index fb53c90f3..104fbf64b 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ To get the CNTI Test Catalog up and running, see the [Installation Guide](INSTAL Prereqs: kubernetes cluster, wget, curl, helm 3.1.1 or greater on your system already. 1. Install the latest test suite binary: `source <(curl -s https://raw.githubusercontent.com/cnti-testcatalog/testsuite/main/curl_install.sh)` -2. Run `setup` to prepare the cnf-testsuite: `cnf-testsuite setup` +2. Run `install_dependencies` to prepare the cnf-testsuite: `cnf-testsuite install_dependencies` 3. Pull down an example CNF configuration to try: `curl -o cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/testsuite/main/example-cnfs/coredns/cnf-testsuite.yml` -4. Initialize the test suite for using the CNF: `cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml` +4. Initialize the test suite for using the CNF: `cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml` 5. Run all of application/workload tests: `cnf-testsuite workload` #### More Usage docs diff --git a/SOURCE_INSTALL.md b/SOURCE_INSTALL.md index 1822e131c..1167f540e 100644 --- a/SOURCE_INSTALL.md +++ b/SOURCE_INSTALL.md @@ -8,7 +8,7 @@ This INSTALL guide will detail the minimum requirements needed for cnf-testsuite - [**Pre-Requisites**](#Pre-Requisites) - [**Installation**](#Installation) -- [**Setup**](#Setup) +- [**Preparation**](#Preparation) - [**Configuration**](#Configuration) - [**Running cnf-testsuite for the first time**](#Running-cnf-testsuite-for-the-first-time) @@ -149,23 +149,23 @@ crystal spec

-### Setup +### Preparation -Now that we have a `cnf-testsuite` binary, we can run `setup` to ensure it has all the pre-requisites needed in order to successfully run tests and setup required installed_cnf_files/ directory and other files required for cnf-testsuite. +Now that we have a `cnf-testsuite` binary, we can run `install_dependencies` to ensure it has all the pre-requisites needed in order to successfully run tests and prepare required installed_cnf_files/ directory and other files required for cnf-testsuite. -- Run the following to setup cnf-testsuite: +- Run the following to prepare cnf-testsuite: ``` - ./cnf-testsuite setup + ./cnf-testsuite install_dependencies ``` - If you have crystal installed, you can also run by: ``` - crystal src/cnf-testsuite.cr setup + crystal src/cnf-testsuite.cr install_dependencies ``` This should display output of all the pre-requisites (and install helm if not found on the system you intend to run from). Any missing requirements will need to be satisfied before proceeding or could result in errors, etc. ### Configuration -Now that cnf-testsuite is installed and setup, we can now run CNF workloads and tests. We recommend installing and running a sample CNF to ensure cnf-testsuite is operational and set expectations of the output. +Now that cnf-testsuite is installed and prepared, we can now run CNF workloads and tests. We recommend installing and running a sample CNF to ensure cnf-testsuite is operational and set expectations of the output. #### Configuring an example CNF @@ -178,12 +178,12 @@ To use CoreDNS as an example CNF. Download the testsuite configuration to test C - Prepare the test suite to use the CNF by running: ``` # via built binary - ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml + ./cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml ``` Or ``` # via crystal - crystal src/cnf-testsuite.cr cnf_setup cnf-config=./cnf-testsuite.yml + crystal src/cnf-testsuite.cr cnf_install cnf-config=./cnf-testsuite.yml ``` There are other examples in the [example cnfs](example-cnfs) folder if you would like to test others. @@ -197,7 +197,7 @@ There are other examples in the [example cnfs](example-cnfs) folder if you would #### Running Tests -If you want to run all tests for CoreDNS Example CNF, do the following (this is assuming your `cnf_setup` ran without errors in the [configuration](#Configuring-an-example-CNF) steps:) +If you want to run all tests for CoreDNS Example CNF, do the following (this is assuming your `cnf_install` ran without errors in the [configuration](#Configuring-an-example-CNF) steps:) _For complete usage, see the [USAGE.md](USAGE.md) doc._ ``` @@ -221,7 +221,7 @@ You can also run via `crystal` by replacing the `./cnf-testsuite` with `crystal #### More Example Usage (also see the [complete usage documentation](https://github.com/cnti-testcatalog/testsuite/blob/main/USAGE.md)) ``` -# These assume you've already run the cnf_setup pointing at a cnf-testsuite.yml config above. You can always specify your config at the end of each command as well, eg: +# These assume you've already run the cnf_install pointing at a cnf-testsuite.yml config above. You can always specify your config at the end of each command as well, eg: ./cnf-testsuite all cnf-config=/cnf-testsuite.yml # Runs all ga tests (generally available workload and platform tests) @@ -253,17 +253,18 @@ A test log file, eg. `cnf-testsuite-results-20201216.txt`, will be created which For more details on points, see our [POINTS.md](./POINTS.md) documentation. -#### Cleaning Up +#### Uninstallation Run the following to uninstall the CNF: ``` -./cnf-testsuite cnf_cleanup +./cnf-testsuite cnf_uninstall ``` -You can also run `cleanall` and cnf-testsuite will attempt to cleanup everything. +You can also run `cleanup` and cnf-testsuite will attempt to cleanup everything. + +_NOTE: CNF uninstallation does not handle manually deployed CNFs_ -_NOTE: Cleanup does not handle manually deployed CNFs_ #### NOTE: If the OpenMetrics version changes, the protobuf file will need to be regenerated ``` git clone https://github.com/jeromegn/protobuf.cr diff --git a/USAGE.md b/USAGE.md index 4fe382f8f..552beace0 100644 --- a/USAGE.md +++ b/USAGE.md @@ -62,12 +62,12 @@ crystal build src/cnf-testsuite.cr #### Installing a cnf: ``` -./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml +./cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml ``` ##### Skip waiting for resource readiness during installation: ``` -./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml skip_wait_for_install +./cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml skip_wait_for_install ``` #### Running all of the platform and workload tests: diff --git a/example-cnfs/coredns/README.md b/example-cnfs/coredns/README.md index be15d3c84..b8ea1d80e 100644 --- a/example-cnfs/coredns/README.md +++ b/example-cnfs/coredns/README.md @@ -18,13 +18,13 @@ Follow [Pre-req steps](../../INSTALL.md#pre-requisites), including Initialize the test suite ``` -./cnf-testsuite setup +./cnf-testsuite install_dependencies ``` Configure and deploy CoreDNS as the target CNF ``` -./cnf-testsuite cnf_setup cnf-path=example-cnfs/coredns/cnf-testsuite.yml +./cnf-testsuite cnf_install cnf-path=example-cnfs/coredns/cnf-testsuite.yml ``` Run the all the tests @@ -38,5 +38,5 @@ Check the results file Uninstall the CNF (including undeployment of CoreDNS) ``` -./cnf-testsuite cnf_cleanup +./cnf-testsuite cnf_uninstall ``` diff --git a/example-cnfs/envoy/README.md b/example-cnfs/envoy/README.md index f64fcacfd..47ba03644 100644 --- a/example-cnfs/envoy/README.md +++ b/example-cnfs/envoy/README.md @@ -12,16 +12,16 @@ Set the KUBECONFIG environment to point to the remote K8s cluster ### Automated Envoy installation -Run cnf-testsuite setup +Run cnf-testsuite install_dependencies ``` -./cnf-testsuite setup +./cnf-testsuite install_dependencies ``` Install Envoy ``` -./cnf-testsuite cnf_setup cnf-config=example-cnfs/envoy/cnf-testsuite.yml +./cnf-testsuite cnf_install cnf-config=example-cnfs/envoy/cnf-testsuite.yml ``` Run the test suite: @@ -33,5 +33,5 @@ Run the test suite: Envoy uninstallation ``` -./cnf-testsuite cnf_cleanup +./cnf-testsuite cnf_uninstall ``` \ No newline at end of file diff --git a/example-cnfs/ip-forwarder/README.md b/example-cnfs/ip-forwarder/README.md index 9312754e9..3eb300e85 100644 --- a/example-cnfs/ip-forwarder/README.md +++ b/example-cnfs/ip-forwarder/README.md @@ -18,14 +18,14 @@ To run this example CNF, note the following requirements: ### Install helm version 3 ### Automated installation -Run cnf-testsuite setup +Run cnf-testsuite install_dependencies ``` -export KUBECONFIG=$(pwd)/ ; ./cnf-testsuite setup +export KUBECONFIG=$(pwd)/ ; ./cnf-testsuite install_dependencies ``` Setup and deploy service chain ``` -export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_setup cnf-path=example-cnfs/ip-forwarder/cnf-testsuite.yml deploy_with_chart=false +export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_install cnf-path=example-cnfs/ip-forwarder/cnf-testsuite.yml deploy_with_chart=false ``` ### Testing @@ -33,7 +33,7 @@ Run the test suite: `export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite all` ### Automated uninstallation ``` -export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_cleanup +export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_uninstall ``` diff --git a/example-cnfs/linkerd2/README.md b/example-cnfs/linkerd2/README.md index 9dfa48b71..f25f82d9e 100644 --- a/example-cnfs/linkerd2/README.md +++ b/example-cnfs/linkerd2/README.md @@ -9,10 +9,10 @@ Set the KUBECONFIG environment to point to the remote K8s cluster ### Automated Linkerd installation -Run cnf-testsuite setup +Run cnf-testsuite install_dependencies ``` -./cnf-testsuite setup +./cnf-testsuite install_dependencies ``` Install linkerd @@ -24,7 +24,7 @@ helm repo add linkerd https://helm.linkerd.io/stable helm install linkerd-crds linkerd/linkerd-crds -n linkerd --create-namespace -.cnf-testsuite cnf_setup cnf-path=example-cnfs/linkerd2/cnf-testsuite.yml +.cnf-testsuite cnf_install cnf-path=example-cnfs/linkerd2/cnf-testsuite.yml ``` Run the test suite: @@ -36,5 +36,5 @@ Run the test suite: linkerd uninstallation ``` -./cnf-testsuite cnf_cleanup +./cnf-testsuite cnf_uninstall ``` diff --git a/example-cnfs/nsm/README.md b/example-cnfs/nsm/README.md index 4b22b72f4..bf21017ee 100644 --- a/example-cnfs/nsm/README.md +++ b/example-cnfs/nsm/README.md @@ -14,13 +14,13 @@ Follow [Pre-req steps](../../INSTALL.md#pre-requisites), including Initialize the test suite ``` -./cnf-testsuite setup +./cnf-testsuite install_dependencies ``` Configure and deploy NSM as the target CNF ``` -./cnf-testsuite cnf_setup cnf-config=./example-cnfs/nsm/cnf-testsuite.yml deploy_with_chart=false +./cnf-testsuite cnf_install cnf-config=./example-cnfs/nsm/cnf-testsuite.yml deploy_with_chart=false ``` Run the all the tests @@ -34,5 +34,5 @@ Check the results file Uninstall the CNF (including undeployment of NSM) ``` -./cnf-testsuite cnf_cleanup +./cnf-testsuite cnf_uninstall ``` diff --git a/example-cnfs/pantheon-nsm-nat/README.md b/example-cnfs/pantheon-nsm-nat/README.md index cc90d14ba..eb1a66c6e 100644 --- a/example-cnfs/pantheon-nsm-nat/README.md +++ b/example-cnfs/pantheon-nsm-nat/README.md @@ -22,15 +22,15 @@ Follow [Pre-req steps](../../INSTALL.md#pre-requisites), including Initialize the test suite ``` -./cnf-testsuite setup +./cnf-testsuite install_dependencies ``` Configure and deploy nsm and nsm-nat as the target CNF ``` -./cnf-testsuite cnf_setup cnf-config=./example-cnfs/nsm/cnf-testsuite.yml deploy_with_chart=false +./cnf-testsuite cnf_install cnf-config=./example-cnfs/nsm/cnf-testsuite.yml deploy_with_chart=false -./cnf-testsuite cnf_setup cnf-config=./example-cnfs/pantheon-nsm-nat/cnf-testsuite.yml deploy_with_chart=false +./cnf-testsuite cnf_install cnf-config=./example-cnfs/pantheon-nsm-nat/cnf-testsuite.yml deploy_with_chart=false ``` Run the all the tests @@ -44,5 +44,5 @@ Check the results file Uninstall the CNF (including undeployment of nsm-nat) ``` -./cnf-testsuite cnf_cleanup +./cnf-testsuite cnf_uninstall ``` diff --git a/example-cnfs/pantheon-nsm-nat/nat-cnf/README.md b/example-cnfs/pantheon-nsm-nat/nat-cnf/README.md index 331564fac..5c47ce728 100644 --- a/example-cnfs/pantheon-nsm-nat/nat-cnf/README.md +++ b/example-cnfs/pantheon-nsm-nat/nat-cnf/README.md @@ -14,13 +14,13 @@ Follow [Pre-req steps](../../INSTALL.md#pre-requisites), including Initialize the test suite ``` -crystal src/cnf-testsuite.cr setup +crystal src/cnf-testsuite.cr install_dependencies ``` Configure and deploy nsm-nat as the target CNF ``` -crystal src/cnf-testsuite.cr cnf_setup cnf-config=./example-cnfs/pantheon-nsm-nat/cnf-testsuite.yml deploy_with_chart=false +crystal src/cnf-testsuite.cr cnf_install cnf-config=./example-cnfs/pantheon-nsm-nat/cnf-testsuite.yml deploy_with_chart=false ``` Run the all the tests @@ -34,5 +34,5 @@ Check the results file Uninstall the CNF (including undeployment of nsm-nat) ``` -crystal src/cnf-testsuite.cr cnf_cleanup +crystal src/cnf-testsuite.cr cnf_uninstall ``` diff --git a/example-cnfs/vpp-3c2n-csp-use-case/README.md b/example-cnfs/vpp-3c2n-csp-use-case/README.md index 8914286f7..22053ea32 100644 --- a/example-cnfs/vpp-3c2n-csp-use-case/README.md +++ b/example-cnfs/vpp-3c2n-csp-use-case/README.md @@ -16,14 +16,14 @@ Description: ### Install helm version 3 ### Automated installation -Run cnf-testsuite setup +Run cnf-testsuite install_dependencies ``` -export KUBECONFIG=$(pwd)/ ; ./cnf-testsuite setup +export KUBECONFIG=$(pwd)/ ; ./cnf-testsuite install_dependencies ``` Setup and deploy service chain ``` -export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_setup example-cnf-path=example-cnfs/vpp-3c2n-csp-use-case/cnf-testsuite.yml +export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_install example-cnf-path=example-cnfs/vpp-3c2n-csp-use-case/cnf-testsuite.yml ``` ### Testing @@ -31,6 +31,6 @@ Run the test suite: `export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite all` ### Automated uninstallation ``` -export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_cleanup +export KUBECONFIG=$(pwd)/admin.conf ; ./cnf-testsuite cnf_uninstall ``` diff --git a/spec/5g/core_spec.cr b/spec/5g/core_spec.cr index bbd0ba5ba..6f7bfb447 100644 --- a/spec/5g/core_spec.cr +++ b/spec/5g/core_spec.cr @@ -8,50 +8,50 @@ require "sam" describe "Core" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose") (/(PASSED).*(Chaos service degradation is less than 50%)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose baseline_count=300") (/(FAILED).*(Chaos service degradation is more than 50%)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("suci_enabled verbose") (/(PASSED).*(Core uses SUCI 5g authentication)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'suci_enabled' should fail if the 5G core does not have suci enabled", tags: ["5g"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") result = ShellCmd.run_testsuite("suci_enabled verbose") (/(FAILED).*(Core does not use SUCI 5g authentication)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end diff --git a/spec/5g/ran_spec.cr b/spec/5g/ran_spec.cr index 31655a4d1..7144d8ad1 100644 --- a/spec/5g/ran_spec.cr +++ b/spec/5g/ran_spec.cr @@ -30,7 +30,7 @@ end describe "5g" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end diff --git a/spec/cluster_setup_spec.cr b/spec/cluster_setup_spec.cr index 97312bb55..4e0575c8a 100644 --- a/spec/cluster_setup_spec.cr +++ b/spec/cluster_setup_spec.cr @@ -16,8 +16,8 @@ describe "Cluster Setup" do (/Error: Namespace cnf-testsuite does not exist./ =~ result[:output]).should_not be_nil end - it "'install_cluster_tools' should give a message if namespace does not exist even after setup", tags: ["cluster_setup"] do - result = ShellCmd.run_testsuite("setup") + it "'install_cluster_tools' should give a message if namespace does not exist even after dependency installation", tags: ["cluster_setup"] do + result = ShellCmd.run_testsuite("install_dependencies") KubectlClient::Delete.command("namespace #{ClusterTools.namespace}") diff --git a/spec/cnf_testsuite_all/cnf_testsuite_spec.cr b/spec/cnf_testsuite_all/cnf_testsuite_spec.cr index ed3ef5caf..ea1eed8c0 100644 --- a/spec/cnf_testsuite_all/cnf_testsuite_spec.cr +++ b/spec/cnf_testsuite_all/cnf_testsuite_spec.cr @@ -4,7 +4,7 @@ require "colorize" describe CnfTestSuite do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end diff --git a/spec/platform/cluster_api_spec.cr b/spec/platform/cluster_api_spec.cr index 9db6eefa0..3e2a72a03 100644 --- a/spec/platform/cluster_api_spec.cr +++ b/spec/platform/cluster_api_spec.cr @@ -5,7 +5,7 @@ require "../../src/tasks/utils/utils.cr" describe "Cluster API" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end after_each do diff --git a/spec/platform/hardware_and_scheduler_spec.cr b/spec/platform/hardware_and_scheduler_spec.cr index 9ab2ab322..cbf97451b 100644 --- a/spec/platform/hardware_and_scheduler_spec.cr +++ b/spec/platform/hardware_and_scheduler_spec.cr @@ -5,7 +5,7 @@ require "./../../src/tasks/utils/utils.cr" describe "Platform" do before_all do result = ShellCmd.environment_cleanup() - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end diff --git a/spec/platform/observability_spec.cr b/spec/platform/observability_spec.cr index 2ed59119b..42efe622f 100644 --- a/spec/platform/observability_spec.cr +++ b/spec/platform/observability_spec.cr @@ -5,7 +5,7 @@ require "kubectl_client" describe "Platform Observability" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end diff --git a/spec/platform/platform_spec.cr b/spec/platform/platform_spec.cr index 486422779..056d1c0ab 100644 --- a/spec/platform/platform_spec.cr +++ b/spec/platform/platform_spec.cr @@ -5,7 +5,7 @@ require "../../src/tasks/utils/utils.cr" describe "Platform" do before_all do result = ShellCmd.environment_cleanup() - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end it "'platform:*' should not error out when no cnf is installed", tags: ["platform"] do diff --git a/spec/platform/resilience_spec.cr b/spec/platform/resilience_spec.cr index 89dde4dec..eb869a08f 100644 --- a/spec/platform/resilience_spec.cr +++ b/spec/platform/resilience_spec.cr @@ -4,7 +4,7 @@ require "./../../src/tasks/utils/utils.cr" describe "Platform" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end it "'worker_reboot_recovery' should pass if platform successfully recovers after reboot", tags: ["platform:worker_reboot_recovery"] do diff --git a/spec/platform/security_spec.cr b/spec/platform/security_spec.cr index eb596e4d1..daa85189a 100644 --- a/spec/platform/security_spec.cr +++ b/spec/platform/security_spec.cr @@ -4,7 +4,7 @@ require "./../../src/tasks/utils/utils.cr" describe "Platform" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end diff --git a/spec/setup_spec.cr b/spec/setup_spec.cr index 1952892eb..c380875d5 100644 --- a/spec/setup_spec.cr +++ b/spec/setup_spec.cr @@ -6,114 +6,114 @@ require "helm" require "file_utils" require "sam" -describe "Setup" do +describe "Installation" do after_each do result = ShellCmd.environment_cleanup() end - it "'setup' should completely setup the cnf testsuite environment before installing cnfs", tags: ["setup"] do - result = ShellCmd.run_testsuite("setup") + it "'install_dependencies' should install all cnf-testsuite dependencies before installing cnfs", tags:["installation"] do + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true - (/Setup complete/ =~ result[:output]).should_not be_nil + (/Dependency installation complete/ =~ result[:output]).should_not be_nil end - it "'cnf_setup' should pass with a minimal cnf-testsuite.yml", tags: ["setup"] do - result = ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-minimal-cnf/") + it "'cnf_install' should pass with a minimal cnf-testsuite.yml", tags:["installation"] do + result = ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample-minimal-cnf/") (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end - it "'cnf_setup/cnf_cleanup' should install/cleanup with cnf-config arg as an alias for cnf-path", tags: ["setup"] do - result = ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-minimal-cnf/") + it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-config arg as an alias for cnf-path", tags:["installation"] do + result = ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-minimal-cnf/") (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end - it "'cnf_setup/cnf_cleanup' should install/cleanup with cnf-path arg as an alias for cnf-config", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should install/uninstall with cnf-path arg as an alias for cnf-config", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=example-cnfs/coredns/cnf-testsuite.yml") + result = ShellCmd.cnf_install("cnf-path=example-cnfs/coredns/cnf-testsuite.yml") (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup/cnf_cleanup' should fail on incorrect config", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should fail on incorrect config", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=spec/fixtures/sample-bad-config.yml", expect_failure: true) + result = ShellCmd.cnf_install("cnf-path=spec/fixtures/sample-bad-config.yml", expect_failure: true) (/Error during parsing CNF config/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup(expect_failure: true) + result = ShellCmd.cnf_uninstall(expect_failure: true) end end - it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf with a cnf-testsuite.yml", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should install/uninstall a cnf with a cnf-testsuite.yml", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-config=example-cnfs/coredns/cnf-testsuite.yml") + result = ShellCmd.cnf_install("cnf-config=example-cnfs/coredns/cnf-testsuite.yml") (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup/cnf_cleanup' should work with cnf-testsuite.yml that has no directory associated with it", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should work with cnf-testsuite.yml that has no directory associated with it", tags:["installation"] do begin #TODO force cnfs/ to be deployment name and not the directory name - result = ShellCmd.cnf_setup("cnf-config=spec/fixtures/cnf-testsuite.yml verbose") + result = ShellCmd.cnf_install("cnf-config=spec/fixtures/cnf-testsuite.yml verbose") (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup/cnf_cleanup' should install/cleanup with helm_directory that descends multiple directories", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should install/uninstall with helm_directory that descends multiple directories", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") + result = ShellCmd.cnf_install("cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup/cnf_cleanup' should properly install/uninstall old versions of cnf configs", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should properly install/uninstall old versions of cnf configs", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=spec/fixtures/cnf-testsuite-v1-example.yml") + result = ShellCmd.cnf_install("cnf-path=spec/fixtures/cnf-testsuite-v1-example.yml") (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup' should fail if another CNF is already installed", tags: ["setup"] do + it "'cnf_install' should fail if another CNF is already installed", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") + result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") (/CNF installation complete/ =~ result[:output]).should_not be_nil - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-minimal-cnf/cnf-testsuite.yml") + result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-minimal-cnf/cnf-testsuite.yml") (/A CNF is already set up. Setting up multiple CNFs is not allowed./ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf with multiple deployments", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should install/uninstall a cnf with multiple deployments", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_multiple_deployments/cnf-testsuite.yml") + result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_multiple_deployments/cnf-testsuite.yml") (/All "coredns" deployment resources are up/ =~ result[:output]).should_not be_nil (/All "memcached" deployment resources are up/ =~ result[:output]).should_not be_nil (/All "nginx" deployment resources are up/ =~ result[:output]).should_not be_nil (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/Successfully uninstalled helm deployment "coredns"/ =~ result[:output]).should_not be_nil (/Successfully uninstalled helm deployment "memcached"/ =~ result[:output]).should_not be_nil (/Successfully uninstalled helm deployment "nginx"/ =~ result[:output]).should_not be_nil @@ -121,48 +121,48 @@ describe "Setup" do end end - it "'cnf_setup/cnf_cleanup' should install/cleanup deployment with mixed installation methods", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should install/uninstall deployment with mixed installation methods", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-nginx-redis/cnf-testsuite.yml") + result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-nginx-redis/cnf-testsuite.yml") (/All "nginx" deployment resources are up/ =~ result[:output]).should_not be_nil (/All "redis" deployment resources are up/ =~ result[:output]).should_not be_nil (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/Successfully uninstalled helm deployment "nginx"/ =~ result[:output]).should_not be_nil (/Successfully uninstalled manifest deployment "redis"/ =~ result[:output]).should_not be_nil (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup/cnf_cleanup' should handle partial deployment failures gracefully", tags: ["setup"] do + it "'cnf_install/cnf_uninstall' should handle partial deployment failures gracefully", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-partial-deployment-failure/cnf-testsuite.yml", expect_failure: true) + result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-partial-deployment-failure/cnf-testsuite.yml", expect_failure: true) (/All "nginx" deployment resources are up/ =~ result[:output]).should_not be_nil (/Deployment of "coredns" failed during CNF installation/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() (/Successfully uninstalled helm deployment "nginx"/ =~ result[:output]).should_not be_nil end end - it "'cnf_setup' should detect and report conflicts between deployments", tags: ["setup"] do + it "'cnf_install' should detect and report conflicts between deployments", tags:["installation"] do begin - result = ShellCmd.cnf_setup("cnf-path=spec/fixtures/sample-conflicting-deployments.yml", expect_failure: true) + result = ShellCmd.cnf_install("cnf-path=spec/fixtures/sample-conflicting-deployments.yml", expect_failure: true) (/Deployment names should be unique/ =~ result[:output]).should_not be_nil ensure - ShellCmd.cnf_cleanup(expect_failure: true) + ShellCmd.cnf_uninstall(expect_failure: true) end end - it "'cnf_setup' should correctly handle deployment priority", tags: ["setup"] do + it "'cnf_install' should correctly handle deployment priority", tags:["installation"] do # (kosstennbl) ELK stack requires to be installed with specific order, otherwise it would give errors begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-elk-stack/cnf-testsuite.yml timeout=600") + result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-elk-stack/cnf-testsuite.yml timeout=600") result[:status].success?.should be_true (/CNF installation complete/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true (/All CNF deployments were uninstalled/ =~ result[:output]).should_not be_nil end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 4fe7ac863..7c23cc35f 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -21,9 +21,9 @@ module ShellCmd run(cmd, log_prefix: "ShellCmd.run_testsuite", force_output: true, joined_output: true) end - def self.cnf_setup(setup_params, cmd_prefix="", expect_failure=false) - timeout_parameter = setup_params.includes?("timeout") ? "" : "timeout=300" - result = run_testsuite("cnf_setup #{setup_params} #{timeout_parameter}", cmd_prefix) + def self.cnf_install(install_params, cmd_prefix="", expect_failure=false) + timeout_parameter = install_params.includes?("timeout") ? "" : "timeout=300" + result = run_testsuite("cnf_install #{install_params} #{timeout_parameter}", cmd_prefix) if !expect_failure result[:status].success?.should be_true else @@ -32,9 +32,9 @@ module ShellCmd result end - def self.cnf_cleanup(cleanup_params="", cmd_prefix="", expect_failure=false) - timeout_parameter = cleanup_params.includes?("timeout") ? "" : "timeout=300" - result = run_testsuite("cnf_cleanup #{cleanup_params} #{timeout_parameter}", cmd_prefix) + def self.cnf_uninstall(uninstall_params="", cmd_prefix="", expect_failure=false) + timeout_parameter = uninstall_params.includes?("timeout") ? "" : "timeout=300" + result = run_testsuite("cnf_uninstall #{uninstall_params} #{timeout_parameter}", cmd_prefix) if !expect_failure result[:status].success?.should be_true else diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr index 08644842c..e307ae905 100644 --- a/spec/utils/cnf_manager_spec.cr +++ b/spec/utils/cnf_manager_spec.cr @@ -176,7 +176,7 @@ describe "SampleUtils" do begin args = Sam::Args.new() config_path = "./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml" - ShellCmd.cnf_setup("cnf-config=#{config_path}") + ShellCmd.cnf_install("cnf-config=#{config_path}") config = CNFInstall::Config.parse_cnf_config_from_file(config_path) task_response = CNFManager.workload_resource_test(args, config) do |resource, container, initialized| test_passed = true @@ -192,20 +192,20 @@ describe "SampleUtils" do end (task_response).should be_true ensure - ShellCmd.cnf_cleanup() + ShellCmd.cnf_uninstall() end end it "Helm_values should be used during the installation of a cnf", tags: ["cnf-config"] do begin # fails because doesn't have a service - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_coredns_values") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample_coredns_values") deployment_containers = KubectlClient::Get.resource_containers("deployment", "coredns-coredns", "cnf-default") image_tags = KubectlClient::Get.container_image_tags(deployment_containers) Log.info { "image_tags: #{image_tags}" } (/1.6.9/ =~ image_tags[0][:tag]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end end diff --git a/spec/utils/k8s_instrumentation_spec.cr b/spec/utils/k8s_instrumentation_spec.cr index 3a4cda09e..79e55fa32 100644 --- a/spec/utils/k8s_instrumentation_spec.cr +++ b/spec/utils/k8s_instrumentation_spec.cr @@ -6,7 +6,7 @@ require "sam" describe "K8sInstrumentation" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end diff --git a/spec/utils/kubescape_spec.cr b/spec/utils/kubescape_spec.cr index a3df6ad9c..36ffa8078 100644 --- a/spec/utils/kubescape_spec.cr +++ b/spec/utils/kubescape_spec.cr @@ -8,13 +8,13 @@ describe "K8sInstrumentation" do end it "'#scan and #test_by_test_name' should return the results of a kubescape scan", tags: ["kubescape"] do - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") Kubescape.scan results_json = Kubescape.parse test_json = Kubescape.test_by_test_name(results_json, "Network policies") (test_json).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end diff --git a/spec/utils/utils_spec.cr b/spec/utils/utils_spec.cr index 389abe526..31a69a2b4 100644 --- a/spec/utils/utils_spec.cr +++ b/spec/utils/utils_spec.cr @@ -10,7 +10,7 @@ require "json" describe "Utils" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true # Ensure a results file is present to test different scenarios @@ -18,10 +18,10 @@ describe "Utils" do end before_each do - result = ShellCmd.run_testsuite("results_yml_cleanup") + result = ShellCmd.run_testsuite("delete_results") end after_each do - result = ShellCmd.run_testsuite("results_yml_cleanup") + result = ShellCmd.run_testsuite("delete_results") end it "'toggle' should return a boolean for a toggle in the config.yml", tags: ["args"] do @@ -77,8 +77,8 @@ describe "Utils" do begin args = Sam::Args.new() config_path = "./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml" - ShellCmd.cnf_setup("cnf-config=#{config_path}") - task_response = CNFManager::Task.single_task_runner(args) do |args, config| + ShellCmd.cnf_install("cnf-config=#{config_path}") + task_response = CNFManager::Task.single_task_runner(args) do |args, config| Log.info { "single_task_runner spec args #{args.inspect}" } white_list_container_names = config.common.white_list_container_names Log.info { "white_list_container_names #{white_list_container_names.inspect}" } @@ -109,7 +109,7 @@ describe "Utils" do end (task_response).should eq("✔️ PASSED: No privileged containers") ensure - ShellCmd.cnf_cleanup() + ShellCmd.cnf_uninstall() end end @@ -176,13 +176,13 @@ describe "Utils" do it "'logger' or verbose output should be shown when verbose flag is set", tags: ["logger"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("helm_deploy verbose", cmd_prefix: "LOG_LEVEL=info") puts result[:output] result[:status].success?.should be_true (/helm_deploy args/ =~ result[:output]).should_not be_nil ensure - ShellCmd.cnf_cleanup() + ShellCmd.cnf_uninstall() end end diff --git a/spec/workload/compatibility_spec.cr b/spec/workload/compatibility_spec.cr index fd6fb0e29..d9c62a4b0 100644 --- a/spec/workload/compatibility_spec.cr +++ b/spec/workload/compatibility_spec.cr @@ -8,13 +8,13 @@ require "sam" describe "Compatibility" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end it "'cni_compatible' should pass if the cnf works with calico and flannel", tags: ["compatibility"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") retry_limit = 5 retries = 1 result = ShellCmd.run_testsuite("cni_compatible verbose") @@ -28,19 +28,19 @@ describe "Compatibility" do (/(SKIPPED).*(cni_compatible test was temporarily disabled)/ =~ result[:output]).should_not be_nil #(/(PASSED).*(CNF compatible with both Calico and Cilium)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'increase_decrease_capacity' should pass ", tags: ["increase_decrease_capacity"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("increase_decrease_capacity verbose") result[:status].success?.should be_true (/(PASSED).*(Replicas increased to)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end end diff --git a/spec/workload/configuration_spec.cr b/spec/workload/configuration_spec.cr index 0c4ee33bf..4459e77f7 100644 --- a/spec/workload/configuration_spec.cr +++ b/spec/workload/configuration_spec.cr @@ -11,80 +11,80 @@ describe CnfTestSuite do Log.debug { result[:output] } result = ShellCmd.environment_cleanup() - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") end it "'liveness' should pass when livenessProbe is set", tags: ["liveness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml") result = ShellCmd.run_testsuite("liveness verbose", cmd_prefix:"LOG_LEVEL=debug") result[:status].success?.should be_true (/(PASSED).*(Helm liveness probe)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'liveness' should fail when livenessProbe is not set", tags: ["liveness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("liveness verbose") result[:status].success?.should be_true (/(FAILED).*(No livenessProbe found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'readiness' should pass when readinessProbe is set", tags: ["readiness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml") result = ShellCmd.run_testsuite("readiness verbose", cmd_prefix: "LOG_LEVEL=debug") result[:status].success?.should be_true (/(PASSED).*(Helm readiness probe)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'readiness' should fail when readinessProbe is not set", tags: ["readiness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("readiness verbose") result[:status].success?.should be_true (/(FAILED).*(No readinessProbe found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'rolling_update' should pass when valid version is given", tags: ["rolling_update"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'rolling_update' should fail when invalid version is given", tags: ["rolling_update"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'rolling_downgrade' should pass when valid version is given", tags: ["rolling_downgrade"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") retry_limit = 5 retries = 1 result = ShellCmd.run_testsuite("rolling_downgrade verbose") @@ -98,51 +98,51 @@ describe CnfTestSuite do result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'rolling_downgrade' should fail when invalid version is given", tags: ["rolling_downgrade"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") result = ShellCmd.run_testsuite("rolling_downgrade verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'rolling_version_change' should pass when valid version is given", tags: ["rolling_version_change"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'rolling_version_change' should fail when invalid version is given", tags: ["rolling_version_change"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'rollback' should pass ", tags: ["rollback"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("rollback verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end @@ -150,212 +150,212 @@ describe CnfTestSuite do it "'nodeport_not_used' should fail when a node port is being used", tags: ["nodeport_not_used"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_nodeport") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_nodeport") result = ShellCmd.run_testsuite("nodeport_not_used verbose") result[:status].success?.should be_true (/(FAILED).*(NodePort is being used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'nodeport_not_used' should pass when a node port is not being used", tags: ["nodeport_not_used"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("nodeport_not_used verbose") result[:status].success?.should be_true (/(PASSED).*(NodePort is not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'hostport_not_used' should fail when a node port is being used", tags: ["hostport_not_used"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_hostport") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_hostport") result = ShellCmd.run_testsuite("hostport_not_used verbose") result[:status].success?.should be_true (/(FAILED).*(HostPort is being used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'hostport_not_used' should pass when a node port is not being used", tags: ["hostport_not_used"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("hostport_not_used verbose") result[:status].success?.should be_true (/(PASSED).*(HostPort is not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should fail when a hardcoded ip is found in the K8s configuration", tags: ["ip_addresses"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns_hardcoded_ips") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns_hardcoded_ips") result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose", cmd_prefix: "LOG_LEVEL=info") result[:status].success?.should be_true (/(FAILED).*(Hard-coded IP addresses found in the runtime K8s configuration)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should pass when no ip addresses are found in the K8s configuration", tags: ["ip_addresses"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose") result[:status].success?.should be_true (/(PASSED).*(No hard-coded IP addresses found in the runtime K8s configuration)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'secrets_used' should pass when secrets are provided as volumes and used by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(PASSED).*(Secrets defined and used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end end it "'secrets_used' should be skipped when secrets are provided as volumes and not mounted by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end end it "'secrets_used' should pass when secrets are provided as environment variables and used by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(PASSED).*(Secrets defined and used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end end it "'secrets_used' should skip when secrets are not referenced as environment variables by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml skip_wait_for_install verbose") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml skip_wait_for_install verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end end it "'secrets_used' should be skipped when no secret volumes are mounted or no container secrets are provided (secrets ignored)`", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end end it "'immutable_configmap' fail with some mutable configmaps in container env or volume mount", tags: ["immutable_configmap"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/ndn-mutable-configmap") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/ndn-mutable-configmap") result = ShellCmd.run_testsuite("immutable_configmap verbose") result[:status].success?.should be_true (/(FAILED).*(Found mutable configmap)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'immutable_configmap' pass with all immutable configmaps in container env or volume mounts", tags: ["immutable_configmap"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/ndn-immutable-configmap") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/ndn-immutable-configmap") result = ShellCmd.run_testsuite("immutable_configmap verbose") result[:status].success?.should be_true (/(PASSED).*(All volume or container mounted configmaps immutable)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'require_labels' should fail if a cnf does not have the app.kubernetes.io/name label", tags: ["require_labels"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") result = ShellCmd.run_testsuite("require_labels verbose") result[:status].success?.should be_true (/(FAILED).*(Pods should have the app.kubernetes.io\/name label)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'require_labels' should pass if a cnf has the app.kubernetes.io/name label", tags: ["require_labels"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("require_labels verbose") result[:status].success?.should be_true (/(PASSED).*(Pods have the app.kubernetes.io\/name label)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'default_namespace' should fail if a cnf creates resources in the default namespace", tags: ["default_namespace"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_default_namespace") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns_default_namespace") result = ShellCmd.run_testsuite("default_namespace verbose") result[:status].success?.should be_true (/(FAILED).*(Resources are created in the default namespace)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() KubectlClient::Utils.wait_for_terminations() end end it "'default_namespace' should pass if a cnf does not create resources in the default namespace", tags: ["default_namespace"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("default_namespace verbose") result[:status].success?.should be_true (/(PASSED).*(default namespace is not being used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() KubectlClient::Utils.wait_for_terminations() end end it "'latest_tag' should fail if a cnf has containers that use images with the latest tag", tags: ["latest_tag"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("latest_tag verbose") result[:status].success?.should be_true (/(FAILED).*(Container images are using the latest tag)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'latest_tag' should pass if a cnf does not have containers that use images with the latest tag", tags: ["latest_tag"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("latest_tag verbose") result[:status].success?.should be_true (/(PASSED).*(Container images are not using the latest tag)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end diff --git a/spec/workload/installability_spec.cr b/spec/workload/installability_spec.cr index 0e0bf870a..8d8c05c01 100644 --- a/spec/workload/installability_spec.cr +++ b/spec/workload/installability_spec.cr @@ -4,66 +4,66 @@ require "colorize" describe CnfTestSuite do before_all do result = ShellCmd.environment_cleanup() - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") end it "'helm_deploy' should fail on a manifest CNF", tags: ["helm"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/k8s-non-helm") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/k8s-non-helm") result = ShellCmd.run_testsuite("helm_deploy verbose") result[:status].success?.should be_true (/(FAILED).*(CNF has deployments that are not installed with helm)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end it "'helm_deploy' should fail if command is not supplied cnf-config argument", tags: ["helm"] do result = ShellCmd.run_testsuite("helm_deploy") result[:status].success?.should be_true - (/No cnf_testsuite.yml found! Did you run the setup task/ =~ result[:output]).should_not be_nil + (/No cnf_testsuite.yml found! Did you run the install_depenencies task/ =~ result[:output]).should_not be_nil end it "'helm_chart_valid' should pass on a good helm chart", tags: ["helm"] do - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("helm_chart_valid verbose") result[:status].success?.should be_true (/Helm chart lint passed on all charts/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end it "'helm_chart_valid' should fail on a bad helm chart", tags: ["helm"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose skip_wait_for_install", expect_failure: true) + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose skip_wait_for_install", expect_failure: true) result = ShellCmd.run_testsuite("helm_chart_valid") result[:status].success?.should be_true (/Helm chart lint failed on one or more charts/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("verbose") + result = ShellCmd.cnf_uninstall("verbose") end end it "'helm_chart_published' should pass on a good helm chart repo", tags: ["helm_chart_published"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("helm_chart_published") result[:status].success?.should be_true (/(PASSED).*(All Helm charts are published)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'helm_chart_published' should fail on a bad helm chart repo", tags: ["helm_chart_published"] do begin result = ShellCmd.run("helm search repo stable/coredns", force_output: true) - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-bad-helm-repo skip_wait_for_install", expect_failure: true) + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-bad-helm-repo skip_wait_for_install", expect_failure: true) result = ShellCmd.run("helm search repo stable/coredns", force_output: true) result = ShellCmd.run_testsuite("helm_chart_published verbose") result[:status].success?.should be_true (/(FAILED).*(One or more Helm charts are not published)/ =~ result[:output]).should_not be_nil ensure result = ShellCmd.run("#{Helm::BinarySingleton.helm} repo remove badrepo") - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end end diff --git a/spec/workload/microservice_spec.cr b/spec/workload/microservice_spec.cr index 2c15d5401..f59825156 100644 --- a/spec/workload/microservice_spec.cr +++ b/spec/workload/microservice_spec.cr @@ -9,57 +9,57 @@ require "sam" describe "Microservice" do before_all do - Log.info { "Running testsuite setup" } - result = ShellCmd.run_testsuite("setup") + Log.info { "Running testsuite install_dependencies" } + result = ShellCmd.run_testsuite("install_dependencies") process_result = result[:status].success? - Log.info(&.emit("Testsuite setup process result", process_result: process_result)) + Log.info(&.emit("Testsuite install_dependencies process result", process_result: process_result)) process_result.should be_true end it "'shared_database' should be skipped no MariaDB containers are found", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(N\/A).*(No MariaDB containers were found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'shared_database' should pass if no database is used by two microservices", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(PASSED).*(No shared database found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'shared_database' should pass if one service connects to a database but other non-service connections are made to the database", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(PASSED).*(No shared database found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'shared_database' should fail if two services on the cluster connect to the same database", tags: ["shared_database2"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(FAILED).*(Found a shared database)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true KubectlClient::Delete.command("pvc data-test-mariadb-0 -n wordpress") end @@ -67,82 +67,82 @@ describe "Microservice" do it "'shared_database' should pass if two services on the cluster connect to the same database but they are not in the helm chart of the cnf", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns") Helm.install("multi-db sample-cnfs/ndn-multi-db-connections-fail/wordpress/") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress", wait_count: 180, namespace: "default") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress2", wait_count: 180, namespace: "default") # todo kubctl appy of all resourcesin ndn-multi-db-connections-fail - # todo cnf_setup of coredns + # todo cnf_install of coredns # todo run shared_database (should pass) # todo kubectl delete on ndn resourcws - # toto cnf_cleanup on coredns + # toto cnf_uninstall on coredns result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(PASSED).*(No shared database found)/ =~ result[:output]).should_not be_nil ensure Helm.delete("multi-db") KubectlClient::Delete.command("pvc data-multi-db-mariadb-0") - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'single_process_type' should pass if the containers in the cnf have only one process type", tags: ["process_check"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(PASSED).*(Only one process type used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'single_process_type' should fail if the containers in the cnf have more than one process type", tags: ["process_check"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/k8s-multiple-processes") + ShellCmd.cnf_install("cnf-path=sample-cnfs/k8s-multiple-processes") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(FAILED).*(More than one process type used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'single_process_type' should fail if the containers in the cnf have more than one process type and in a pod", tags: ["process_check"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-multiple-processes") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample-multiple-processes") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(FAILED).*(More than one process type used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'reasonable_startup_time' should pass if the cnf has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("reasonable_startup_time verbose") result[:status].success?.should be_true (/(PASSED).*(CNF had a reasonable startup time)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'reasonable_startup_time' should fail if the cnf doesn't has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") begin result = ShellCmd.run_testsuite("reasonable_startup_time verbose") result[:status].success?.should be_true (/(FAILED).*(CNF had a startup time of)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -157,59 +157,59 @@ describe "Microservice" do #else cnf = "./sample-cnfs/sample-coredns-cnf" #end - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.cnf_install("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end it "'reasonable_image_size' should fail if image is larger than 5gb", tags: ["reasonable_image_size"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/ndn-reasonable-image-size skip_wait_for_install") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/ndn-reasonable-image-size skip_wait_for_install") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size too large/ =~ result[:output]).should_not be_nil end it "'specialized_init_system' should fail if pods do not use specialized init systems", tags: ["specialized_init_system"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("specialized_init_system") result[:status].success?.should be_true (/Containers do not use specialized init systems/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end it "'specialized_init_system' should pass if pods use specialized init systems", tags: ["specialized_init_system"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-init-systems") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample-init-systems") result = ShellCmd.run_testsuite("specialized_init_system") result[:status].success?.should be_true (/Containers use specialized init systems/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end it "'service_discovery' should pass if any containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("service_discovery verbose") result[:status].success?.should be_true (/(PASSED).*(Some containers exposed as a service)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'service_discovery' should fail if no containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-ndn-privileged") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample-ndn-privileged") result = ShellCmd.run_testsuite("service_discovery verbose") result[:status].success?.should be_true (/(FAILED).*(No containers exposed as a service)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -221,12 +221,12 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling/") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample_good_signal_handling/") result = ShellCmd.run_testsuite("sig_term_handled verbose") result[:status].success?.should be_true (/(PASSED).*(Sig Term handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -238,12 +238,12 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_bad_signal_handling/") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample_bad_signal_handling/") result = ShellCmd.run_testsuite("sig_term_handled verbose") result[:status].success?.should be_true (/(FAILED).*(Sig Term not handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -255,7 +255,7 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") # Workaround to wait using kubectl because Jenkins pod takes a LONG time to start. result = ShellCmd.run("kubectl wait --for=condition=ready=True pod/jenkins-0 -n cnfspace --timeout=500s", force_output: true) @@ -264,7 +264,7 @@ describe "Microservice" do result[:status].success?.should be_true (/(PASSED).*(Sig Term handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -272,24 +272,24 @@ describe "Microservice" do it "'zombie_handled' should pass if a zombie is succesfully reaped by PID 1", tags: ["zombie"] do begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_zombie_handling/") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample_good_zombie_handling/") result = ShellCmd.run_testsuite("zombie_handled verbose") result[:status].success?.should be_true (/(PASSED).*(Zombie handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end it "'zombie_handled' should failed if a zombie is not succesfully reaped by PID 1", tags: ["zombie"] do begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-bad-zombie/") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample-bad-zombie/") result = ShellCmd.run_testsuite("zombie_handled verbose") result[:status].success?.should be_true (/(FAILED).*(Zombie not handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end diff --git a/spec/workload/observability_spec.cr b/spec/workload/observability_spec.cr index 60e18eb25..ac3355adb 100644 --- a/spec/workload/observability_spec.cr +++ b/spec/workload/observability_spec.cr @@ -6,34 +6,34 @@ require "../../src/tasks/jaeger_setup.cr" describe "Observability" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result[:status].success?.should be_true end it "'log_output' should pass with a cnf that outputs logs to stdout", tags: ["observability"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("log_output verbose") result[:status].success?.should be_true (/(PASSED).*(Resources output logs to stdout and stderr)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'log_output' should fail with a cnf that does not output logs to stdout", tags: ["observability"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("log_output verbose") result[:status].success?.should be_true (/(FAILED).*(Resources do not output logs to stdout and stderr)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'prometheus_traffic' should pass if there is prometheus traffic", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm Log.info { "Add prometheus helm repo" } @@ -49,26 +49,26 @@ describe "Observability" do test_result = ShellCmd.run_testsuite("prometheus_traffic") (/(PASSED).*(Your cnf is sending prometheus traffic)/ =~ test_result[:output]).should_not be_nil ensure - ShellCmd.cnf_cleanup() + ShellCmd.cnf_uninstall() result = ShellCmd.run("#{helm} delete prometheus", "helm_delete_prometheus") result[:status].success?.should be_true end it "'prometheus_traffic' should skip if there is no prometheus installed", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result = ShellCmd.run_testsuite("prometheus_traffic") (/(SKIPPED).*(Prometheus server not found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end it "'prometheus_traffic' should fail if the cnf is not registered with prometheus", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) @@ -80,13 +80,13 @@ describe "Observability" do result = ShellCmd.run_testsuite("prometheus_traffic") (/(FAILED).*(Your cnf is not sending prometheus traffic)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result[:status].success?.should be_true end it "'open_metrics' should fail if there is not a valid open metrics response from the cnf", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm @@ -98,13 +98,13 @@ describe "Observability" do result = ShellCmd.run_testsuite("open_metrics") (/(FAILED).*(Your cnf's metrics traffic is not OpenMetrics compatible)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result[:status].success?.should be_true end it "'open_metrics' should pass if there is a valid open metrics response from the cnf", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm @@ -116,36 +116,36 @@ describe "Observability" do result = ShellCmd.run_testsuite("open_metrics") (/(PASSED).*(Your cnf's metrics traffic is OpenMetrics compatible)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result[:status].success?.should be_true end it "'routed_logs' should pass if cnfs logs are captured by fluentd bitnami", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("install_fluentdbitnami") result = ShellCmd.run_testsuite("routed_logs") (/(PASSED).*(Your CNF's logs are being captured)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result = ShellCmd.run_testsuite("uninstall_fluentdbitnami") result[:status].success?.should be_true end it "'routed_logs' should pass if cnfs logs are captured by fluentbit", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-fluentbit") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-fluentbit") result = ShellCmd.run_testsuite("install_fluentbit") result = ShellCmd.run_testsuite("routed_logs") (/(PASSED).*(Your CNF's logs are being captured)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result = ShellCmd.run_testsuite("uninstall_fluentbit") result[:status].success?.should be_true end it "'routed_logs' should fail if cnfs logs are not captured", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Helm.helm_repo_add("bitnami","https://charts.bitnami.com/bitnami") #todo #helm install --values ./override.yml fluentd ./fluentd Helm.install("--values ./spec/fixtures/fluentd-values-bad.yml -n #{TESTSUITE_NAMESPACE} fluentd bitnami/fluentd") @@ -155,7 +155,7 @@ describe "Observability" do result = ShellCmd.run_testsuite("routed_logs") (/(FAILED).*(Your CNF's logs are not being captured)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result = ShellCmd.run_testsuite("uninstall_fluentd") result[:status].success?.should be_true end diff --git a/spec/workload/operator_spec.cr b/spec/workload/operator_spec.cr index 41deb90f6..0689d3bed 100644 --- a/spec/workload/operator_spec.cr +++ b/spec/workload/operator_spec.cr @@ -23,11 +23,11 @@ describe "Operator" do Helm.install("operator --set olm.image.ref=quay.io/operator-framework/olm:v0.22.0 --set catalog.image.ref=quay.io/operator-framework/olm:v0.22.0 --set package.image.ref=quay.io/operator-framework/olm:v0.22.0 #{install_dir}/deploy/chart/") begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_operator", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.cnf_install("cnf-path=./sample-cnfs/sample_operator", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("operator_installed", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(Operator is installed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup(cmd_prefix: "LOG_LEVEL=info") + result = ShellCmd.cnf_uninstall(cmd_prefix: "LOG_LEVEL=info") result[:status].success?.should be_true pods = KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("catalog-operator", "operator-lifecycle-manager"), "operator-lifecycle-manager") + KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("olm-operator", "operator-lifecycle-manager"), "operator-lifecycle-manager") + KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("packageserver", "operator-lifecycle-manager"), "operator-lifecycle-manager") @@ -67,11 +67,11 @@ describe "Operator" do it "'operator_test' operator should not be found", tags: ["operator_test"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("operator_installed", cmd_prefix: "LOG_LEVEL=info") (/(N\/A).*(No Operators Found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end diff --git a/spec/workload/registry_spec.cr b/spec/workload/registry_spec.cr index 1e00da346..9a8be87b1 100644 --- a/spec/workload/registry_spec.cr +++ b/spec/workload/registry_spec.cr @@ -13,7 +13,7 @@ end describe "Private Registry: Image" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") Dockerd.install install_registry = KubectlClient::Apply.file(registry_manifest_path) KubectlClient::Get.resource_wait_for_install("Pod", "registry") @@ -39,23 +39,23 @@ describe "Private Registry: Image" do it "'reasonable_image_size' should pass if using local registry and a port", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.cnf_install("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end it "'reasonable_image_size' should pass if using local registry, a port and an org", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry_org_image" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.cnf_install("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end after_all do @@ -66,7 +66,7 @@ end describe "Private Registry: Rolling" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") Dockerd.install install_registry = KubectlClient::Apply.file(registry_manifest_path) KubectlClient::Get.resource_wait_for_install("Pod", "registry") @@ -88,12 +88,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.cnf_install("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("timeout=0") + result = ShellCmd.cnf_uninstall("timeout=0") end end @@ -101,12 +101,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.cnf_install("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("timeout=0") + result = ShellCmd.cnf_uninstall("timeout=0") end end @@ -114,12 +114,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.cnf_install("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("timeout=0") + result = ShellCmd.cnf_uninstall("timeout=0") end end diff --git a/spec/workload/resilience/disk_fill_spec.cr b/spec/workload/resilience/disk_fill_spec.cr index 0a5a6edc6..1a82e61d7 100644 --- a/spec/workload/resilience/disk_fill_spec.cr +++ b/spec/workload/resilience/disk_fill_spec.cr @@ -7,19 +7,19 @@ require "sam" describe "Resilience Disk Fill Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'disk_fill' A 'Good' CNF should not crash when disk fill occurs", tags: ["disk_fill"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml skip_wait_for_install") result = ShellCmd.run_testsuite("disk_fill verbose") result[:status].success?.should be_true (/(PASSED).*(disk_fill chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup("timeout=0") + result = ShellCmd.cnf_uninstall("timeout=0") result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/network_chaos_spec.cr b/spec/workload/resilience/network_chaos_spec.cr index 84812e28e..e0baa8c23 100644 --- a/spec/workload/resilience/network_chaos_spec.cr +++ b/spec/workload/resilience/network_chaos_spec.cr @@ -7,7 +7,7 @@ require "sam" describe "Resilience Network Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/node_drain_spec.cr b/spec/workload/resilience/node_drain_spec.cr index a526a80fa..9237a6b00 100644 --- a/spec/workload/resilience/node_drain_spec.cr +++ b/spec/workload/resilience/node_drain_spec.cr @@ -7,14 +7,14 @@ require "sam" describe "Resilience Node Drain Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'node_drain' A 'Good' CNF should not crash when node drain occurs", tags: ["node_drain"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("node_drain verbose") result[:status].success?.should be_true if KubectlClient::Get.schedulable_nodes_list.size > 1 @@ -23,7 +23,7 @@ describe "Resilience Node Drain Chaos" do (/(SKIPPED).*(node_drain chaos test requires the cluster to have atleast two)/ =~ result[:output]).should_not be_nil end ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/pod_delete_spec.cr b/spec/workload/resilience/pod_delete_spec.cr index 447835f0e..1dcc3529b 100644 --- a/spec/workload/resilience/pod_delete_spec.cr +++ b/spec/workload/resilience/pod_delete_spec.cr @@ -7,19 +7,19 @@ require "sam" describe "Resilience pod delete Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'pod_delete' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_delete"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_delete verbose") result[:status].success?.should be_true (/(PASSED).*(pod_delete chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/pod_dns_error_spec.cr b/spec/workload/resilience/pod_dns_error_spec.cr index fd00b3835..1e1f06255 100644 --- a/spec/workload/resilience/pod_dns_error_spec.cr +++ b/spec/workload/resilience/pod_dns_error_spec.cr @@ -7,24 +7,24 @@ require "sam" describe "Resilience pod dns error Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'pod_dns_error' A 'Good' CNF should not crash when pod dns error occurs", tags: ["pod_dns_error"] do begin - ShellCmd.cnf_setup("cnf-config=example-cnfs/envoy/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=example-cnfs/envoy/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_dns_error verbose") result[:status].success?.should be_true ((/(SKIPPED).*(pod_dns_error docker runtime not found)/) =~ result[:output] || (/(PASSED).*(pod_dns_error chaos test passed)/ =~ result[:output])).should_not be_nil rescue ex # Raise back error to ensure test fails. - # The ensure block will cleanup the CNF and the litmus installation. + # The ensure block will uninstall the CNF and Litmus. raise "Test failed with #{ex.message}" ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/pod_io_stress_spec.cr b/spec/workload/resilience/pod_io_stress_spec.cr index 41a73c808..a4090f581 100644 --- a/spec/workload/resilience/pod_io_stress_spec.cr +++ b/spec/workload/resilience/pod_io_stress_spec.cr @@ -7,19 +7,19 @@ require "sam" describe "Resilience pod delete Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'pod_io_stress' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_io_stress"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_io_stress verbose") result[:status].success?.should be_true (/(PASSED).*(pod_io_stress chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/pod_memory_hog_spec.cr b/spec/workload/resilience/pod_memory_hog_spec.cr index 0d982e60a..84676933a 100644 --- a/spec/workload/resilience/pod_memory_hog_spec.cr +++ b/spec/workload/resilience/pod_memory_hog_spec.cr @@ -7,19 +7,19 @@ require "sam" describe "Resilience pod memory hog Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'pod_memory_hog' A 'Good' CNF should not crash when pod memory hog occurs", tags: ["pod_memory_hog"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_memory_hog verbose") result[:status].success?.should be_true (/(PASSED).*(pod_memory_hog chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/pod_network_corruption_spec.cr b/spec/workload/resilience/pod_network_corruption_spec.cr index 8d726dd92..bc070e811 100644 --- a/spec/workload/resilience/pod_network_corruption_spec.cr +++ b/spec/workload/resilience/pod_network_corruption_spec.cr @@ -7,19 +7,19 @@ require "sam" describe "Resilience Pod Network corruption Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'pod_network_corruption' A 'Good' CNF should not crash when network corruption occurs", tags: ["pod_network_corruption"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_corruption verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_corruption chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/pod_network_duplication_spec.cr b/spec/workload/resilience/pod_network_duplication_spec.cr index a827f5016..e20fb8d21 100644 --- a/spec/workload/resilience/pod_network_duplication_spec.cr +++ b/spec/workload/resilience/pod_network_duplication_spec.cr @@ -7,19 +7,19 @@ require "sam" describe "Resilience Pod Network duplication Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'pod_network_duplication' A 'Good' CNF should not crash when network duplication occurs", tags: ["pod_network_duplication"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_duplication verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_duplication chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/resilience/pod_network_latency_spec.cr b/spec/workload/resilience/pod_network_latency_spec.cr index 18426d214..4439e2150 100644 --- a/spec/workload/resilience/pod_network_latency_spec.cr +++ b/spec/workload/resilience/pod_network_latency_spec.cr @@ -7,19 +7,19 @@ require "sam" describe "Resilience Pod Network Latency Chaos" do before_all do - result = ShellCmd.run_testsuite("setup") + result = ShellCmd.run_testsuite("install_dependencies") result = ShellCmd.run_testsuite("configuration_file_setup") result[:status].success?.should be_true end it "'pod_network_latency' A 'Good' CNF should not crash when network latency occurs", tags: ["pod_network_latency"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_latency verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_latency chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true diff --git a/spec/workload/security_spec.cr b/spec/workload/security_spec.cr index d05d08266..b75e62e95 100644 --- a/spec/workload/security_spec.cr +++ b/spec/workload/security_spec.cr @@ -6,348 +6,348 @@ describe "Security" do it "'privileged_containers' should pass with a non-privileged cnf", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privileged_containers verbose") result[:status].success?.should be_true (/No privileged containers/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() Log.debug { result[:output] } end end it "'privileged_containers' should fail on a non-whitelisted, privileged cnf", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("privileged_containers verbose") result[:status].success?.should be_true (/Found.*privileged containers.*/ =~ result[:output]).should_not be_nil (/Privileged container (privileged-coredns) in.*/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'privileged_containers' should pass on a whitelisted, privileged cnf", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("privileged_containers verbose") result[:status].success?.should be_true (/Found.*privileged containers.*/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'privilege_escalation' should fail on a cnf that has escalated privileges", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privilege_escalation") result[:status].success?.should be_true (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'privilege_escalation' should pass on a cnf that does not have escalated privileges", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privilege_escalation") result[:status].success?.should be_true (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'symlink_file_system' should pass on a cnf that does not allow a symlink attack", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("symlink_file_system") result[:status].success?.should be_true (/(PASSED).*(No containers allow a symlink attack)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'insecure_capabilities' should pass on a cnf that does not have containers with insecure capabilities", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("insecure_capabilities") result[:status].success?.should be_true (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'insecure_capabilities' should fail on a cnf that has containers with insecure capabilities", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") result = ShellCmd.run_testsuite("insecure_capabilities") result[:status].success?.should be_true (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'linux_hardening' should fail on a cnf that does not make use of security services", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("linux_hardening") result[:status].success?.should be_true (/(PASSED).*(Security services are being used to harden applications)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'application_credentials' should fail on a cnf that allows applications credentials in configuration files", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") result = ShellCmd.run_testsuite("application_credentials") result[:status].success?.should be_true (/(FAILED).*(Found applications credentials in configuration files)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'host_network' should pass on a cnf that does not have a host network attached to pod", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("host_network") result[:status].success?.should be_true (/(PASSED).*(No host network attached to pod)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'service_account_mapping' should fail on a cnf that automatically maps the service account", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") result = ShellCmd.run_testsuite("service_account_mapping") result[:status].success?.should be_true (/(FAILED).*(Service accounts automatically mapped)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'cpu_limits' should pass on a cnf that has containers with cpu limits set", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("cpu_limits") result[:status].success?.should be_true (/(PASSED).*(Containers have CPU limits set)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'memory_limits' should pass on a cnf that has containers with memory limits set", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("memory_limits") result[:status].success?.should be_true (/(PASSED).*(Containers have memory limits set)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'ingress_egress_blocked' should fail on a cnf that has no ingress and egress traffic policy", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("ingress_egress_blocked") result[:status].success?.should be_true (/(PASSED).*(Ingress and Egress traffic blocked on pods)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'host_pid_ipc_privileges' should pass on a cnf that does not have containers with host PID/IPC privileges", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("host_pid_ipc_privileges") result[:status].success?.should be_true (/(FAILED).*(Found containers with hostPID and hostIPC privileges)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'non_root_containers' should pass on a cnf that does not have containers running with root user or user with root group memberships", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-nonroot") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-nonroot") result = ShellCmd.run_testsuite("non_root_containers") result[:status].success?.should be_true (/(FAILED).*(Found containers running with root user or user with root group membership)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'non_root_containers' should fail on a cnf that has containers running with root user or user with root group memberships", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("non_root_containers") result[:status].success?.should be_true (/(PASSED).*(Containers are running with non-root user with non-root group membership)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'immutable_file_systems' should fail when the cnf containers with mutable file systems", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("immutable_file_systems") result[:status].success?.should be_true (/(PASSED).*(Containers have immutable file systems)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'immutable_file_systems' should pass when the cnf containers with immutable file systems", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-immutable-fs") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-immutable-fs") result = ShellCmd.run_testsuite("immutable_file_systems") result[:status].success?.should be_true (/(PASSED).*(Containers have immutable file systems)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'hostpath_mounts' should pass when the cnf has no containers with hostPath mounts", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf") ClusterTools.uninstall result = ShellCmd.run_testsuite("hostpath_mounts") result[:status].success?.should be_true (/(FAILED).*(Found containers with hostPath mounts)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() ClusterTools.install end end it "'hostpath_mounts' should fail when the cnf has containers with hostPath mounts", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-hostpath") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-hostpath") ClusterTools.uninstall result = ShellCmd.run_testsuite("hostpath_mounts") result[:status].success?.should be_true (/(FAILED).*(Found containers with hostPath mounts)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() ClusterTools.install end end it "'container_sock_mounts' should pass if a cnf has no pods that mount container engine socket", tags: ["container_sock_mounts"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("container_sock_mounts verbose") result[:status].success?.should be_true (/(PASSED).*(Container engine daemon sockets are not mounted as volumes)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'container_sock_mounts' should fail if the CNF has pods with container engine sockets mounted", tags: ["container_sock_mounts"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") result = ShellCmd.run_testsuite("container_sock_mounts verbose") result[:status].success?.should be_true (/(FAILED).*(Container engine daemon sockets are mounted as volumes)/ =~ result[:output]).should_not be_nil (/Unix socket is not allowed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'external_ips' should pass if a cnf has no services with external IPs", tags: ["external_ips"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("external_ips verbose") result[:status].success?.should be_true (/(PASSED).*(Services are not using external IPs)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'external_ips' should fail if a cnf has services with external IPs", tags: ["external_ips"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") result = ShellCmd.run_testsuite("external_ips verbose") result[:status].success?.should be_true (/(FAILED).*(Services are using external IPs)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'selinux_options' should fail if containers have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(FAILED).*(Pods are using custom SELinux options that can be used for privilege escalations)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'selinux_options' should be skipped if containers do not use custom selinux options", tags: ["selinux_options"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(N\/A).*(Pods are not using SELinux)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'selinux_options' should pass if containers do not have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_valid_selinux_options") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_valid_selinux_options") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(PASSED).*(Pods are not using custom SELinux options that can be used for privilege escalations)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'sysctls' should fail if Pods have restricted sysctls values", tags: ["sysctls"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_sysctls") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_sysctls") result = ShellCmd.run_testsuite("sysctls verbose") result[:status].success?.should be_true (/(FAILED).*(Restricted values for are being used for sysctls)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end it "'sysctls' should pass if Pods have allowed sysctls values", tags: ["sysctls"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("sysctls verbose") result[:status].success?.should be_true (/(PASSED).*(No restricted values found for sysctls)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() end end end diff --git a/spec/workload/state_spec.cr b/spec/workload/state_spec.cr index 2964ea623..5d8c389f8 100644 --- a/spec/workload/state_spec.cr +++ b/spec/workload/state_spec.cr @@ -14,11 +14,11 @@ describe "State" do it "'elastic_volumes' should fail if the cnf does not use volumes that are elastic volume", tags: ["elastic_volume"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(All used volumes are elastic)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -27,11 +27,11 @@ describe "State" do # This CNF does not use any volumes except the ones that Kubernetes might mount by default (like the service account token) it "'elastic_volumes' should fail if the cnf does not use any elastic volumes", tags: ["elastic_volume"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample_nonroot", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/FAILED/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -40,23 +40,23 @@ describe "State" do begin Log.info { "Installing Mysql " } # todo make helm directories work with parameters - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") result = ShellCmd.run_testsuite("database_persistence", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(CNF uses database with cloud-native persistence)/ =~ result[:output]).should_not be_nil ensure #todo fix cleanup for helm directory with parameters - ShellCmd.cnf_cleanup() + ShellCmd.cnf_uninstall() ShellCmd.run("kubectl delete pvc data-mysql-0", "delete_pvc") end end it "'elastic_volumes' should fail if the cnf doesn't use an elastic volume", tags: ["elastic_volume"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.cnf_install("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/(FAILED).*(Some of the used volumes are not elastic)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end @@ -66,11 +66,11 @@ describe "State" do # update the helm parameter with a schedulable node for the pv chart schedulable_nodes = KubectlClient::Get.schedulable_nodes update_yml("sample-cnfs/sample-local-storage/worker-node-value.yml", "worker_node", "#{schedulable_nodes[0]}") - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") (/(FAILED).*(local storage configuration volumes found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() update_yml("sample-cnfs/sample-local-storage/worker-node-value.yml", "worker_node", "") result[:status].success?.should be_true end @@ -78,11 +78,11 @@ describe "State" do it "'no_local_volume_configuration' should pass if local storage configuration is not found", tags: ["no_local_volume_configuration"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + ShellCmd.cnf_install("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") (/(PASSED).*(local storage configuration volumes not found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.cnf_cleanup() + result = ShellCmd.cnf_uninstall() result[:status].success?.should be_true end end diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr index f66b6cf6f..2388171d6 100644 --- a/src/tasks/cleanup.cr +++ b/src/tasks/cleanup.cr @@ -3,13 +3,13 @@ require "file_utils" require "colorize" require "totem" -desc "Cleans up the CNF test suite, the K8s cluster, and upstream projects" -task "cleanup", ["cnf_cleanup"] do |_, args| +desc "Alias for cnf_uninstall" +task "uninstall", ["cnf_uninstall"] do |_, args| end desc "Cleans up the CNF Test Suite helper tools and containers" -task "tools_cleanup", [ - "sonobuoy_cleanup", +task "tools_uninstall", [ + "uninstall_sonobuoy", "uninstall_chaosmesh", "uninstall_litmus", "uninstall_dockerd", @@ -19,15 +19,15 @@ task "tools_cleanup", [ # Helm needs to be uninstalled last to allow other uninstalls to use helm if necessary. # Check this issue for details - https://github.com/cncf/cnf-testsuite/issues/1586 - "helm_local_cleanup" + "uninstall_local_helm" ] do |_, args| end desc "Cleans up the CNF Test Suite sample projects, helper tools, and containers" -task "cleanup_all", ["uninstall_cnf", "tools_cleanup"] do |_, args| +task "uninstall_all", ["cnf_uninstall", "tools_uninstall"] do |_, args| end -task "results_yml_cleanup" do |_, args| +task "delete_results" do |_, args| if CNFManager::Points::Results.file_exists? File.delete(CNFManager::Points::Results.file) Log.for("verbose").info { "Deleted results file at #{CNFManager::Points::Results.file}" } if check_verbose(args) diff --git a/src/tasks/cluster_api_setup.cr b/src/tasks/cluster_api_setup.cr index d2d8e50c1..1b915c864 100644 --- a/src/tasks/cluster_api_setup.cr +++ b/src/tasks/cluster_api_setup.cr @@ -9,7 +9,7 @@ require "json" require "yaml" desc "Install Cluster API for Kind" -task "cluster_api_setup" do |_, args| +task "cluster_api_install" do |_, args| current_dir = FileUtils.pwd HttpHelper.download("https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/clusterctl-linux-amd64", "clusterctl") @@ -57,8 +57,8 @@ task "cluster_api_setup" do |_, args| Log.info { "cluster api setup complete" } end -desc "Cleanup Cluster API" -task "cluster_api_cleanup" do |_, args| +desc "Uninstall Cluster API" +task "cluster_api_uninstall" do |_, args| current_dir = FileUtils.pwd #KubectlClient::Delete.command("cluster capi-quickstart") delete_cluster_file = "#{current_dir}/capi.yaml" diff --git a/src/tasks/cluster_setup.cr b/src/tasks/cluster_setup.cr index 730708458..31300a69e 100644 --- a/src/tasks/cluster_setup.cr +++ b/src/tasks/cluster_setup.cr @@ -13,7 +13,7 @@ task "install_cluster_tools" do |_, args| ClusterTools.install rescue e : ClusterTools::NamespaceDoesNotExistException Log.error { "#{e.message}" } - stdout_failure "Error: Namespace cnf-testsuite does not exist.\nPlease run 'cnf-testsuite setup' to create the necessary namespace." + stdout_failure "Error: Namespace cnf-testsuite does not exist.\nPlease run 'cnf-testsuite install_dependencies' to create the necessary namespace." exit(1) end end @@ -24,7 +24,7 @@ task "uninstall_cluster_tools" do |_, args| ClusterTools.uninstall rescue e : ClusterTools::NamespaceDoesNotExistException Log.error { "#{e.message}" } - stdout_failure "Error: Namespace cnf-testsuite does not exist.\nPlease run 'cnf-testsuite setup' to create the necessary namespace." + stdout_failure "Error: Namespace cnf-testsuite does not exist.\nPlease run 'cnf-testsuite install_dependencies' to create the necessary namespace." exit(1) end end diff --git a/src/tasks/cnf_setup.cr b/src/tasks/cnf_setup.cr index d34949c3a..217e4fca2 100644 --- a/src/tasks/cnf_setup.cr +++ b/src/tasks/cnf_setup.cr @@ -9,10 +9,10 @@ task "generate_config" do |_, args| interactively_create_config() end -task "cnf_setup", ["helm_local_install", "create_namespace"] do |_, args| +task "cnf_install", ["helm_local_install", "create_namespace"] do |_, args| if CNFManager.cnf_installed? - stdout_warning "A CNF is already set up. Setting up multiple CNFs is not allowed." - stdout_warning "To set up a new CNF, uninstall the existing one by running: cnf_cleanup" + stdout_warning "A CNF is already installed. Installation of multiple CNFs is not allowed." + stdout_warning "To install a new CNF, uninstall the existing one by running: cnf_uninstall" exit 0 end if ClusterTools.install @@ -26,7 +26,7 @@ task "cnf_setup", ["helm_local_install", "create_namespace"] do |_, args| stdout_success "CNF installation complete." end -task "cnf_cleanup" do |_, args| +task "cnf_uninstall" do |_, args| CNFInstall.uninstall_cnf() end diff --git a/src/tasks/helmenv_setup.cr b/src/tasks/helmenv_setup.cr index 74b8c9da3..906e5808c 100644 --- a/src/tasks/helmenv_setup.cr +++ b/src/tasks/helmenv_setup.cr @@ -64,8 +64,8 @@ task "helm_local_install", ["cnf_directory_setup"] do |_, args| end desc "Cleans up helm 3.8.2" -task "helm_local_cleanup" do |_, args| - Log.for("verbose").info { "helm_local_cleanup" } if check_verbose(args) +task "uninstall_local_helm" do |_, args| + Log.for("verbose").info { "uninstall_local_helm" } if check_verbose(args) current_dir = FileUtils.pwd FileUtils.rm_rf("#{tools_path}/helm") end diff --git a/src/tasks/prereqs.cr b/src/tasks/prereqs.cr index e1e35bcec..cbdc5ca43 100644 --- a/src/tasks/prereqs.cr +++ b/src/tasks/prereqs.cr @@ -21,7 +21,7 @@ task "prereqs" do |_, args| ] if checks.includes?(false) - stdout_failure "Setup failed. Some prerequisites are missing. Please install all of the prerequisites before continuing." + stdout_failure "Dependency installation failed. Some prerequisites are missing. Please install all of the prerequisites before continuing." exit 1 else stdout_success "All prerequisites found." diff --git a/src/tasks/setup.cr b/src/tasks/setup.cr index 9a14fb222..5d73d1d5d 100644 --- a/src/tasks/setup.cr +++ b/src/tasks/setup.cr @@ -5,8 +5,8 @@ require "totem" desc "Sets up the CNF test suite, the K8s cluster, and upstream projects" -task "setup", ["version", "helm_local_install", "prereqs", "create_namespace", "configuration_file_setup", "install_apisnoop", "install_sonobuoy", "install_chart_testing", "cnf_testsuite_setup", "install_kind"] do |_, args| - stdout_success "Setup complete" +task "install_dependencies", ["version", "helm_local_install", "prereqs", "create_namespace", "configuration_file_setup", "install_apisnoop", "install_sonobuoy", "install_chart_testing", "cnf_testsuite_setup", "install_kind"] do |_, args| + stdout_success "Dependency installation complete" end task "create_namespace" do |_, args| diff --git a/src/tasks/sonobuoy_setup.cr b/src/tasks/sonobuoy_setup.cr index f2dc525f5..27d9217e5 100644 --- a/src/tasks/sonobuoy_setup.cr +++ b/src/tasks/sonobuoy_setup.cr @@ -46,8 +46,8 @@ task "install_sonobuoy" do |_, args| end end -desc "Cleans up Sonobuoy" -task "sonobuoy_cleanup" do |_, args| +desc "Uninstalls Sonobuoy" +task "uninstall_sonobuoy" do |_, args| current_dir = FileUtils.pwd sonobuoy = "#{tools_path}/sonobuoy/sonobuoy" status = Process.run( diff --git a/src/tasks/utils/cnf_installation/install_common.cr b/src/tasks/utils/cnf_installation/install_common.cr index f454d2df8..ec9868bf7 100644 --- a/src/tasks/utils/cnf_installation/install_common.cr +++ b/src/tasks/utils/cnf_installation/install_common.cr @@ -19,7 +19,7 @@ module CNFInstall end def self.parse_cli_args(cli_args) - Log.for("cnf_setup").debug { "cli_args = #{cli_args.inspect}" } + Log.for("cnf_install").debug { "cli_args = #{cli_args.inspect}" } cnf_config_path = "" timeout = 1800 skip_wait_for_install = cli_args.raw.includes? "skip_wait_for_install" @@ -35,7 +35,7 @@ module CNFInstall timeout = cli_args.named["timeout"].to_i end parsed_args = {config_path: cnf_config_path, timeout: timeout, skip_wait_for_install: skip_wait_for_install} - Log.for("cnf_setup").debug { "parsed_cli_args = #{parsed_args}"} + Log.for("cnf_install").debug { "parsed_cli_args = #{parsed_args}"} parsed_args end @@ -119,8 +119,8 @@ module CNFInstall stdout_success "Waiting for resource for \"#{deployment_name}\" deployment (#{current_resource_number}/#{total_resource_count}): [#{resource_info[:kind]}] #{resource_info[:name]}", same_line: true ready = KubectlClient::Get.resource_wait_for_install(resource_info[:kind], resource_info[:name], wait_count: timeout, namespace: resource_info[:namespace]) if !ready - stdout_failure "\"#{deployment_name}\" deployment setup has timed-out, [#{resource_info[:kind]}] #{resource_info[:name]} is not ready after #{timeout} seconds.", same_line: true - stdout_failure "Recommended course of actions would be to investigate the resource in cluster, then call cnf_cleanup and try to reinstall the CNF." + stdout_failure "\"#{deployment_name}\" deployment installation has timed-out, [#{resource_info[:kind]}] #{resource_info[:name]} is not ready after #{timeout} seconds.", same_line: true + stdout_failure "Recommended course of actions would be to investigate the resource in cluster, then call cnf_uninstall and try to reinstall the CNF." exit 1 end current_resource_number += 1 diff --git a/src/tasks/utils/cnf_manager.cr b/src/tasks/utils/cnf_manager.cr index e93f2f5e8..bf21b68d5 100644 --- a/src/tasks/utils/cnf_manager.cr +++ b/src/tasks/utils/cnf_manager.cr @@ -131,7 +131,7 @@ module CNFManager cnf_testsuite = find_stdout.to_s.split("\n").select{ |x| x.empty? == false } Log.info { "find response: #{cnf_testsuite}" } if cnf_testsuite.size == 0 && !silent - raise "No cnf_testsuite.yml found! Did you run the setup task?" + raise "No cnf_testsuite.yml found! Did you run the installation task?" end cnf_testsuite end diff --git a/src/tasks/workload/compatibility.cr b/src/tasks/workload/compatibility.cr index 7c29aa433..30603a000 100644 --- a/src/tasks/workload/compatibility.cr +++ b/src/tasks/workload/compatibility.cr @@ -397,7 +397,7 @@ task "helm_deploy" do |t, args| CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "CNF has deployments that are not installed with helm") end else - CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "No cnf_testsuite.yml found! Did you run the setup task?") + CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "No cnf_testsuite.yml found! Did you run the installation task?") end end end diff --git a/src/tasks/workload/observability.cr b/src/tasks/workload/observability.cr index 6d10a7d58..3e0747f42 100644 --- a/src/tasks/workload/observability.cr +++ b/src/tasks/workload/observability.cr @@ -224,7 +224,7 @@ task "tracing" do |t, args| CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Skipped, "Jaeger not configured") end else - CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "No cnf_testsuite.yml found! Did you run the setup task?") + CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "No cnf_testsuite.yml found! Did you run the installation task?") end end end diff --git a/tools/curl_install_tester_docker_setup/Dockerfile b/tools/curl_install_tester_docker_setup/Dockerfile index 1c58b0be5..1ea5b2e81 100644 --- a/tools/curl_install_tester_docker_setup/Dockerfile +++ b/tools/curl_install_tester_docker_setup/Dockerfile @@ -19,6 +19,6 @@ WORKDIR /test_workspace from base as ci_test # add binary AND ONLY the binary in this container so we can fail if binary is broken without the rest of the source COPY cnf-testsuite . -RUN ./cnf-testsuite setup verbose +RUN ./cnf-testsuite install_dependencies verbose RUN wget https://raw.githubusercontent.com/cnti-testcatalog/testsuite/main/example-cnfs/coredns/cnf-testsuite.yml RUN ./cnf-testsuite helm_chart_valid cnf-config=./cnf-testsuite.yml verbose diff --git a/tools/curl_install_tester_docker_setup/README.md b/tools/curl_install_tester_docker_setup/README.md index 77abb4bde..01b08d52e 100644 --- a/tools/curl_install_tester_docker_setup/README.md +++ b/tools/curl_install_tester_docker_setup/README.md @@ -21,11 +21,11 @@ source <(curl https://raw.githubusercontent.com/cnti-testcatalog/testsuite/main/ cd # to make sure you are in home dir -./cnf-testsuite setup +./cnf-testsuite install_dependencies wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/testsuite/main/example-cnfs/coredns/cnf-testsuite.yml -./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml +./cnf-testsuite cnf_install cnf-config=./cnf-testsuite.yml ```