diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 27aae9f5e..a5562d846 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -478,7 +478,7 @@ jobs: ./cnf-testsuite setup wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/testsuite/${GITHUB_SHA}/example-cnfs/coredns/cnf-testsuite.yml ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml - LOG_LEVEL=info ./cnf-testsuite all ~resilience ~compatibility ~pod_network_latency ~platform ~increase_capacity ~decrease_capacity ~ip_addresses ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~install_script_helm ~helm_chart_valid ~helm_chart_published ~rollback ~secrets_used ~immutable_configmap verbose + LOG_LEVEL=info ./cnf-testsuite all ~resilience ~compatibility ~pod_network_latency ~platform ~increase_capacity ~decrease_capacity ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~install_script_helm ~helm_chart_valid ~helm_chart_published ~rollback ~secrets_used ~immutable_configmap verbose - name: Delete Cluster if: ${{ always() }} run: | @@ -553,7 +553,7 @@ jobs: ./cnf-testsuite setup wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cnti-testcatalog/testsuite/${GITHUB_SHA}/example-cnfs/coredns/cnf-testsuite.yml ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml - LOG_LEVEL=info ./cnf-testsuite all ~resilience ~platform ~ip_addresses ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~rollback ~secrets_used ~immutable_configmap ~reasonable_startup_time ~reasonable_image_size verbose + LOG_LEVEL=info ./cnf-testsuite all ~resilience ~platform ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~rollback ~secrets_used ~immutable_configmap ~reasonable_startup_time ~reasonable_image_size verbose - name: Delete Cluster if: ${{ always() }} run: | diff --git a/spec/workload/configuration_spec.cr b/spec/workload/configuration_spec.cr index 51b239105..859d97840 100644 --- a/spec/workload/configuration_spec.cr +++ b/spec/workload/configuration_spec.cr @@ -193,17 +193,6 @@ describe CnfTestSuite do end end - it "'ip_addresses' should pass when no uncommented ip addresses are found in helm chart source", tags: ["ip_addresses"] do - begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf-source/cnf-testsuite.yml verbose skip_wait_for_install") - result = ShellCmd.run_testsuite("ip_addresses verbose") - result[:status].success?.should be_true - (/(PASSED).*(No IP addresses found)/ =~ result[:output]).should_not be_nil - ensure - result = ShellCmd.run_testsuite("sample_coredns_source_cleanup verbose") - end - end - it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should fail when a hardcoded ip is found in the K8s configuration", tags: ["ip_addresses"] do begin ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false") diff --git a/src/tasks/static.cr b/src/tasks/static.cr index 0b4a41c5d..64afe506e 100644 --- a/src/tasks/static.cr +++ b/src/tasks/static.cr @@ -4,6 +4,6 @@ require "colorize" require "totem" desc "Static tests" -task "static", ["liveness", "ip_addresses"] do |_, args| +task "static", ["liveness"] do |_, args| end diff --git a/src/tasks/workload/configuration.cr b/src/tasks/workload/configuration.cr index 5d793181a..3a5c752a3 100644 --- a/src/tasks/workload/configuration.cr +++ b/src/tasks/workload/configuration.cr @@ -11,7 +11,6 @@ rolling_version_change_test_names = ["rolling_update", "rolling_downgrade", "rol desc "Configuration should be managed in a declarative manner, using ConfigMaps, Operators, or other declarative interfaces." task "configuration", [ - "ip_addresses", "nodeport_not_used", "hostport_not_used", "hardcoded_ip_addresses_in_k8s_runtime_configuration", @@ -101,50 +100,6 @@ task "latest_tag" do |t, args| end end -desc "Does a search for IP addresses or subnets come back as negative?" -task "ip_addresses" do |t, args| - CNFManager::Task.task_runner(args, task: t) do |args, config| - cdir = FileUtils.pwd() - response = String::Builder.new - helm_chart_path = CNFInstall::Config.get_helm_chart_path(config) - Log.info { "Path: #{helm_chart_path}" } - if File.directory?(helm_chart_path) - # Switch to the helm chart directory - Dir.cd(helm_chart_path) - # Look for all ip addresses that are not comments - Log.for(t.name).info { "current directory: #{ FileUtils.pwd()}" } - # should catch comments (# // or /*) and ignore 0.0.0.0 - # note: grep wants * escaped twice - Process.run("grep -r -P '^(?!.+0\.0\.0\.0)(?![[:space:]]*0\.0\.0\.0)(?!#)(?![[:space:]]*#)(?!\/\/)(?![[:space:]]*\/\/)(?!\/\\*)(?![[:space:]]*\/\\*)(.+([0-9]{1,3}[\.]){3}[0-9]{1,3})' --exclude=*.txt", shell: true) do |proc| - while line = proc.output.gets - response << line - VERBOSE_LOGGING.info "#{line}" if check_verbose(args) - end - end - Dir.cd(cdir) - parsed_resp = response.to_s - if parsed_resp.size > 0 - response_lines = parsed_resp.split("\n") - stdout_failure("Lines with hard-coded IP addresses:") - response_lines.each do |line| - line_parts = line.split(":") - file_name = line_parts.shift() - matching_line = line_parts.join(":").strip() - stdout_failure(" * In file #{file_name}: #{matching_line}") - end - CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "IP addresses found") - else - CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "No IP addresses found") - end - else - # TODO If no helm chart directory, exit with 0 points - # ADD SKIPPED tag for points.yml to allow for 0 points - Dir.cd(cdir) - CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "No IP addresses found") - end - end -end - desc "Do all cnf images have versioned tags?" task "versioned_tag", ["install_opa"] do |t, args| CNFManager::Task.task_runner(args, task: t) do |args,config| @@ -261,22 +216,13 @@ task "hardcoded_ip_addresses_in_k8s_runtime_configuration" do |t, args| helm_chart = config.deployments.get_deployment_param(:helm_chart) helm_directory = config.deployments.get_deployment_param(:helm_directory) destination_cnf_dir = config.dynamic.destination_cnf_dir - helm_chart_yml_path = "#{destination_cnf_dir}/helm_chart.yml" current_dir = FileUtils.pwd helm = Helm::BinarySingleton.helm VERBOSE_LOGGING.info "Helm Path: #{helm}" if check_verbose(args) - KubectlClient::Create.command("namespace hardcoded-ip-test") - unless helm_chart.empty? - helm_install = Helm.install("--namespace hardcoded-ip-test hardcoded-ip-test #{helm_chart} --dry-run --debug > #{helm_chart_yml_path}") - else - helm_install = Helm.install("--namespace hardcoded-ip-test hardcoded-ip-test #{destination_cnf_dir}/#{helm_directory} --dry-run --debug > #{helm_chart_yml_path}") - VERBOSE_LOGGING.info "helm_directory: #{helm_directory}" if check_verbose(args) - end - found_violations = [] of NamedTuple(line_number: Int32, line: String) line_number = 1 - File.open("#{helm_chart_yml_path}") do |file| + File.open(COMMON_MANIFEST_FILE_PATH) do |file| file.each_line do |line| if line.matches?(/NOTES:/) break @@ -294,7 +240,7 @@ task "hardcoded_ip_addresses_in_k8s_runtime_configuration" do |t, args| if found_violations.empty? CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "No hard-coded IP addresses found in the runtime K8s configuration") else - stdout_failure("Hard-coded IP addresses found in #{helm_chart_yml_path}") + stdout_failure("Hard-coded IP addresses found in #{COMMON_MANIFEST_FILE_PATH}") found_violations.each do |violation| stdout_failure(" * Line #{violation[:line_number]}: #{violation[:line]}") end diff --git a/src/tasks/workload/observability.cr b/src/tasks/workload/observability.cr index 3e53db9bb..3d365158a 100644 --- a/src/tasks/workload/observability.cr +++ b/src/tasks/workload/observability.cr @@ -42,7 +42,6 @@ end desc "Does the CNF emit prometheus traffic" task "prometheus_traffic" do |t, args| task_response = CNFManager::Task.task_runner(args, task: t) do |args, config| - release_name = config.deployments.get_deployment_param(:name) destination_cnf_dir = config.dynamic.destination_cnf_dir do_this_on_each_retry = ->(ex : Exception, attempt : Int32, elapsed_time : Time::Span, next_interval : Time::Span) do @@ -112,7 +111,7 @@ task "prometheus_traffic" do |t, args| end if msg[:status].success? metrics_config_map = Prometheus::OpenMetricConfigMapTemplate.new( - "cnf-testsuite-#{release_name}-open-metrics", + "cnf-testsuite-open-metrics", true, "", immutable_configmap @@ -120,7 +119,7 @@ task "prometheus_traffic" do |t, args| else Log.info { "Openmetrics failure reason: #{msg[:output]}"} metrics_config_map = Prometheus::OpenMetricConfigMapTemplate.new( - "cnf-testsuite-#{release_name}-open-metrics", + "cnf-testsuite-open-metrics", false, msg[:output], immutable_configmap @@ -159,8 +158,7 @@ end desc "Does the CNF emit prometheus open metric compatible traffic" task "open_metrics", ["prometheus_traffic"] do |t, args| task_response = CNFManager::Task.task_runner(args, task: t) do |args, config| - release_name = config.deployments.get_deployment_param(:name) - configmap = KubectlClient::Get.configmap("cnf-testsuite-#{release_name}-open-metrics") + configmap = KubectlClient::Get.configmap("cnf-testsuite-open-metrics") if configmap != EMPTY_JSON open_metrics_validated = configmap["data"].as_h["open_metrics_validated"].as_s diff --git a/tools/curl_install_tester_docker_setup/Dockerfile b/tools/curl_install_tester_docker_setup/Dockerfile index 21f0c33e3..1c58b0be5 100644 --- a/tools/curl_install_tester_docker_setup/Dockerfile +++ b/tools/curl_install_tester_docker_setup/Dockerfile @@ -21,4 +21,4 @@ from base as ci_test COPY cnf-testsuite . RUN ./cnf-testsuite setup verbose RUN wget https://raw.githubusercontent.com/cnti-testcatalog/testsuite/main/example-cnfs/coredns/cnf-testsuite.yml -RUN ./cnf-testsuite ip_addresses cnf-config=./cnf-testsuite.yml verbose +RUN ./cnf-testsuite helm_chart_valid cnf-config=./cnf-testsuite.yml verbose