From d2b191e1672c06d9321e5dc15f4c3ca574e1601e Mon Sep 17 00:00:00 2001 From: Konstantin Yarovoy Date: Thu, 15 Aug 2024 08:00:19 +0000 Subject: [PATCH] config: Switch to usage of new config Adapt all usages of config to the new format. Remove code that was used by old config. Update all sample and example configs. Refs: #2135 Signed-off-by: Konstantin Yarovoy --- spec/utils/cnf_manager_spec.cr | 59 +++++---------------- src/tasks/setup.cr | 10 ++-- src/tasks/utils/cnf_manager.cr | 80 +++++------------------------ src/tasks/workload/compatibility.cr | 11 ---- 4 files changed, 30 insertions(+), 130 deletions(-) diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr index 918aeca9f..c342e9da2 100644 --- a/spec/utils/cnf_manager_spec.cr +++ b/spec/utils/cnf_manager_spec.cr @@ -291,72 +291,37 @@ describe "SampleUtils" do CNFManager.helm_repo_add("invalid", "invalid").should eq(false) end - it "'CNFManager.validate_cnf_testsuite_yml' (function) should pass, when a cnf has a valid config file yml", tags: ["validate_config"] do - args = Sam::Args.new(["cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml"]) - - yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - Log.info { yml.inspect } - ("#{yml.get("release_name").as_s?}").should eq("coredns") - - valid, command_output = CNFManager.validate_cnf_testsuite_yml(yml) - - (valid).should eq(true) - (command_output).should eq (nil) - end - - it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, when a cnf has a valid config file yml", tags: ["validate_config"] do - result = ShellCmd.run_testsuite("validate_config cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true - (/CNF configuration validated/ =~ result[:output]).should_not be_nil - end - - - it "'CNFManager.validate_cnf_testsuite_yml' (function) should warn, but be valid when a cnf config file yml has fields that are not a part of the validation type", tags: ["validate_config"] do - args = Sam::Args.new(["cnf-config=./spec/fixtures/cnf-testsuite-unmapped-keys-and-subkeys.yml"]) - - yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - Log.info { yml.inspect } - ("#{yml.get("release_name").as_s?}").should eq("coredns") - - status, warning_output = CNFManager.validate_cnf_testsuite_yml(yml) - - Log.warn { "WARNING: #{warning_output}" } - - (status).should eq(true) - (warning_output).should_not be_nil - end - - - it "'CNFManager.validate_cnf_testsuite_yml' (command) should warn, but be valid when a cnf config file yml has fields that are not a part of the validation type", tags: ["validate_config"] do - result = ShellCmd.run_testsuite("validate_config cnf-config=spec/fixtures/cnf-testsuite-unmapped-keys-and-subkeys.yml") + it "'validate_config' should pass, when a cnf has a valid config file yml", tags: ["validate_config"] do + result = ShellCmd.run_testsuite("validate_config cnf-config=embedded_files/cnf-testsuite-v2-example.yml") result[:status].success?.should be_true - Log.debug { "validate_config resp: #{result[:output]}" } - (/CNF configuration validated/ =~ result[:output]).should_not be_nil + (/Successfully validated CNF config/ =~ result[:output]).should_not be_nil end - - it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, for all sample-cnfs", tags: ["validate_config"] do + it "'validate_config' should pass, for all sample-cnfs", tags: ["validate_config"] do get_dirs = Dir.entries("sample-cnfs") dir_list = get_dirs - [".", ".."] dir_list.each do |dir| testsuite_yml = "sample-cnfs/#{dir}/cnf-testsuite.yml" result = ShellCmd.run_testsuite("validate_config cnf-config=#{testsuite_yml}") - (/CNF configuration validated/ =~ result[:output]).should_not be_nil + unless result[:status].success? + Log.info {"Could not validate config: #{testsuite_yml}"} + end + (/Successfully validated CNF config/ =~ result[:output]).should_not be_nil end end - it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, for all example-cnfs", tags: ["validate_config"] do + it "'validate_config' should pass, for all example-cnfs", tags: ["validate_config"] do get_dirs = Dir.entries("example-cnfs") dir_list = get_dirs - [".", ".."] dir_list.each do |dir| testsuite_yml = "example-cnfs/#{dir}/cnf-testsuite.yml" result = ShellCmd.run_testsuite("validate_config cnf-config=#{testsuite_yml}") - if (/Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file/ =~ result[:output]) - Log.info { "\n #{testsuite_yml}: #{result[:output]}" } + unless result[:status].success? + Log.info {"Could not validate config: #{testsuite_yml}"} end - (/CNF configuration validated/ =~ result[:output]).should_not be_nil + (/Successfully validated CNF config/ =~ result[:output]).should_not be_nil end end diff --git a/src/tasks/setup.cr b/src/tasks/setup.cr index d819a7e19..9a14fb222 100644 --- a/src/tasks/setup.cr +++ b/src/tasks/setup.cr @@ -25,11 +25,13 @@ task "configuration_file_setup" do |_, args| CNFManager::Points.create_points_yml end -task "test_config" do |_, args| - if args.named["cfg"]? - puts CNFInstall::Config.parse_cnf_config_from_file(args.named["cfg"].to_s).inspect +task "validate_config" do |_, args| + if args.named["cnf-config"]? + config = CNFInstall::Config.parse_cnf_config_from_file(args.named["cnf-config"].to_s) + puts "Successfully validated CNF config" + Log.info {"Config: #{config.inspect}"} else - stdout_failure "cfg parameter needed" + stdout_failure "cnf-config parameter needed" exit 1 end end diff --git a/src/tasks/utils/cnf_manager.cr b/src/tasks/utils/cnf_manager.cr index 698aa9c5c..821646372 100644 --- a/src/tasks/utils/cnf_manager.cr +++ b/src/tasks/utils/cnf_manager.cr @@ -27,79 +27,25 @@ module CNFManager ECR.def_to_s("src/templates/elapsed_time_configmap.yml.ecr") end - # TODO: figure out recursively check for unmapped json and warn on that - # https://github.com/Nicolab/crystal-validator#check - def self.validate_cnf_testsuite_yml(config) - ccyt_validator = nil - valid = true - - begin - ccyt_validator = CnfTestSuiteYmlType.from_json(config.settings.to_json) - rescue ex - valid = false - Log.error { "✖ ERROR: cnf_testsuite.yml field validation error.".colorize(:red) } - Log.error { " please check info in the the field name near the text 'CnfTestSuiteYmlType#' in the error below".colorize(:red) } - Log.error { ex.message } - ex.backtrace.each do |x| - Log.error { x } - end - end - - unmapped_keys_warning_msg = "WARNING: Unmapped cnf_testsuite.yml keys. Please add them to the validator".colorize(:yellow) - unmapped_subkeys_warning_msg = "WARNING: helm_repository is unset or has unmapped subkeys. Please update your cnf_testsuite.yml".colorize(:yellow) - - if ccyt_validator && !ccyt_validator.try &.json_unmapped.empty? - warning_output = [unmapped_keys_warning_msg] of String | Colorize::Object(String) - warning_output.push(ccyt_validator.try &.json_unmapped.to_s) - if warning_output.size > 1 - Log.warn { warning_output.join("\n") } - end - end - - #TODO Differentiate between unmapped subkeys or unset top level key. - if ccyt_validator && !ccyt_validator.try &.helm_repository.try &.json_unmapped.empty? - root = {} of String => (Hash(String, JSON::Any) | Nil) - root["helm_repository"] = ccyt_validator.try &.helm_repository.try &.json_unmapped - - warning_output = [unmapped_subkeys_warning_msg] of String | Colorize::Object(String) - warning_output.push(root.to_s) - if warning_output.size > 1 - Log.warn { warning_output.join("\n") } - end - end - - { valid, warning_output } - end - def self.cnf_resource_ymls(args, config) Log.info { "cnf_resource_ymls" } - destination_cnf_dir = config.cnf_config[:destination_cnf_dir] - helm_directory = sandbox_helm_directory(config.cnf_config[:helm_directory]) - manifest_directory = config.cnf_config[:manifest_directory] - release_name = config.cnf_config[:release_name] - helm_chart_path = Config.get_helm_chart_path(config) - manifest_file_path = Config.get_manifest_file_path(config) - helm_values = config.cnf_config[:helm_values] - test_passed = true - - deployment_namespace = CNFManager.get_deployment_namespace(config) - install_method = config.cnf_config[:install_method] - Log.debug { "install_method: #{install_method}" } template_ymls = [] of YAML::Any - case install_method[0] + case config.dynamic.install_method[0] when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory + helm_chart_path = CNFInstall::Config.get_helm_chart_path(config) + generated_manifest_file_path = CNFInstall::Config.get_manifest_file_path(config) Log.info { "EXPORTED CHART PATH: #{helm_chart_path}" } - Helm.generate_manifest_from_templates(release_name, - helm_chart_path, - manifest_file_path, - deployment_namespace, - helm_values) - template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(manifest_file_path) + Helm.generate_manifest_from_templates(release_name: config.deployments.get_deployment_param(:name), + helm_chart: helm_chart_path, + output_file: generated_manifest_file_path, + namespace: CNFManager.get_deployment_namespace(config), + helm_values: config.deployments.get_deployment_param(:helm_values)) + template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(generated_manifest_file_path) when CNFInstall::InstallMethod::ManifestDirectory - # if release_name.empty? # no helm chart - template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list(CNFInstall::Manifest.manifest_file_list( destination_cnf_dir + "/" + manifest_directory)) - # else + template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list( + CNFInstall::Manifest.manifest_file_list(config.dynamic.destination_cnf_dir + "/" + config.deployments.get_deployment_param(:manifest_directory)) + ) end template_ymls = template_ymls.reject! {|x| @@ -886,8 +832,6 @@ module CNFManager end def self.cnf_to_new_cluster(config, kubeconfig) - release_name = config.cnf_config[:release_name] - install_method = config.cnf_config[:install_method] release_name = config.cnf_config[:release_name] install_method = config.cnf_config[:install_method] helm_directory = config.cnf_config[:helm_directory] diff --git a/src/tasks/workload/compatibility.cr b/src/tasks/workload/compatibility.cr index e4a46b2cb..58e1cbaa9 100644 --- a/src/tasks/workload/compatibility.cr +++ b/src/tasks/workload/compatibility.cr @@ -495,17 +495,6 @@ task "helm_chart_valid", ["helm_local_install"] do |t, args| end end -task "validate_config" do |_, args| - yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - valid, warning_output = CNFManager.validate_cnf_testsuite_yml(yml) - emoji_config="📋" - if valid - stdout_success "CNF configuration validated #{emoji_config}" - else - stdout_failure "Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file #{emoji_config}" - end -end - def setup_calico_cluster(cluster_name : String) : KindManager::Cluster Log.info { "Running cni_compatible(Cluster Creation)" } Helm.helm_repo_add("projectcalico","https://docs.projectcalico.org/charts")