From e232602ed9219662fb2944abaaab07f51806989d Mon Sep 17 00:00:00 2001 From: Konstantin Yarovoy Date: Thu, 15 Aug 2024 08:00:19 +0000 Subject: [PATCH] config: Switch to usage of new config Adapt all usages of config to the new format. Remove code that was used by old config. Update all sample and example configs. Refs: #2135 Signed-off-by: Konstantin Yarovoy --- spec/setup_spec.cr | 45 ---- spec/utils/cnf_install/install_common_spec.cr | 7 - spec/utils/cnf_manager_spec.cr | 105 +++----- spec/utils/utils_spec.cr | 4 +- src/tasks/cleanup.cr | 4 +- src/tasks/cnf_setup.cr | 22 +- src/tasks/setup.cr | 10 +- src/tasks/ueransim_setup.cr | 2 +- .../utils/cnf_installation/install_common.cr | 108 -------- src/tasks/utils/cnf_manager.cr | 247 +++--------------- src/tasks/utils/config.cr | 154 ----------- src/tasks/utils/generate_config.cr | 128 --------- src/tasks/utils/task.cr | 48 +--- src/tasks/workload/compatibility.cr | 12 - src/tasks/workload/microservice.cr | 2 - src/tasks/workload/state.cr | 2 +- 16 files changed, 92 insertions(+), 808 deletions(-) delete mode 100644 spec/utils/cnf_install/install_common_spec.cr delete mode 100644 src/tasks/utils/generate_config.cr diff --git a/spec/setup_spec.cr b/spec/setup_spec.cr index 4bd4de634..016f4e03b 100644 --- a/spec/setup_spec.cr +++ b/spec/setup_spec.cr @@ -19,51 +19,6 @@ describe "Setup" do (/Setup complete/ =~ result[:output]).should_not be_nil end - it "'generate_config' should generate a cnf-testsuite.yml for a helm chart", tags: ["setup-generate"] do - result = ShellCmd.run_testsuite("setup") - result = ShellCmd.run_testsuite("generate_config config-src=stable/coredns output-file=./cnf-testsuite-test.yml") - result[:status].success?.should be_true - - yaml = File.open("./cnf-testsuite-test.yml") do |file| - YAML.parse(file) - end - Log.debug { "test yaml: #{yaml}" } - result = ShellCmd.run("cat ./cnf-testsuite-test.yml", force_output: true) - (yaml["helm_chart"] == "stable/coredns").should be_true - ensure - result = ShellCmd.run("rm ./cnf-testsuite-test.yml", force_output: true) - end - - it "'generate_config' should generate a cnf-testsuite.yml for a helm directory", tags: ["setup-generate"] do - result = ShellCmd.run_testsuite("setup") - result = ShellCmd.run_testsuite("generate_config config-src=sample-cnfs/k8s-sidecar-container-pattern/chart output-file=./cnf-testsuite-test.yml") - result[:status].success?.should be_true - - yaml = File.open("./cnf-testsuite-test.yml") do |file| - YAML.parse(file) - end - Log.debug { "test yaml: #{yaml}" } - result = ShellCmd.run("cat ./cnf-testsuite-test.yml", force_output: true) - (yaml["helm_directory"] == "sample-cnfs/k8s-sidecar-container-pattern/chart").should be_true - ensure - result = ShellCmd.run("rm ./cnf-testsuite-test.yml", force_output: true) - end - - it "'generate_config' should generate a cnf-testsuite.yml for a manifest directory", tags: ["setup-generate"] do - result = ShellCmd.run_testsuite("setup") - result = ShellCmd.run_testsuite("generate_config config-src=sample-cnfs/k8s-non-helm/manifests output-file=./cnf-testsuite-test.yml") - result[:status].success?.should be_true - - yaml = File.open("./cnf-testsuite-test.yml") do |file| - YAML.parse(file) - end - Log.debug { "test yaml: #{yaml}" } - result = ShellCmd.run("cat ./cnf-testsuite-test.yml", force_output: true) - (yaml["manifest_directory"] == "sample-cnfs/k8s-non-helm/manifests").should be_true - ensure - result = ShellCmd.run("rm ./cnf-testsuite-test.yml", force_output: true) - end - it "'cnf_setup/cnf_cleanup' should install/cleanup with cnf-path arg as alias for cnf-config", tags: ["setup"] do begin result = ShellCmd.cnf_setup("cnf-path=example-cnfs/coredns/cnf-testsuite.yml") diff --git a/spec/utils/cnf_install/install_common_spec.cr b/spec/utils/cnf_install/install_common_spec.cr deleted file mode 100644 index 14ad8aa1e..000000000 --- a/spec/utils/cnf_install/install_common_spec.cr +++ /dev/null @@ -1,7 +0,0 @@ -require "../../spec_helper" - -it "'CNFInstall.exclusive_install_method_tags' should return false if install method tags are not exclusive", tags: ["cnf-config"] do - config = CNFManager.parsed_config_file("./spec/fixtures/cnf-testsuite-not-exclusive.yml") - resp = CNFInstall.exclusive_install_method_tags?(config) - (resp).should be_false -end \ No newline at end of file diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr index 918aeca9f..af06b4d81 100644 --- a/spec/utils/cnf_manager_spec.cr +++ b/spec/utils/cnf_manager_spec.cr @@ -24,11 +24,6 @@ describe "SampleUtils" do result[:status].success?.should be_true end - it "'images_from_config_src' should return a list of containers for a cnf", tags: ["cnf-setup"] do - (CNFManager::GenerateConfig.images_from_config_src("stable/coredns").find {|x| x[:image_name] =="coredns/coredns" && - x[:container_name] =="coredns"}).should be_truthy - end - it "'cnf_setup' should pass with a minimal cnf-testsuite.yml", tags: ["cnf-setup"] do ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install") ensure @@ -161,8 +156,8 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml", "verbose"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(cli_hash[:config_file])) - release_name = config.cnf_config[:release_name] + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(cli_hash[:config_file])) + release_name = config.deployments.get_deployment_param(:name) (Dir.exists? "cnfs/#{release_name}").should be_true (File.exists?("cnfs/#{release_name}/cnf-testsuite.yml")).should be_true @@ -177,8 +172,8 @@ describe "SampleUtils" do cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) # check if directory exists - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - release_name = config.cnf_config[:release_name] + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + release_name = config.deployments.get_deployment_param(:name) (Dir.exists? "cnfs/#{release_name}").should be_true (File.exists?("cnfs/#{release_name}/cnf-testsuite.yml")).should be_true @@ -193,8 +188,8 @@ describe "SampleUtils" do cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) # check if directory exists - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - release_name = config.cnf_config[:release_name] + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + release_name = config.deployments.get_deployment_param(:name) (Dir.exists? "cnfs/#{release_name}").should be_true (File.exists?("cnfs/#{release_name}/cnf-testsuite.yml")).should be_true CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-generic-cnf", verbose: true) @@ -207,8 +202,8 @@ describe "SampleUtils" do cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) # check if directory exists - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - release_name = config.cnf_config[:release_name] + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + release_name = config.deployments.get_deployment_param(:name) (Dir.exists? "sample-cnfs/sample-generic-cnf").should be_true (File.exists?("cnfs/#{release_name}/cnf-testsuite.yml")).should be_true CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-generic-cnf", verbose: true) @@ -231,8 +226,8 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - release_name = config.cnf_config[:release_name] + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + release_name = config.deployments.get_deployment_param(:name) (Dir.exists? "cnfs/#{release_name}").should be_true # should not clone (Dir.exists? "cnfs/#{release_name}/privileged-coredns").should be_false @@ -249,8 +244,8 @@ describe "SampleUtils" do Log.info { "Running Setup" } CNFManager.sample_setup(cli_hash) Log.info { "Parse Config" } - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - release_name = config.cnf_config[:release_name] + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + release_name = CNFInstall::Config.parse_cnf_config_from_file (Dir.exists? "cnfs/#{release_name}").should be_true (Dir.exists? "cnfs/#{release_name}/manifests").should be_true (File.exists? "cnfs/#{release_name}/cnf-testsuite.yml").should be_true @@ -261,11 +256,6 @@ describe "SampleUtils" do (Dir.exists? "cnfs/#{release_name}").should be_false end - it "'cnf_destination_dir' should return the full path of the potential destination cnf directory based on the deployment name", tags: "WIP" do - args = Sam::Args.new - CNFManager.cnf_destination_dir("spec/fixtures/cnf-testsuite.yml").should contain("/cnfs/coredns") - end - it "'CNFManager.cnf_config_list' should return a list of all of the config files from the cnf directory", tags: ["cnf-setup"] do config_file = "sample-cnfs/sample-generic-cnf" args = Sam::Args.new(["cnf-config=./#{config_file}/cnf-testsuite.yml", "verbose", "skip_wait_for_install"]) @@ -274,8 +264,8 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml", "verbose"]) cli_hash = CNFManager.sample_setup_cli_args(args) CNFManager.sample_setup(cli_hash) - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - release_name = config.cnf_config[:release_name] + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + release_name = CNFInstall::Config.parse_cnf_config_from_file CNFManager.cnf_config_list()[0].should contain("#{release_name}/#{CONFIG_FILE}") end @@ -291,80 +281,45 @@ describe "SampleUtils" do CNFManager.helm_repo_add("invalid", "invalid").should eq(false) end - it "'CNFManager.validate_cnf_testsuite_yml' (function) should pass, when a cnf has a valid config file yml", tags: ["validate_config"] do - args = Sam::Args.new(["cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml"]) - - yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - Log.info { yml.inspect } - ("#{yml.get("release_name").as_s?}").should eq("coredns") - - valid, command_output = CNFManager.validate_cnf_testsuite_yml(yml) - - (valid).should eq(true) - (command_output).should eq (nil) - end - - it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, when a cnf has a valid config file yml", tags: ["validate_config"] do - result = ShellCmd.run_testsuite("validate_config cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true - (/CNF configuration validated/ =~ result[:output]).should_not be_nil - end - - - it "'CNFManager.validate_cnf_testsuite_yml' (function) should warn, but be valid when a cnf config file yml has fields that are not a part of the validation type", tags: ["validate_config"] do - args = Sam::Args.new(["cnf-config=./spec/fixtures/cnf-testsuite-unmapped-keys-and-subkeys.yml"]) - - yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - Log.info { yml.inspect } - ("#{yml.get("release_name").as_s?}").should eq("coredns") - - status, warning_output = CNFManager.validate_cnf_testsuite_yml(yml) - - Log.warn { "WARNING: #{warning_output}" } - - (status).should eq(true) - (warning_output).should_not be_nil - end - - - it "'CNFManager.validate_cnf_testsuite_yml' (command) should warn, but be valid when a cnf config file yml has fields that are not a part of the validation type", tags: ["validate_config"] do - result = ShellCmd.run_testsuite("validate_config cnf-config=spec/fixtures/cnf-testsuite-unmapped-keys-and-subkeys.yml") + it "'validate_config' should pass, when a cnf has a valid config file yml", tags: ["validate_config"] do + result = ShellCmd.run_testsuite("validate_config cnf-config=embedded_files/cnf-testsuite-v2-example.yml") result[:status].success?.should be_true - Log.debug { "validate_config resp: #{result[:output]}" } - (/CNF configuration validated/ =~ result[:output]).should_not be_nil + (/Successfully validated CNF config/ =~ result[:output]).should_not be_nil end - - it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, for all sample-cnfs", tags: ["validate_config"] do + it "'validate_config' should pass, for all sample-cnfs", tags: ["validate_config"] do get_dirs = Dir.entries("sample-cnfs") dir_list = get_dirs - [".", ".."] dir_list.each do |dir| testsuite_yml = "sample-cnfs/#{dir}/cnf-testsuite.yml" result = ShellCmd.run_testsuite("validate_config cnf-config=#{testsuite_yml}") - (/CNF configuration validated/ =~ result[:output]).should_not be_nil + unless result[:status].success? + Log.info {"Could not validate config: #{testsuite_yml}"} + end + (/Successfully validated CNF config/ =~ result[:output]).should_not be_nil end end - it "'CNFManager.validate_cnf_testsuite_yml' (command) should pass, for all example-cnfs", tags: ["validate_config"] do + it "'validate_config' should pass, for all example-cnfs", tags: ["validate_config"] do get_dirs = Dir.entries("example-cnfs") dir_list = get_dirs - [".", ".."] dir_list.each do |dir| testsuite_yml = "example-cnfs/#{dir}/cnf-testsuite.yml" result = ShellCmd.run_testsuite("validate_config cnf-config=#{testsuite_yml}") - if (/Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file/ =~ result[:output]) - Log.info { "\n #{testsuite_yml}: #{result[:output]}" } + unless result[:status].success? + Log.info {"Could not validate config: #{testsuite_yml}"} end - (/CNF configuration validated/ =~ result[:output]).should_not be_nil + (/Successfully validated CNF config/ =~ result[:output]).should_not be_nil end end - it "'CNFManager::Config#parse_config_yml' should return a populated CNFManager::Config.cnf_config", tags: ["cnf-config"] do + it "'CNFInstall::Config.parse_cnf_config_from_file' should return a populated CNFInstall::Config::Config", tags: ["cnf-config"] do begin - yaml = CNFManager::Config.parse_config_yml("spec/fixtures/cnf-testsuite.yml") - (yaml.cnf_config[:release_name]).should eq("coredns") + config = CNFInstall::Config.parse_cnf_config_from_file("spec/fixtures/cnf-testsuite.yml") + (config.deployments.get_deployment_param(:name)).should eq("coredns") ensure end end @@ -373,7 +328,7 @@ describe "SampleUtils" do args = Sam::Args.new(["cnf-config=./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml"]) cli_hash = CNFManager.sample_setup_cli_args(args, false) CNFManager.sample_setup(cli_hash) if cli_hash["config_file"] - config = CNFManager::Config.parse_config_yml("./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml") + config = CNFInstall::Config.parse_cnf_config_from_file("./sample-cnfs/sample-generic-cnf/cnf-testsuite.yml") task_response = CNFManager.workload_resource_test(args, config) do |resource, container, initialized| test_passed = true begin diff --git a/spec/utils/utils_spec.cr b/spec/utils/utils_spec.cr index 94ef6583c..eb12eb92c 100644 --- a/spec/utils/utils_spec.cr +++ b/spec/utils/utils_spec.cr @@ -123,8 +123,8 @@ describe "Utils" do task_response = CNFManager::Task.single_task_runner(args) do cdir = FileUtils.pwd() response = String::Builder.new - config = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - helm_directory = "#{config.get("helm_directory").as_s?}" + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) + helm_directory = config.deployment.get_deployment_param(:helm_directory) if File.directory?(CNFManager.ensure_cnf_testsuite_dir(args.named["cnf-config"].as(String)) + helm_directory) Dir.cd(CNFManager.ensure_cnf_testsuite_dir(args.named["cnf-config"].as(String)) + helm_directory) Process.run("grep -r -P '^(?!.+0\.0\.0\.0)(?![[:space:]]*0\.0\.0\.0)(?!#)(?![[:space:]]*#)(?!\/\/)(?![[:space:]]*\/\/)(?!\/\\*)(?![[:space:]]*\/\\*)(.+([0-9]{1,3}[\.]){3}[0-9]{1,3})'", shell: true) do |proc| diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr index d03e87c3c..b639b72eb 100644 --- a/src/tasks/cleanup.cr +++ b/src/tasks/cleanup.cr @@ -23,8 +23,8 @@ task "samples_cleanup" do |_, args| cnf_config_file = task_args["cnf-config"].as(String) cnf_config_file = CNFManager.ensure_cnf_testsuite_yml_path(cnf_config_file) - config = CNFManager.parsed_config_file(cnf_config_file) - install_method = CNFInstall.cnf_installation_method(config) + config = CNFInstall::Config.parse_cnf_config_from_file(cnf_config_file) + install_method = config.dynamic.install_method if install_method[0] == CNFInstall::InstallMethod::ManifestDirectory installed_from_manifest = true else diff --git a/src/tasks/cnf_setup.cr b/src/tasks/cnf_setup.cr index 1c7391a34..539d23693 100644 --- a/src/tasks/cnf_setup.cr +++ b/src/tasks/cnf_setup.cr @@ -37,8 +37,8 @@ task "cnf_cleanup" do |_, args| else force = false end - config = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(cnf)) - install_method = CNFInstall.cnf_installation_method(config) + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(cnf)) + install_method = config.dynamic.install_method if install_method[0] == CNFInstall::InstallMethod::ManifestDirectory installed_from_manifest = true else @@ -58,24 +58,6 @@ task "CNFManager.helm_repo_add" do |_, args| end -task "generate_config" do |_, args| - Log.for("verbose").info { "CNFManager.generate_config" } if check_verbose(args) - Log.for("verbose").debug { "args = #{args.inspect}" } if check_verbose(args) - if args.named["config-src"]? - config_src = args.named["config-src"].as(String) - output_file = args.named["output-file"].as(String) if args.named["output-file"]? - output_file = args.named["of"].as(String) if args.named["of"]? - if output_file && !output_file.empty? - Log.info { "generating config with an output file" } - CNFManager::GenerateConfig.generate_config(config_src, output_file) - else - Log.info { "generating config without an output file" } - CNFManager::GenerateConfig.generate_config(config_src) - end - end - -end - #TODO force all cleanups to use generic cleanup task "bad_helm_cnf_cleanup" do |_, args| CNFManager.sample_cleanup(config_file: "sample-cnfs/sample-bad_helm_coredns-cnf", verbose: true) diff --git a/src/tasks/setup.cr b/src/tasks/setup.cr index d819a7e19..9a14fb222 100644 --- a/src/tasks/setup.cr +++ b/src/tasks/setup.cr @@ -25,11 +25,13 @@ task "configuration_file_setup" do |_, args| CNFManager::Points.create_points_yml end -task "test_config" do |_, args| - if args.named["cfg"]? - puts CNFInstall::Config.parse_cnf_config_from_file(args.named["cfg"].to_s).inspect +task "validate_config" do |_, args| + if args.named["cnf-config"]? + config = CNFInstall::Config.parse_cnf_config_from_file(args.named["cnf-config"].to_s) + puts "Successfully validated CNF config" + Log.info {"Config: #{config.inspect}"} else - stdout_failure "cfg parameter needed" + stdout_failure "cnf-config parameter needed" exit 1 end end diff --git a/src/tasks/ueransim_setup.cr b/src/tasks/ueransim_setup.cr index f5c295e41..3fbfcef99 100644 --- a/src/tasks/ueransim_setup.cr +++ b/src/tasks/ueransim_setup.cr @@ -8,7 +8,7 @@ task "install_ueransim" do |_, args| Log.info {"UERANSIM Setup"} if args["cnf-config"]? cnf_config_file = args["cnf-config"].as(String) - config = CNFManager::Config.parse_config_yml(cnf_config_file) + config = CNFInstall::Config.parse_cnf_config_from_file(cnf_config_file) UERANSIM.install(config) else puts "you must provide a cnf-testsuite.yml".colorize(:red) diff --git a/src/tasks/utils/cnf_installation/install_common.cr b/src/tasks/utils/cnf_installation/install_common.cr index 41a51248a..783e7b361 100644 --- a/src/tasks/utils/cnf_installation/install_common.cr +++ b/src/tasks/utils/cnf_installation/install_common.cr @@ -28,112 +28,4 @@ module CNFInstall exit 1 end end - - #Determine, for cnf, whether a helm chart, helm directory, or manifest directory is being used for installation - def self.cnf_installation_method(config : Totem::Config) : Tuple(CNFInstall::InstallMethod, String) - Log.info { "cnf_installation_method" } - Log.info { "cnf_installation_method config: #{config}" } - Log.info { "cnf_installation_method config: #{config.config_paths[0]}/#{config.config_name}.#{config.config_type}" } - helm_chart = optional_key_as_string(config, "helm_chart") - helm_directory = ensure_directory(optional_key_as_string(config, "helm_directory")) - manifest_directory = optional_key_as_string(config, "manifest_directory") - release_name = optional_key_as_string(config, "release_name") - full_helm_directory = "" - full_manifest_directory = "" - Log.info { "release_name: #{release_name}" } - Log.info { "helm_directory: #{helm_directory}" } - Log.info { "manifest_directory: #{manifest_directory}" } - #todo did this ever work? should be full path to destination. This is not - # even the relative path - if Dir.exists?(helm_directory) - Log.info { "Change helm_directory relative path into full path" } - full_helm_directory = Path[CNFManager.sandbox_helm_directory(helm_directory)].expand.to_s - elsif Dir.exists?(manifest_directory) - Log.info { "Change manifest_directory relative path into full path" } - full_manifest_directory = Path[manifest_directory].expand.to_s - else - Log.info { "Building helm_directory and manifest_directory full paths" } - full_helm_directory = Path[CNF_DIR + "/" + release_name + "/" + CNFManager.sandbox_helm_directory(helm_directory)].expand.to_s - full_manifest_directory = Path[CNF_DIR + "/" + release_name + "/" + CNFManager.sandbox_helm_directory(manifest_directory)].expand.to_s - end - - Log.info { "full_helm_directory: #{full_helm_directory} exists? #{Dir.exists?(full_helm_directory)}" } - Log.info { "full_manifest_directory: #{full_manifest_directory} exists? #{Dir.exists?(full_manifest_directory)}" } - - unless exclusive_install_method_tags?(config) - puts "Error: Must populate at lease one installation type in #{config.config_paths[0]}/#{config.config_name}.#{config.config_type}: choose either helm_chart, helm_directory, or manifest_directory in cnf-testsuite.yml!".colorize(:red) - exit 1 - end - - if !helm_chart.empty? - {CNFInstall::InstallMethod::HelmChart, helm_chart} - elsif !helm_directory.empty? - Log.info { "helm_directory not empty, using: #{full_helm_directory}" } - {CNFInstall::InstallMethod::HelmDirectory, full_helm_directory} - elsif !manifest_directory.empty? - Log.info { "manifest_directory not empty, using: #{full_manifest_directory}" } - {CNFInstall::InstallMethod::ManifestDirectory, full_manifest_directory} - else - puts "Error: Must populate at lease one installation type in #{config.config_paths[0]}/#{config.config_name}.#{config.config_type}: choose either helm_chart, helm_directory, or manifest_directory.".colorize(:red) - exit 1 - end - end - - def self.exclusive_install_method_tags?(config) - installation_type_count = ["helm_chart", "helm_directory", "manifest_directory"].reduce(0) do |acc, install_type| - begin - test_tag = config[install_type] - Log.debug { "install type count install_type: #{install_type}" } - if install_type.empty? - acc - else - acc = acc + 1 - end - rescue ex - Log.debug { "install_type: #{install_type} not found in #{config.config_paths[0]}/#{config.config_name}.#{config.config_type}" } - acc - end - end - Log.debug { "installation_type_count: #{installation_type_count}" } - if installation_type_count > 1 - false - else - true - end - end - - def self.install_parameters(config) - Log.info { "install_parameters" } - install_method = config.cnf_config[:install_method] - helm_chart = config.cnf_config[:helm_chart] - helm_directory = config.cnf_config[:helm_directory] - manifest_directory = config.cnf_config[:manifest_directory] - case install_method[0] - when CNFInstall::InstallMethod::ManifestDirectory - directory_parameters = directory_parameter_split(manifest_directory)["parameters"] - when CNFInstall::InstallMethod::HelmChart - directory_parameters = directory_parameter_split(helm_chart)["parameters"] - when CNFInstall::InstallMethod::HelmDirectory - directory_parameters = directory_parameter_split(helm_directory)["parameters"] - else - directory_parameters = "" - end - Log.info { "directory_parameters :#{directory_parameters}" } - directory_parameters - end - - def self.directory_parameter_split(directory_with_parameters) - Log.info { "directory_parameter_split : #{directory_with_parameters}" } - directory = directory_with_parameters.split(" ")[0] - parameters = directory_with_parameters.split(" ")[1..-1].join(" ") - Log.info { "directory : #{directory} parameters: #{parameters}"} - {"directory" => directory, "parameters" => parameters} - end - - def self.ensure_directory(directory_with_parameters) - Log.info { "directory_parameter_split : #{directory_with_parameters}" } - split = directory_parameter_split(directory_with_parameters) - split["directory"] - end - end \ No newline at end of file diff --git a/src/tasks/utils/cnf_manager.cr b/src/tasks/utils/cnf_manager.cr index 698aa9c5c..6b57cec2f 100644 --- a/src/tasks/utils/cnf_manager.cr +++ b/src/tasks/utils/cnf_manager.cr @@ -10,7 +10,6 @@ require "./task.cr" require "./config.cr" require "./jaeger.cr" require "tar" -require "./generate_config.cr" require "./oran_monitor.cr" require "./cnf_installation/install_common.cr" require "./cnf_installation/manifest.cr" @@ -27,79 +26,25 @@ module CNFManager ECR.def_to_s("src/templates/elapsed_time_configmap.yml.ecr") end - # TODO: figure out recursively check for unmapped json and warn on that - # https://github.com/Nicolab/crystal-validator#check - def self.validate_cnf_testsuite_yml(config) - ccyt_validator = nil - valid = true - - begin - ccyt_validator = CnfTestSuiteYmlType.from_json(config.settings.to_json) - rescue ex - valid = false - Log.error { "✖ ERROR: cnf_testsuite.yml field validation error.".colorize(:red) } - Log.error { " please check info in the the field name near the text 'CnfTestSuiteYmlType#' in the error below".colorize(:red) } - Log.error { ex.message } - ex.backtrace.each do |x| - Log.error { x } - end - end - - unmapped_keys_warning_msg = "WARNING: Unmapped cnf_testsuite.yml keys. Please add them to the validator".colorize(:yellow) - unmapped_subkeys_warning_msg = "WARNING: helm_repository is unset or has unmapped subkeys. Please update your cnf_testsuite.yml".colorize(:yellow) - - if ccyt_validator && !ccyt_validator.try &.json_unmapped.empty? - warning_output = [unmapped_keys_warning_msg] of String | Colorize::Object(String) - warning_output.push(ccyt_validator.try &.json_unmapped.to_s) - if warning_output.size > 1 - Log.warn { warning_output.join("\n") } - end - end - - #TODO Differentiate between unmapped subkeys or unset top level key. - if ccyt_validator && !ccyt_validator.try &.helm_repository.try &.json_unmapped.empty? - root = {} of String => (Hash(String, JSON::Any) | Nil) - root["helm_repository"] = ccyt_validator.try &.helm_repository.try &.json_unmapped - - warning_output = [unmapped_subkeys_warning_msg] of String | Colorize::Object(String) - warning_output.push(root.to_s) - if warning_output.size > 1 - Log.warn { warning_output.join("\n") } - end - end - - { valid, warning_output } - end - def self.cnf_resource_ymls(args, config) Log.info { "cnf_resource_ymls" } - destination_cnf_dir = config.cnf_config[:destination_cnf_dir] - helm_directory = sandbox_helm_directory(config.cnf_config[:helm_directory]) - manifest_directory = config.cnf_config[:manifest_directory] - release_name = config.cnf_config[:release_name] - helm_chart_path = Config.get_helm_chart_path(config) - manifest_file_path = Config.get_manifest_file_path(config) - helm_values = config.cnf_config[:helm_values] - test_passed = true - - deployment_namespace = CNFManager.get_deployment_namespace(config) - install_method = config.cnf_config[:install_method] - Log.debug { "install_method: #{install_method}" } template_ymls = [] of YAML::Any - case install_method[0] + case config.dynamic.install_method[0] when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory + helm_chart_path = CNFInstall::Config.get_helm_chart_path(config) + generated_manifest_file_path = CNFInstall::Config.get_manifest_file_path(config) Log.info { "EXPORTED CHART PATH: #{helm_chart_path}" } - Helm.generate_manifest_from_templates(release_name, - helm_chart_path, - manifest_file_path, - deployment_namespace, - helm_values) - template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(manifest_file_path) + Helm.generate_manifest_from_templates(release_name: config.deployments.get_deployment_param(:name), + helm_chart: helm_chart_path, + output_file: generated_manifest_file_path, + namespace: CNFManager.get_deployment_namespace(config), + helm_values: config.deployments.get_deployment_param(:helm_values)) + template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(generated_manifest_file_path) when CNFInstall::InstallMethod::ManifestDirectory - # if release_name.empty? # no helm chart - template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list(CNFInstall::Manifest.manifest_file_list( destination_cnf_dir + "/" + manifest_directory)) - # else + template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list( + CNFInstall::Manifest.manifest_file_list(config.dynamic.destination_cnf_dir + "/" + config.deployments.get_deployment_param(:manifest_directory)) + ) end template_ymls = template_ymls.reject! {|x| @@ -133,12 +78,13 @@ module CNFManager # ``` def self.get_deployment_namespace(config) - install_method = config.cnf_config[:install_method] + install_method = config.dynamic.install_method case install_method[0] when CNFInstall::InstallMethod::HelmChart, Helm::InstallMethod::HelmDirectory - if !config.cnf_config[:helm_install_namespace].empty? - Log.info { "deployment namespace was set to: #{config.cnf_config[:helm_install_namespace]}" } - config.cnf_config[:helm_install_namespace] + config_namespace = config.deployments.get_deployment_param(:namespace) + if !config_namespace.empty? + Log.info { "deployment namespace was set to: #{config_namespace}" } + config_namespace else Log.info { "deployment namespace was set to: #{DEFAULT_CNF_NAMESPACE}" } DEFAULT_CNF_NAMESPACE @@ -164,15 +110,6 @@ module CNFManager resource_resp end - def self.namespace_from_parameters(parameters) - Log.info { "namespace_from_parameters: #{parameters}" } - parameter_list = parameters.strip().split(" ") - namespace_index = parameter_list.index{|x| x =="--namespace"} - namespace = "--namespace #{parameter_list[(namespace_index + 1)]}" if namespace_index - Log.info { "namespace_from_parameters namespace: #{namespace}" } - namespace - end - #test_passes_completely = workload_resource_test do | cnf_config, resource, container, initialized | def self.workload_resource_test(args, config, check_containers = true, @@ -181,14 +118,10 @@ module CNFManager JSON::Any, JSON::Any, Bool | Nil) -> Bool | Nil) # resp = yield resource, container, volumes, initialized test_passed = true - namespace = namespace_from_parameters(CNFInstall.install_parameters(config)) - resource_ymls = cnf_workload_resources(args, config) do |resource| resource end - deployment_namespace = CNFManager.get_deployment_namespace(config) - resource_names = Helm.workload_resource_kind_names(resource_ymls, default_namespace: deployment_namespace) Log.info { "resource names: #{resource_names}" } if resource_names && resource_names.size > 0 @@ -368,67 +301,6 @@ module CNFManager hth["NAME"] end - - def self.generate_and_set_release_name(config_yml_path, src_mode=false) - Log.info { "generate_and_set_release_name" } - Log.info { "generate_and_set_release_name config_yml_path: #{config_yml_path}" } - - yml_file = CNFManager.ensure_cnf_testsuite_yml_path(config_yml_path) - yml_path = CNFManager.ensure_cnf_testsuite_dir(config_yml_path) - - config = CNFManager.parsed_config_file(yml_file) - - predefined_release_name = optional_key_as_string(config, "release_name") - src_helm_directory = optional_key_as_string(config, "helm_directory") - Log.info { "src_helm_directory: #{src_helm_directory}" } - Log.debug { "predefined_release_name: #{predefined_release_name}" } - if predefined_release_name.empty? - install_method = CNFInstall.cnf_installation_method(config) - Log.debug { "install_method: #{install_method}" } - case install_method[0] - when CNFInstall::InstallMethod::HelmChart - Log.info { "generate_and_set_release_name install method: #{install_method[0]} data: #{install_method[1]}" } - Log.info { "generate_and_set_release_name helm_chart_or_directory: #{install_method[1]}" } - release_name = helm_chart_template_release_name(install_method[1]) - when CNFInstall::InstallMethod::HelmDirectory - Log.info { "helm_directory install method: #{yml_path}/#{install_method[1]}" } - # todo get the release name by looking through everything under /tmp/repositories - Log.info { "generate_and_set_release_name helm_chart_or_directory: #{install_method[1]}" } - if src_mode - release_name = helm_chart_template_release_name("#{src_helm_directory}") - else - release_name = helm_chart_template_release_name("#{install_method[1]}") - end - when CNFInstall::InstallMethod::ManifestDirectory - Log.debug { "manifest_directory install method" } - release_name = UUID.random.to_s - else - raise "Install method should be either helm_chart, helm_directory, or manifest_directory" - end - #set generated helm chart release name in yml file - Log.debug { "generate_and_set_release_name: #{release_name}" } - update_yml(yml_file, "release_name", release_name) - end - end - - - # TODO move to sandbox module - def self.cnf_destination_dir(config_file) - Log.info { "cnf_destination_dir config_file: #{config_file}" } - if path_has_yml?(config_file) - yml = config_file - else - yml = config_file + "/cnf-testsuite.yml" - end - config = parsed_config_file(yml) - Log.debug { "cnf_destination_dir parsed_config_file config: #{config}" } - current_dir = FileUtils.pwd - release_name = optional_key_as_string(config, "release_name").split(" ")[0] - Log.info { "release_name: #{release_name}" } - Log.info { "cnf destination dir: #{current_dir}/#{CNF_DIR}/#{release_name}" } - "#{current_dir}/#{CNF_DIR}/#{release_name}" - end - def self.config_source_dir(config_file) if File.directory?(config_file) config_file @@ -441,17 +313,12 @@ module CNFManager Log.info { "helm_repo_add repo_name: #{helm_repo_name} repo_url: #{helm_repo_url} args: #{args.inspect}" } ret = false if helm_repo_name == nil || helm_repo_url == nil - # config = get_parsed_cnf_testsuite_yml(args) - # config = parsed_config_file(ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) Log.info { "helm path: #{Helm::BinarySingleton.helm}" } helm = Helm::BinarySingleton.helm - # helm_repo_name = config.get("helm_repository.name").as_s? - helm_repository = config.cnf_config[:helm_repository] - helm_repo_name = "#{helm_repository && helm_repository["name"]}" - helm_repo_url = "#{helm_repository && helm_repository["repo_url"]}" + helm_repo_name = config.deployments.get_deployment_param(:helm_repo_name) + helm_repo_url = config.deployments.get_deployment_param(:helm_repo_url) Log.info { "helm_repo_name: #{helm_repo_name}" } - # helm_repo_url = config.get("helm_repository.repo_url").as_s? Log.info { "helm_repo_url: #{helm_repo_url}" } end if helm_repo_name && helm_repo_url @@ -496,14 +363,11 @@ module CNFManager # Use manifest directory if helm directory empty def self.sandbox_setup(config, cli_args) Log.info { "sandbox_setup" } - Log.info { "sandbox_setup config: #{config.cnf_config}" } + Log.info { "sandbox_setup config: #{config.inspect}" } verbose = cli_args[:verbose] - config_file = config.cnf_config[:source_cnf_dir] - release_name = config.cnf_config[:release_name] - install_method = config.cnf_config[:install_method] - helm_directory = config.cnf_config[:helm_directory] - manifest_directory = config.cnf_config[:manifest_directory] - destination_cnf_dir = config.cnf_config[:destination_cnf_dir] + config_file = config.dynamic.source_cnf_dir + install_method = config.dynamic.install_method + destination_cnf_dir = config.dynamic.destination_cnf_dir # Create a CNF sandbox dir FileUtils.mkdir_p(destination_cnf_dir) @@ -521,6 +385,7 @@ module CNFManager case install_method[0] when CNFInstall::InstallMethod::ManifestDirectory Log.info { "preparing manifest_directory sandbox" } + manifest_directory = config.deployments.get_deployment_param(:manifest_directory) source_directory = config_source_dir(config_file) + "/" + manifest_directory src_path = Path[source_directory].expand.to_s Log.info { "cp -a #{src_path} #{destination_cnf_dir}" } @@ -532,6 +397,7 @@ module CNFManager end when CNFInstall::InstallMethod::HelmDirectory Log.info { "preparing helm_directory sandbox" } + helm_directory = config.deployments.get_deployment_param(:helm_directory) source_directory = config_source_dir(config_file) + "/" + helm_directory.split(" ")[0] # todo support parameters separately src_path = Path[source_directory].expand.to_s Log.info { "cp -a #{src_path} #{destination_cnf_dir}" } @@ -542,7 +408,7 @@ module CNFManager Log.info { "helm sandbox dir already exists at #{destination_cnf_dir}/#{File.basename(src_path)}" } rescue File::NotFoundError Log.info { "helm directory not found at #{src_path}" } - raise HelmDirectoryMissingError.new(src_path) + stdout_warning "helm directory at #{e.helm_directory} is missing" end when CNFInstall::InstallMethod::HelmChart Log.info { "preparing helm chart sandbox" } @@ -561,14 +427,9 @@ module CNFManager def self.export_published_chart(config, cli_args) Log.info { "exported_chart cli_args: #{cli_args}" } verbose = cli_args[:verbose] - config_file = config.cnf_config[:source_cnf_dir] - helm_directory = config.cnf_config[:helm_directory] - helm_chart = config.cnf_config[:helm_chart] - destination_cnf_dir = config.cnf_config[:destination_cnf_dir] - - #TODO don't think we need to make this here - FileUtils.mkdir_p("#{destination_cnf_dir}/#{helm_directory}") - + config_file = config.dynamic.source_cnf_dir + helm_chart = config.deployments.get_deployment_param(:helm_chart_name) + destination_cnf_dir = config.dynamic.destination_cnf_dir config_path = CNFManager.ensure_cnf_testsuite_yml_path(config_file) # Pulling chart @@ -586,7 +447,6 @@ module CNFManager raise "Helm pull error" end - config = CNFManager::Config.parse_config_yml(config_path) # Discover newly pulled tgz file tgz_name = get_and_verify_tgz_name(helm_chart) @@ -620,44 +480,26 @@ module CNFManager wait_count = cli_args[:wait_count] skip_wait_for_install = cli_args[:skip_wait_for_install] verbose = cli_args[:verbose] - config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) Log.debug { "config in sample_setup: #{config.cnf_config}" } - release_name = config.cnf_config[:release_name] - install_method = config.cnf_config[:install_method] Log.for("verbose").info { "sample_setup" } if verbose Log.info { "config_file #{config_file}" } - # config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) release_name = config.cnf_config[:release_name] install_method = config.cnf_config[:install_method] Log.info { "install_method #{install_method}" } - helm_directory = config.cnf_config[:helm_directory] - helm_values = config.cnf_config[:helm_values] - manifest_directory = config.cnf_config[:manifest_directory] - helm_repository = config.cnf_config[:helm_repository] - helm_repo_name = "#{helm_repository && helm_repository["name"]}" - helm_repo_url = "#{helm_repository && helm_repository["repo_url"]}" + helm_values = config.deployments.get_deployment_param(:helm_values) deployment_namespace = CNFManager.get_deployment_namespace(config) helm_namespace_option = "-n #{deployment_namespace}" ensure_namespace_exists!(deployment_namespace) - - - Log.info { "helm_repo_name: #{helm_repo_name}" } - Log.info { "helm_repo_url: #{helm_repo_url}" } - Log.debug { "helm_directory: #{helm_directory}" } - destination_cnf_dir = config.cnf_config[:destination_cnf_dir] Log.for("verbose").info { "destination_cnf_dir: #{destination_cnf_dir}" } if verbose Log.debug { "mkdir_p destination_cnf_dir: #{destination_cnf_dir}" } FileUtils.mkdir_p(destination_cnf_dir) - begin - sandbox_setup(config, cli_args) - rescue e : HelmDirectoryMissingError - stdout_warning "helm directory at #{e.helm_directory} is missing" - end + sandbox_setup(config, cli_args) helm = Helm::BinarySingleton.helm Log.info { "helm path: #{Helm::BinarySingleton.helm}" } @@ -701,9 +543,12 @@ module CNFManager case install_method[0] when CNFInstall::InstallMethod::ManifestDirectory Log.for("verbose").info { "deploying by manifest file" } if verbose + manifest_directory = config.deployments.get_deployment_param(:manifest_directory) KubectlClient::Apply.file("#{destination_cnf_dir}/#{manifest_directory}") when CNFInstall::InstallMethod::HelmChart - helm_chart = config.cnf_config[:helm_chart] + helm_chart = config.deployments.get_deployment_param(:helm_chart_name) + helm_repo_name = config.deployments.get_deployment_param(:helm_repo_name) + helm_repo_url = config.deployments.get_deployment_param(:helm_repo_url) if !helm_repo_name.empty? || !helm_repo_url.empty? Helm.helm_repo_add(helm_repo_name, helm_repo_url) end @@ -726,7 +571,6 @@ module CNFManager #TODO Add helm options into cnf-testsuite yml #e.g. helm install nsm --set insecure=true ./nsm/helm_chart begin - # helm_install = Helm.install("#{release_name} #{destination_cnf_dir}/#{helm_directory} #{helm_namespace_option}") helm_install = Helm.install(release_name, "#{install_method[1]}", helm_namespace_option, helm_values) rescue e : Helm::InstallationFailed stdout_failure "Helm installation failed" @@ -886,8 +730,6 @@ module CNFManager end def self.cnf_to_new_cluster(config, kubeconfig) - release_name = config.cnf_config[:release_name] - install_method = config.cnf_config[:install_method] release_name = config.cnf_config[:release_name] install_method = config.cnf_config[:install_method] helm_directory = config.cnf_config[:helm_directory] @@ -944,8 +786,7 @@ module CNFManager def self.sample_cleanup(config_file, force=false, installed_from_manifest=false, verbose=true) Log.info { "sample_cleanup" } Log.info { "sample_cleanup installed_from_manifest: #{installed_from_manifest}" } - config = parsed_config_file(ensure_cnf_testsuite_yml_path(config_file)) - parsed_config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) + config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) Log.for("verbose").info { "cleanup config: #{config.inspect}" } if verbose destination_cnf_dir = parsed_config.cnf_config[:destination_cnf_dir] Log.info { "destination_cnf_dir: #{destination_cnf_dir}" } @@ -960,7 +801,7 @@ module CNFManager end # Strips all the helm options from the release name option in config - release_name = "#{config.get("release_name").as_s?}" + release_name = config.deployments.get_deployment_param(:name) release_name = release_name.split(" ")[0] Log.for("sample_cleanup:helm_path").info { Helm::BinarySingleton.helm } @@ -987,7 +828,7 @@ module CNFManager FileUtils.rm_rf(destination_cnf_dir) return result[:status].success? when CNFInstall::InstallMethod::ManifestDirectory - manifest_directory = destination_cnf_dir + "/" + "#{config["manifest_directory"]? && config["manifest_directory"].as_s?}" + manifest_directory = config.deployments.get_deployment_param(:manifest_directory) Log.for("cnf_cleanup:manifest_directory").info { manifest_directory } result = KubectlClient::Delete.file("#{manifest_directory}", wait: true) FileUtils.rm_rf(destination_cnf_dir) @@ -1040,14 +881,6 @@ module CNFManager "#{Helm.chart_name(helm_chart)}-*.tgz" end - class HelmDirectoryMissingError < Exception - property helm_directory : String = "" - - def initialize(helm_directory : String) - self.helm_directory = helm_directory - end - end - class TarFileNotFoundError < Exception def initialize(chart_name) super("No .tgz files found for chart #{chart_name}-*.tgz") diff --git a/src/tasks/utils/config.cr b/src/tasks/utils/config.cr index 5faa5a2a2..c2fa805cc 100644 --- a/src/tasks/utils/config.cr +++ b/src/tasks/utils/config.cr @@ -53,139 +53,6 @@ module CNFManager ), image_registry_fqdns: Hash(String, String ) | Nil) - def self.parse_config_yml(config_yml_path : String) : CNFManager::Config - LOGGING.debug "parse_config_yml config_yml_path: #{config_yml_path}" - yml_file = CNFManager.ensure_cnf_testsuite_yml_path(config_yml_path) - #TODO modify the destination testsuite yml instead of the source testsuite yml - # (especially in the case of the release manager). Then reread the destination config - # TODO for cleanup, read source, then find destination and use release name from destination config - # TODO alternatively use a CRD to save the release name - - CNFManager.generate_and_set_release_name(config_yml_path) - config = CNFManager.parsed_config_file(yml_file) - install_method = CNFInstall.cnf_installation_method(config) - - destination_cnf_dir = CNFManager.cnf_destination_dir(yml_file) - - source_cnf_dir = CNFManager.ensure_cnf_testsuite_dir(config_yml_path) - manifest_directory = optional_key_as_string(config, "manifest_directory") - if config["helm_repository"]? - helm_repository = config["helm_repository"].as_h - helm_repo_name = optional_key_as_string(helm_repository, "name") - helm_repo_url = optional_key_as_string(helm_repository, "repo_url") - else - helm_repo_name = "" - helm_repo_url = "" - end - helm_chart = optional_key_as_string(config, "helm_chart") - helm_values = optional_key_as_string(config, "helm_values") - release_name = optional_key_as_string(config, "release_name") - helm_directory = optional_key_as_string(config, "helm_directory") - helm_install_namespace = optional_key_as_string(config, "helm_install_namespace") - if config["enabled"]? - core_enabled = config["enabled"].as_bool.to_s - else - core_enabled = "" - end - if config["emergency"]? - core_emergency = config["emergency"].as_bool.to_s - else - core_emergency = "" - end - if config["sd"]? - core_sd = config["sd"].as_s - else - core_sd = "" - end - fiveG_core = {amf_service_name: optional_key_as_string(config, "amf_service_name"), - mmc: optional_key_as_string(config, "mmc"), - mnc: optional_key_as_string(config, "mnc"), - sst: optional_key_as_string(config, "sst"), - sd: core_sd, - tac: optional_key_as_string(config, "tac"), - protectionScheme: optional_key_as_string(config, "protectionScheme"), - publicKey: optional_key_as_string(config, "publicKey"), - publicKeyId: optional_key_as_string(config, "publicKeyId"), - routingIndicator: optional_key_as_string(config, "routingIndicator"), - enabled: core_enabled, - count: optional_key_as_string(config, "count"), - initialMSISDN: optional_key_as_string(config, "initialMSISDN"), - key: optional_key_as_string(config, "key"), - op: optional_key_as_string(config, "op"), - opType: optional_key_as_string(config, "opType"), - type: optional_key_as_string(config, "type"), - apn: optional_key_as_string(config, "apn"), - emergency: core_emergency, - } - core = optional_key_as_string(config, "amf_label") - smf = optional_key_as_string(config, "smf_label") - upf = optional_key_as_string(config, "upf_label") - ric = optional_key_as_string(config, "ric_label") - - white_list_container_names = optional_key_as_string(config, "allowlist_helm_chart_container_names") - if config["allowlist_helm_chart_container_names"]? - white_list_container_names = config["allowlist_helm_chart_container_names"].as_a.map do |c| - "#{c.as_s?}" - end - else - white_list_container_names = [] of String - end - if config["container_names"]? - container_names_totem = config["container_names"] - container_names = container_names_totem.as_a.map do |container| - {"name" => optional_key_as_string(container, "name"), - "rolling_update_test_tag" => optional_key_as_string(container, "rolling_update_test_tag"), - "rolling_downgrade_test_tag" => optional_key_as_string(container, "rolling_downgrade_test_tag"), - "rolling_version_change_test_tag" => optional_key_as_string(container, "rolling_version_change_test_tag"), - "rollback_from_tag" => optional_key_as_string(container, "rollback_from_tag"), - } - end - else - container_names = [{"name" => "", - "rolling_update_test_tag" => "", - "rolling_downgrade_test_tag" => "", - "rolling_version_change_test_tag" => "", - "rollback_from_tag" => "", - }] - end - - docker_insecure_registries = [] of String - if config["docker_insecure_registries"]? && !config["docker_insecure_registries"].nil? - docker_insecure_registries = config["docker_insecure_registries"].as_a.map do |c| - "#{c.as_s?}" - end - end - - image_registry_fqdns = Hash(String, String).new - if config["image_registry_fqdns"]? && !config["image_registry_fqdns"].nil? - config["image_registry_fqdns"].as_h.each do |key, value| - image_registry_fqdns[key] = value.as_s - end - end - - # if you change this, change instantiation in task.cr/single_task_runner as well - new({ destination_cnf_dir: destination_cnf_dir, - source_cnf_dir: source_cnf_dir, - install_method: install_method, - manifest_directory: manifest_directory, - helm_directory: helm_directory, - release_name: release_name, - helm_repository: {name: helm_repo_name, repo_url: helm_repo_url}, - helm_chart: helm_chart, - helm_values: helm_values, - helm_install_namespace: helm_install_namespace, - container_names: container_names, - white_list_container_names: white_list_container_names, - docker_insecure_registries: docker_insecure_registries, - amf_label: core, - smf_label: smf, - upf_label: upf, - ric_label: ric, - fiveG_core: fiveG_core, - image_registry_fqdns: image_registry_fqdns,}) - - end - def self.get_helm_chart_path(config) helm_directory = config.cnf_config[:helm_directory] destination_cnf_dir = config.cnf_config[:destination_cnf_dir] @@ -204,26 +71,5 @@ module CNFManager destination_cnf_dir = config.cnf_config[:destination_cnf_dir] manifest_file_path = destination_cnf_dir + "/" + "temp_template.yml" end - - def self.install_method_by_config_file(config_file) : CNFInstall::InstallMethod - LOGGING.info "install_data_by_config_file" - config = CNFManager.parsed_config_file(config_file) - sandbox_config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - install_method = CNFInstall.cnf_installation_method(config) - install_method[0] - end - def self.config_src_by_config_file(config_file) : String - LOGGING.info "install_data_by_config_file" - config = CNFManager.parsed_config_file(config_file) - sandbox_config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - install_method = CNFInstall.cnf_installation_method(config) - install_method[1] - end - def self.release_name_by_config_file(config_file) : String - LOGGING.info "release_name_by_config_file" - config = CNFManager.parsed_config_file(config_file) - sandbox_config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file)) - release_name = sandbox_config.cnf_config[:release_name] - end end end diff --git a/src/tasks/utils/generate_config.cr b/src/tasks/utils/generate_config.cr deleted file mode 100644 index dc9ff5261..000000000 --- a/src/tasks/utils/generate_config.cr +++ /dev/null @@ -1,128 +0,0 @@ -require "totem" -require "colorize" -require "./types/cnf_testsuite_yml_type.cr" -require "helm" -require "uuid" -require "./points.cr" -require "./task.cr" - -module CNFManager - module GenerateConfig - def self.export_manifest(config_src, output_file="./cnf-testsuite.yml") - LOGGING.info "export_manifest" - LOGGING.info "export_manifest config_src: #{config_src}" - generate_initial_testsuite_yml(config_src, output_file) - CNFManager.generate_and_set_release_name(output_file, src_mode: true) - config = CNFManager.parsed_config_file(output_file) - release_name = optional_key_as_string(config, "release_name") - install_method = CNFInstall.install_method_by_config_src(config_src) - LOGGING.info "install_method: #{install_method}" - if install_method == CNFInstall::InstallMethod::ManifestDirectory - template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list(CNFInstall::Manifest.manifest_file_list( config_src)) - else - # todo if success false, raise error - Helm.generate_manifest_from_templates(release_name, - config_src) - template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls() - end - resource_ymls = Helm.all_workload_resources(template_ymls) - resource_ymls - end - - #get list of image:tags from helm chart/helm directory/manifest file - #note: config_src must be an absolute path if a directory, todo: make this more resilient - def self.images_from_config_src(config_src) - LOGGING.info "images_from_config_src" - #return container image name/tag - ret_containers = [] of NamedTuple(container_name: String, image_name: String, tag: String) - resource_ymls = CNFManager::GenerateConfig.export_manifest(config_src) - resource_resp = resource_ymls.map do | resource | - LOGGING.info "gen config resource: #{resource}" - unless resource["kind"].as_s.downcase == "service" ## services have no containers - containers = CNFInstall::Manifest.manifest_containers(resource) - - LOGGING.info "containers: #{containers}" - container_name = containers.as_a[0].as_h["name"].as_s if containers - if containers - container_names = containers.as_a.map do |container| - LOGGING.debug "container: #{container}" - container_name = container.as_h["name"].as_s - image_name = container.as_h["image"].as_s.split(":")[0] - if container.as_h["image"].as_s.split(":").size > 1 - tag = container.as_h["image"].as_s.split(":")[1] - else - tag = "latest" - end - ret_containers << {container_name: container_name, - image_name: image_name, - tag: tag} - LOGGING.debug "ret_containers: #{ret_containers}" - end - end - end - end - ret_containers - end - - def self.generate_config(config_src, output_file="./cnf-testsuite.yml") - resource_ymls = CNFManager::GenerateConfig.export_manifest(config_src, output_file) - resource_resp = resource_ymls.map do | resource | - LOGGING.info "gen config resource: #{resource}" - unless resource["kind"].as_s.downcase == "service" ## services have no containers - containers = CNFInstall::Manifest.manifest_containers(resource) - - LOGGING.info "containers: #{containers}" - container_name = containers.as_a[0].as_h["name"].as_s if containers - if containers - container_names = containers.as_a.map { |container| - LOGGING.debug "container: #{container}" - if container.as_h["image"].as_s.split(":").size > 1 - rolling_update_test_tag = container.as_h["image"].as_s.split(":")[1] - else - rolling_update_test_tag = "" - end -# don't mess with the indentation here - container_names_template = <<-TEMPLATE - - - name: #{container.as_h["name"].as_s} - rolling_update_test_tag: #{rolling_update_test_tag} - rolling_downgrade_test_tag: #{rolling_update_test_tag} - rolling_version_change_test_tag: #{rolling_update_test_tag} - rollback_from_tag: #{rolling_update_test_tag} - TEMPLATE - }.join("") - # Don't add container_names to yml. This isn't needed until the rolling update tests are added to the cert. - # update_yml(output_file, "container_names", container_names) - end - end - # resp - end - end - - def self.generate_initial_testsuite_yml(config_src, config_yml_path="./cnf-testsuite.yml") - if !File.exists?(config_yml_path) - case CNFInstall.install_method_by_config_src(config_src) - when CNFInstall::InstallMethod::HelmChart - testsuite_yml_template_resp = TestSuiteYmlTemplate.new("helm_chart: #{config_src}").to_s - when CNFInstall::InstallMethod::HelmDirectory - testsuite_yml_template_resp = TestSuiteYmlTemplate.new("helm_directory: #{config_src}").to_s - when CNFInstall::InstallMethod::ManifestDirectory - testsuite_yml_template_resp = TestSuiteYmlTemplate.new("manifest_directory: #{config_src}").to_s - else - puts "Error: #{config_src} is neither a helm_chart, helm_directory, or manifest_directory.".colorize(:red) - exit 1 - end - File.write(config_yml_path, testsuite_yml_template_resp) - else - LOGGING.error "#{config_yml_path} already exists" - end - end - end -end - -class TestSuiteYmlTemplate - def initialize(@install_key : String) - end - - ECR.def_to_s("src/templates/testsuite_template.yml.ecr") -end diff --git a/src/tasks/utils/task.cr b/src/tasks/utils/task.cr index 2f07e9a9a..69f580594 100644 --- a/src/tasks/utils/task.cr +++ b/src/tasks/utils/task.cr @@ -62,47 +62,15 @@ module CNFManager begin if args.named["cnf-config"]? # platform tests don't have a cnf-config - config = CNFManager::Config.parse_config_yml(args.named["cnf-config"].as(String)) + config = CNFInstall::Config.parse_cnf_config_from_file(args.named["cnf-config"].as(String)) else - config = CNFManager::Config.new({ destination_cnf_dir: "", - source_cnf_dir: "", - install_method: {CNFInstall::InstallMethod::HelmChart, ""}, - manifest_directory: "", - helm_directory: "", - release_name: "", - helm_repository: {name: "", repo_url: ""}, - helm_chart: "", - helm_values: "", - helm_install_namespace: "", - container_names: [{"name" => "", "rolling_update_test_tag" => ""}], - white_list_container_names: [""], - docker_insecure_registries: [] of String, - amf_label: "", - smf_label: "", - upf_label: "", - ric_label: "", - fiveG_core: {amf_service_name: "", - mmc: "", - mnc: "", - sst: "", - sd: "", - tac: "", - protectionScheme: "", - publicKey: "", - publicKeyId: "", - routingIndicator: "", - enabled: "", - count: "", - initialMSISDN: "", - key: "", - op: "", - opType: "", - type: "", - apn: "", - emergency: "", - }, - image_registry_fqdns: Hash(String, String).new} ) - end + yaml_string = <<-YAML + deployments: + helm_deps: + - name: "platform-test-dummy-deployment" + helm_directory: "" + YAML + config = CNFInstall::Config.parse_cnf_config_from_yaml(yaml_string) test_start_time = Time.utc if task test_name = task.as(Sam::Task).name.as(String) diff --git a/src/tasks/workload/compatibility.cr b/src/tasks/workload/compatibility.cr index e4a46b2cb..3337bcb70 100644 --- a/src/tasks/workload/compatibility.cr +++ b/src/tasks/workload/compatibility.cr @@ -416,7 +416,6 @@ task "helm_chart_published", ["helm_local_install"] do |t, args| end # config = cnf_testsuite_yml - # config = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) # helm_chart = "#{config.get("helm_chart").as_s?}" helm_chart = config.cnf_config[:helm_chart] current_dir = FileUtils.pwd @@ -495,17 +494,6 @@ task "helm_chart_valid", ["helm_local_install"] do |t, args| end end -task "validate_config" do |_, args| - yml = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(args.named["cnf-config"].as(String))) - valid, warning_output = CNFManager.validate_cnf_testsuite_yml(yml) - emoji_config="📋" - if valid - stdout_success "CNF configuration validated #{emoji_config}" - else - stdout_failure "Critical Error with CNF Configuration. Please review USAGE.md for steps to set up a valid CNF configuration file #{emoji_config}" - end -end - def setup_calico_cluster(cluster_name : String) : KindManager::Cluster Log.info { "Running cni_compatible(Cluster Creation)" } Helm.helm_repo_add("projectcalico","https://docs.projectcalico.org/charts") diff --git a/src/tasks/workload/microservice.cr b/src/tasks/workload/microservice.cr index a4301572f..c3258ee06 100644 --- a/src/tasks/workload/microservice.cr +++ b/src/tasks/workload/microservice.cr @@ -35,8 +35,6 @@ task "shared_database", ["install_cluster_tools"] do |t, args| resource_ymls = CNFManager.cnf_workload_resources(args, config) { |resource| resource } resource_names = Helm.workload_resource_kind_names(resource_ymls) helm_chart_cnf_services : Array(JSON::Any) - # namespace = CNFManager.namespace_from_parameters(CNFInstall.install_parameters(config)) - # Log.info { "namespace: #{namespace}"} helm_chart_cnf_services = resource_names.map do |resource_name| Log.info { "helm_chart_cnf_services resource_name: #{resource_name}"} if resource_name[:kind].downcase == "service" diff --git a/src/tasks/workload/state.cr b/src/tasks/workload/state.cr index 592cfb6ce..8570bf76f 100644 --- a/src/tasks/workload/state.cr +++ b/src/tasks/workload/state.cr @@ -369,7 +369,7 @@ task "elastic_volumes" do |t, args| # todo use workload resource # elastic = WorkloadResource.elastic?(volumes) - namespace = CNFManager.namespace_from_parameters(CNFInstall.install_parameters(config)) || CNFManager.get_deployment_namespace(config) + namespace = CNFManager.get_deployment_namespace(config) full_resource = KubectlClient::Get.resource(resource["kind"], resource["name"], namespace) elastic_result = WorkloadResource.elastic?(full_resource, volumes.as_a, namespace)