config: introduce new config format and parser

For supporting multiple deployments in one cnf and
improving overall config usability and structure - new
config format has to be introduced.
Some existing parameters usage need to be changed
for easier adaptation.
This change does not replace the old config format.

Refs: cnti-testcatalog#2121
Signed-off-by: Konstantin Yarovoy <[email protected]>

embedded_files/cnf-testsuite-v2-example.yml

@@ -0,0 +1,23 @@
+---
+config_version: "v2"
+common: 
+  container_names:
+    - name: coredns
+      rolling_update_test_tag: "1.8.0"
+      rolling_downgrade_test_tag: 1.6.7
+      rolling_version_change_test_tag: 1.8.0
+      rollback_from_tag: 1.8.0
+
+deployments:
+  helm_charts:
+    - name: coredns
+      helm_repo_name: stable
+      helm_repo_url: https://cncf.gitlab.io/stable
+      helm_chart_name: coredns
+  helm_dirs:
+    - name: envoy
+      helm_directory: ../example-cnfs/envoy/envoy/
+  manifests:
+    - name: nginx
+      manifest_directory: ../sample_cnfs/ndn-mutable-configmap/manifests
+

src/tasks/setup.cr

@@ -25,3 +25,11 @@ task "configuration_file_setup" do |_, args|
   CNFManager::Points.create_points_yml
 end
 
+task "test_config" do |_, args|
+  if args.named["cfg"]?
+    puts CNFInstall::Config.parse_cnf_config_from_file(args.named["cfg"].to_s).inspect
+  else
+    stdout_failure "cfg parameter needed"
+    exit 1
+  end
+end

src/tasks/utils/cnf_installation/config.cr

@@ -0,0 +1,243 @@
+require "yaml"
+require "../utils.cr"
+
+module CNFInstall
+  module Config
+    @[YAML::Serializable::Options(emit_nulls: true)]
+    alias AnyDeploymentConfig = HelmChartConfig | HelmDirectoryConfig | ManifestDirectoryConfig
+
+    class ConfigBase
+      include YAML::Serializable
+      include YAML::Serializable::Strict
+    end
+
+    class Config < ConfigBase
+      # TODO: remove as much parameters as possible
+      @config_version : String
+      @common : CommonParameters | Nil
+      @dynamic : DynamicParameters | Nil
+      @deployments : DeploymentsConfig
+      getter config_version, common, deployments
+      getter! dynamic
+
+      def after_initialize
+        unless @dynamic
+          @dynamic = DynamicParameters.new()
+        end
+      end
+    end
+
+    class CommonParameters < ConfigBase
+      @service_name : String | Nil
+      @rolling_update_tag : String | Nil
+      @container_names : Array(Hash(String, String)) | Nil
+      @white_list_container_names : Array(String) | Nil
+      @docker_insecure_registries : Array(String) | Nil
+      @image_registry_fqdns : Hash(String, String) | Nil
+      @five_g_parameters : FiveGParameters | Nil
+      getter service_name, rolling_update_tag, container_names, white_list_container_names
+      getter docker_insecure_registries, image_registry_fqdns, five_g_parameters
+    end
+
+    class DynamicParameters < ConfigBase
+      @source_cnf_dir : String | Nil
+      @destination_cnf_dir : String | Nil
+      @install_method : Tuple(CNFInstall::InstallMethod, String) | Nil
+      property source_cnf_dir, destination_cnf_dir
+      property install_method
+
+      def initialize()
+      end
+    end
+
+    class DeploymentsConfig < ConfigBase
+      @helm_charts : Array(HelmChartConfig) = [] of HelmChartConfig
+      @helm_dirs : Array(HelmDirectoryConfig) = [] of HelmDirectoryConfig
+      @manifests : Array(ManifestDirectoryConfig) = [] of ManifestDirectoryConfig
+      # deployments.current and all related functionality should be removed with new installation process.
+      @@current : AnyDeploymentConfig
+      getter helm_charts, helm_dirs, manifests
+
+      def after_initialize
+        if @helm_charts.empty? && @helm_dirs.empty? && @manifests.empty?
+          raise YAML::Error.new("At least one deployment should be configured")
+        end
+
+        deployment_names = Set(String).new
+        deployment_found = false
+        {@helm_charts, @helm_dirs, @manifests}.each do |deployment_array|
+          if deployment_array
+
+            # To be removed with new installation process.
+            if deployment_found
+              raise YAML::Error.new("Multiple deployments are not supported yet")
+            else
+              deployment_found = true
+            end
+            @@current = deployment_array[0]
+            #
+
+            deployment_array.each do |deployment|
+              if deployment_names.includes?(deployment.name)
+                raise YAML::Error.new("Deployment names should be unique: \"#{deployment.name}\"")
+              else
+                deployment_names.add(deployment.name)
+              end
+            end
+          end
+        end
+      end
+
+      def get_deployment_param(param : Symbol) : String?
+        case @@current
+        when HelmChartConfig
+          case param
+          when :name            then @@current.name
+          when :helm_repo_name  then @@current.helm_repo_name
+          when :helm_repo_url   then @@current.helm_repo_url
+          when :helm_chart_name then @@current.helm_chart_name
+          when :helm_values     then @@current.helm_values
+          when :namespace       then @@current.namespace
+          else nil
+          end
+        when HelmDirectoryConfig
+          case param
+          when :name           then @@current.name
+          when :helm_directory then @@current.helm_directory
+          when :helm_values    then @@current.helm_values
+          when :namespace      then @@current.namespace
+          else nil
+          end
+        when ManifestDirectoryConfig
+          case param
+          when :name               then @@current.name
+          when :manifest_directory then @@current.manifest_directory
+          else nil
+          end
+        else
+          nil
+        end
+      end
+    end
+
+    class DeploymentConfig < ConfigBase
+      @name : String
+      getter name
+    end
+
+    class HelmDeploymentConfig < DeploymentConfig
+      @helm_values : String | Nil
+      @namespace : String | Nil
+      getter helm_values, namespace
+    end
+
+    class HelmChartConfig < HelmDeploymentConfig
+      @helm_repo_name : String
+      @helm_repo_url : String
+      @helm_chart_name : String
+
+      getter helm_repo_name, helm_repo_url, helm_chart_name
+    end
+
+    class HelmDirectoryConfig < HelmDeploymentConfig
+      @helm_directory : String
+      getter helm_directory
+    end
+
+    class ManifestDirectoryConfig < DeploymentConfig
+      @manifest_directory : String
+      getter manifest_directory
+    end
+
+    class FiveGParameters < ConfigBase
+      @amf_label : String | Nil
+      @smf_label : String | Nil
+      @upf_label : String | Nil
+      @ric_label : String | Nil
+      @amf_service_name : String | Nil
+      @mmc : String | Nil
+      @mnc : String | Nil
+      @sst : String | Nil
+      @sd : String | Nil
+      @tac : String | Nil
+      @protectionScheme : String | Nil
+      @publicKey : String | Nil
+      @publicKeyId : String | Nil
+      @routingIndicator : String | Nil
+      @enabled : String | Nil
+      @count : String | Nil
+      @initialMSISDN : String | Nil
+      @key : String | Nil
+      @op : String | Nil
+      @opType : String | Nil
+      @type : String | Nil
+      @apn : String | Nil
+      @emergency : String | Nil
+    end
+
+    def self.parse_cnf_config_from_file(path_to_config)
+      yaml_content = File.read(path_to_config)
+      config_dir = CNFManager.ensure_cnf_testsuite_dir(path_to_config)
+      begin
+        parse_cnf_config_from_yaml(yaml_content, config_dir)
+      rescue exception
+        stdout_failure "Error during parsing CNF config on #{path_to_config}"
+        stdout_failure exception.message
+        stdout_failure "Please check your config according to the config template."
+        exit 1
+      end
+    end
+
+    def self.parse_cnf_config_from_yaml(yaml_content, config_dir)
+      config = Config.from_yaml(yaml_content)
+
+      unless config.dynamic.source_cnf_dir
+        config.dynamic.source_cnf_dir = config_dir
+      end
+
+      unless config.dynamic.install_method
+        config.dynamic.install_method = get_install_method_from_deployments(config.deployments)
+      end
+
+      unless config.dynamic.destination_cnf_dir
+        install_method = config.dynamic.install_method.not_nil![0]
+        case install_method
+        when CNFInstall::InstallMethod::HelmChart
+          deployment_name = config.deployments.helm_charts[0].name
+        when CNFInstall::InstallMethod::HelmDirectory
+          deployment_name = config.deployments.helm_dirs[0].name
+        when CNFInstall::InstallMethod::ManifestDirectory
+          deployment_name = config.deployments.manifests[0].name
+        else
+          raise "Unknown install method: #{install_method}"
+        end
+        current_dir = FileUtils.pwd
+        config.dynamic.destination_cnf_dir = "#{current_dir}/#{CNF_DIR}/#{deployment_name}"
+      end
+
+      config
+    end
+
+    def self.get_install_method_from_deployments(deployments_config)
+      if !deployments_config.helm_charts.empty?
+        helm_chart_name = deployments_config.helm_charts[0].helm_chart_name
+        {CNFInstall::InstallMethod::HelmChart, helm_chart_name}
+
+      elsif !deployments_config.helm_dirs.empty?
+        helm_directory = deployments_config.helm_dirs[0].helm_directory
+        deployment_name = deployments_config.helm_dirs[0].name
+        full_helm_directory = Path[CNF_DIR + "/" + deployment_name  + "/" + CNFManager.sandbox_helm_directory(helm_directory)].expand.to_s
+        {CNFInstall::InstallMethod::HelmDirectory, full_helm_directory}
+
+      elsif !deployments_config.manifests.empty?
+        manifest_directory = deployments_config.manifests[0].manifest_directory
+        deployment_name = deployments_config.manifests[0].name
+        full_manifest_directory = Path[CNF_DIR + "/" + deployment_name + "/" + CNFManager.sandbox_helm_directory(manifest_directory)].expand.to_s
+        {CNFInstall::InstallMethod::ManifestDirectory, full_manifest_directory}
+
+      else
+        raise YAML::Error.new("At least one deployment should be configured")
+      end
+    end
+  end
+end

src/tasks/utils/cnf_installation/install_common.cr

@@ -29,14 +29,6 @@ module CNFInstall
     end
   end
 
-  def self.cnf_installation_method(config : CNFManager::Config) : Tuple(CNFInstall::InstallMethod, String)
-    Log.info { "cnf_installation_method config : CNFManager::Config" }
-    Log.info { "config_cnf_config: #{config.cnf_config}" }
-    yml_file_path = config.cnf_config[:source_cnf_file]
-    parsed_config_file = CNFManager.parsed_config_file(yml_file_path)
-    cnf_installation_method(parsed_config_file)
-  end
-
   #Determine, for cnf, whether a helm chart, helm directory, or manifest directory is being used for installation
   def self.cnf_installation_method(config : Totem::Config) : Tuple(CNFInstall::InstallMethod, String)
     Log.info { "cnf_installation_method" }

src/tasks/utils/cnf_manager.cr

@@ -74,17 +74,16 @@ module CNFManager
   def self.cnf_resource_ymls(args, config)
     Log.info { "cnf_resource_ymls" }
     destination_cnf_dir = config.cnf_config[:destination_cnf_dir]
-    yml_file_path = config.cnf_config[:yml_file_path]
     helm_directory = sandbox_helm_directory(config.cnf_config[:helm_directory])
     manifest_directory = config.cnf_config[:manifest_directory]
     release_name = config.cnf_config[:release_name]
-    helm_chart_path = config.cnf_config[:helm_chart_path]
-    manifest_file_path = config.cnf_config[:manifest_file_path]
+    helm_chart_path = Config.get_helm_chart_path(config)
+    manifest_file_path = Config.get_manifest_file_path(config)
     helm_values = config.cnf_config[:helm_values]
     test_passed = true
 
     deployment_namespace = CNFManager.get_deployment_namespace(config)
-    install_method = CNFInstall.cnf_installation_method(config)
+    install_method = config.cnf_config[:install_method]
     Log.debug { "install_method: #{install_method}" }
     template_ymls = [] of YAML::Any
     case install_method[0]
@@ -134,7 +133,7 @@ module CNFManager
   # ```
 
   def self.get_deployment_namespace(config)
-    install_method = CNFInstall.cnf_installation_method(config)
+    install_method = config.cnf_config[:install_method]
     case install_method[0]
     when CNFInstall::InstallMethod::HelmChart, Helm::InstallMethod::HelmDirectory
       if !config.cnf_config[:helm_install_namespace].empty?
@@ -500,10 +499,8 @@ module CNFManager
     release_name = config.cnf_config[:release_name]
     install_method = config.cnf_config[:install_method]
     helm_directory = config.cnf_config[:helm_directory]
-    source_helm_directory = config.cnf_config[:source_helm_directory]
     manifest_directory = config.cnf_config[:manifest_directory]
-    helm_chart_path = config.cnf_config[:helm_chart_path]
-    destination_cnf_dir = CNFManager.cnf_destination_dir(config_file)
+    destination_cnf_dir = config.cnf_config[:destination_cnf_dir]
 
     # Create a CNF sandbox dir
     FileUtils.mkdir_p(destination_cnf_dir)
@@ -532,7 +529,7 @@ module CNFManager
       end
     when CNFInstall::InstallMethod::HelmDirectory
       Log.info { "preparing helm_directory sandbox" }
-      source_directory = config_source_dir(config_file) + "/" + source_helm_directory.split(" ")[0] # todo support parameters separately
+      source_directory = config_source_dir(config_file) + "/" + helm_directory.split(" ")[0] # todo support parameters separately
       src_path = Path[source_directory].expand.to_s
       Log.info { "cp -a #{src_path} #{destination_cnf_dir}" }
 
@@ -564,7 +561,7 @@ module CNFManager
     config_file = config.cnf_config[:source_cnf_dir]
     helm_directory = config.cnf_config[:helm_directory]
     helm_chart = config.cnf_config[:helm_chart]
-    destination_cnf_dir = CNFManager.cnf_destination_dir(config_file)
+    destination_cnf_dir = config.cnf_config[:destination_cnf_dir]
 
     #TODO don't think we need to make this here
     FileUtils.mkdir_p("#{destination_cnf_dir}/#{helm_directory}")
@@ -644,11 +641,9 @@ module CNFManager
 
     Log.info { "helm_repo_name: #{helm_repo_name}" }
     Log.info { "helm_repo_url: #{helm_repo_url}" }
-
-    helm_chart_path = config.cnf_config[:helm_chart_path]
     Log.debug { "helm_directory: #{helm_directory}" }
 
-    destination_cnf_dir = CNFManager.cnf_destination_dir(config_file)
+    destination_cnf_dir = config.cnf_config[:destination_cnf_dir]
 
     Log.for("verbose").info { "destination_cnf_dir: #{destination_cnf_dir}" } if verbose
     Log.debug { "mkdir_p destination_cnf_dir: #{destination_cnf_dir}" }
@@ -884,7 +879,6 @@ module CNFManager
     helm_repository = config.cnf_config[:helm_repository]
     helm_repo_name = "#{helm_repository && helm_repository["name"]}"
     helm_repo_url = "#{helm_repository && helm_repository["repo_url"]}"
-    helm_chart_path = config.cnf_config[:helm_chart_path]
     helm_chart = config.cnf_config[:helm_chart]
     destination_cnf_dir = config.cnf_config[:destination_cnf_dir]
     deployment_namespace = CNFManager.get_deployment_namespace(config)
@@ -934,11 +928,12 @@ module CNFManager
   def self.sample_cleanup(config_file, force=false, installed_from_manifest=false, verbose=true)
     Log.info { "sample_cleanup" }
     Log.info { "sample_cleanup installed_from_manifest: #{installed_from_manifest}" }
-    destination_cnf_dir = CNFManager.cnf_destination_dir(config_file)
-    Log.info { "destination_cnf_dir: #{destination_cnf_dir}" }
     config = parsed_config_file(ensure_cnf_testsuite_yml_path(config_file))
     parsed_config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file))
     Log.for("verbose").info { "cleanup config: #{config.inspect}" } if verbose
+    destination_cnf_dir = parsed_config.cnf_config[:destination_cnf_dir]
+    Log.info { "destination_cnf_dir: #{destination_cnf_dir}" }
+
 
     config_maps_dir = "#{destination_cnf_dir}/config_maps"
     if Dir.exists?(config_maps_dir)
@@ -962,7 +957,7 @@ module CNFManager
       Log.for("sample_cleanup").info { "Destination dir #{destination_cnf_dir} exists" }
     end
 
-    install_method = CNFInstall.cnf_installation_method(parsed_config)
+    install_method = parsed_config.cnf_config[:install_method]
     Log.for("sample_cleanup:install_method").info { install_method }
     case install_method[0]
     when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory