Feature: Group several manifest into one file

- group manifests from several CNFs into one common_manifests.yaml file - add functionality in cleanup.cr to remove this file - depends on PR in helm library: cnf-testsuite/helm#4 - common_manifest functionality: used common_manifest instead of templates - remove methods related to templates - update generated manifest: add namespace to each resource Signed-off-by: barmull <[email protected]>
cnti-testcatalog · Oct 17, 2024 · 70db957 · 70db957
1 parent 8aa5bea
commit 70db957
Show file tree

Hide file tree

Showing 8 changed files with 101 additions and 68 deletions.
diff --git a/shard.lock b/shard.lock
@@ -26,7 +26,7 @@ shards:
 
   helm:
     git: https://github.com/cnf-testsuite/helm.git
-    version: 1.0.2
+    version: 1.0.3
 
   icr:
     git: https://github.com/crystal-community/icr.git

diff --git a/src/tasks/constants.cr b/src/tasks/constants.cr
@@ -4,6 +4,7 @@ ESSENTIAL_PASSED_THRESHOLD = 15
 CNF_DIR = "cnfs"
 CONFIG_FILE = "cnf-testsuite.yml"
 BASE_CONFIG = "./config.yml"
+COMMON_MANIFEST_FILE_PATH = "#{CNF_DIR}/common_manifest.yml"
 PASSED = "passed"
 FAILED = "failed"
 SKIPPED = "skipped"

diff --git a/src/tasks/utils/cnf_installation/manifest.cr b/src/tasks/utils/cnf_installation/manifest.cr
@@ -1,21 +1,35 @@
 module CNFInstall
   module Manifest
-    def self.parse_manifest_as_ymls(template_file_name="cnfs/temp_template.yml")
-      Log.info { "parse_manifest_as_ymls template_file_name: #{template_file_name}" }
-      templates = File.read(template_file_name)
-      split_template = templates.split(/(\s|^)---(\s|$)/)
-      ymls = split_template.map { | template |
+    def self.manifest_path_to_ymls(manifest_path)
+      Log.info { "manifest_path_to_ymls file_path: #{manifest_path}" }
+      manifest = File.read(manifest_path)
+      manifest_string_to_ymls(manifest)
+    end
+
+    def self.manifest_string_to_ymls(manifest_string)
+      Log.info { "method: manifest_string_to_ymls" }
+      split_content = manifest_string.split(/(\s|^)---(\s|$)/)
+      ymls = split_content.map { | manifest |
         #TODO strip out NOTES
-        YAML.parse(template)
+        YAML.parse(manifest)
         # compact seems to have problems with yaml::any
       }.reject{|x|x==nil}
-      Log.debug { "read_template ymls: #{ymls}" }
+      Log.debug { "manifest_string_to_ymls:\n #{ymls}" }
       ymls
     end
+
+    def self.combine_ymls_as_manifest_string(ymls : Array(YAML::Any)) : String
+      manifest = ymls.map do |yaml_object|
+        yaml_object.to_yaml
+      end.join
+
+      Log.debug { "combine_ymls_as_manifest:\n #{manifest}" }
+      manifest
+    end
 
     def self.manifest_ymls_from_file_list(manifest_file_list)
       ymls = manifest_file_list.map do |x|
-        parse_manifest_as_ymls(x)
+        manifest_path_to_ymls(x)
       end
       ymls.flatten
     end
@@ -47,5 +61,53 @@ module CNFInstall
       Log.debug { "manifest_containers: #{manifest_yml}" }
       manifest_yml.dig?("spec", "template", "spec", "containers")
     end
+
+    def self.add_namespace_to_resource(manifest_string, namespace)
+      namespaced_resources = KubectlClient::ShellCmd.run("kubectl api-resources --namespaced=true --no-headers | awk '{print $(NF)}'", "", false).[:output]
+      list_of_namespaced_resources = namespaced_resources.split("\n").select { |item| !item.empty? }
+      parsed_manifest = manifest_string_to_ymls(manifest_string)
+      ymls = [] of YAML::Any
+      parsed_manifest.each do |resource|
+        if resource["kind"].as_s.in?(list_of_namespaced_resources)
+          Helm.ensure_resource_with_namespace(resource, namespace)
+          ymls << resource
+        else
+          ymls << resource
+        end
+      end
+      string_manifest_with_namespaces = combine_ymls_as_manifest_string(ymls)
+    end
+
+    def self.add_manifest_to_file(deployment_name : String, manifest : String, destination_file)
+      File.open(destination_file, "a+") do |file|
+        file.puts manifest
+        Log.info { "#{deployment_name} manifest was appended into #{destination_file} file" }
+      end
+    end
+
+    def self.generate_common_manifest(config, deployment_name, namespace)
+      manifest_generated_successfully = true
+      case config.dynamic.install_method[0]
+      when CNFInstall::InstallMethod::ManifestDirectory
+        destination_cnf_dir = config.dynamic.destination_cnf_dir
+        manifest_directory = config.deployments.get_deployment_param(:manifest_directory)
+        list_of_manifests = manifest_file_list( destination_cnf_dir + "/" + manifest_directory )
+        list_of_manifests.each do |manifest_path|
+          manifest = File.read(manifest_path)
+          add_manifest_to_file(deployment_name, manifest, COMMON_MANIFEST_FILE_PATH)
+        end
+
+      when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory
+        begin
+          generated_manifest = Helm.generate_manifest(deployment_name, namespace)
+          generated_manifest_with_namespaces = add_namespace_to_resource(generated_manifest, namespace)
+          add_manifest_to_file(deployment_name, generated_manifest_with_namespaces, COMMON_MANIFEST_FILE_PATH)
+        rescue ex : Helm::ManifestGenerationError
+          Log.for("generate_common_manifest").error { ex.message }
+          manifest_generated_successfully = false
+        end
+      end
+      manifest_generated_successfully
+    end
   end
 end
diff --git a/src/tasks/utils/cnf_manager.cr b/src/tasks/utils/cnf_manager.cr
@@ -26,36 +26,19 @@ module CNFManager
 
   def self.cnf_resource_ymls(args, config)
     Log.info { "cnf_resource_ymls" }
-    template_ymls = [] of YAML::Any
-    case config.dynamic.install_method[0]
-    when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory
-      helm_chart_path = CNFInstall::Config.get_helm_chart_path(config)
-      generated_manifest_file_path = CNFInstall::Config.get_manifest_file_path(config)
-      Log.info { "EXPORTED CHART PATH: #{helm_chart_path}" } 
-      Helm.generate_manifest_from_templates(release_name: config.deployments.get_deployment_param(:name),
-                                            helm_chart: helm_chart_path,
-                                            output_file: generated_manifest_file_path,
-                                            namespace: CNFManager.get_deployment_namespace(config),
-                                            helm_values: config.deployments.get_deployment_param(:helm_values))
-      template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(generated_manifest_file_path)
-
-    when CNFInstall::InstallMethod::ManifestDirectory
-      template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list(
-        CNFInstall::Manifest.manifest_file_list(File.join(config.dynamic.destination_cnf_dir, config.deployments.get_deployment_param(:manifest_directory)))
-      )
-    end
+    manifest_ymls = CNFInstall::Manifest.manifest_path_to_ymls(COMMON_MANIFEST_FILE_PATH)
 
-    template_ymls = template_ymls.reject! {|x|
+    manifest_ymls = manifest_ymls.reject! {|x|
       # reject resources that contain the 'helm.sh/hook: test' annotation
       x.dig?("metadata","annotations","helm.sh/hook")
     }
-    Log.debug { "template_ymls: #{template_ymls}" }
-    template_ymls 
+    Log.debug { "cnf_resource_ymls: #{manifest_ymls}" }
+    manifest_ymls 
   end
 
   def self.cnf_resources(args, config, &block)
-    template_ymls = cnf_resource_ymls(args, config)
-    resource_resp = template_ymls.map do | resource |
+    manifest_ymls = cnf_resource_ymls(args, config)
+    resource_resp = manifest_ymls.map do | resource |
       resp = yield resource
       Log.debug { "cnf_workload_resource yield resp: #{resp}" }
       resp
@@ -96,9 +79,9 @@ module CNFManager
 
   def self.cnf_workload_resources(args, config, &block)
     deployment_namespace = CNFManager.get_deployment_namespace(config)
-    template_ymls = cnf_resource_ymls(args, config)
+    manifest_ymls = cnf_resource_ymls(args, config)
     # call cnf cnf_resources to get unfiltered yml
-    resource_ymls = Helm.all_workload_resources(template_ymls, deployment_namespace)
+    resource_ymls = Helm.all_workload_resources(manifest_ymls, deployment_namespace)
     resource_resp = resource_ymls.map do | resource |
       resp = yield resource
       Log.debug { "cnf_workload_resource yield resp: #{resp}" }
@@ -229,32 +212,6 @@ module CNFManager
     cnf_testsuite_helm_directory.split("/")[-1]
   end
 
-  #TODO move to helm module
-  def self.helm_template_header(helm_chart_or_directory : String, template_file="/tmp/temp_template.yml")
-    Log.info { "helm_template_header" }
-    Log.info { "helm_template_header helm_chart_or_directory: #{helm_chart_or_directory}" }
-    helm = Helm::BinarySingleton.helm
-    # generate helm chart release name
-    # use --dry-run to generate yml file
-    Helm.install("--dry-run --generate-name #{helm_chart_or_directory} > #{template_file}")
-    raw_template = File.read(template_file)
-    Log.debug { "raw_template: #{raw_template}" }
-    split_template = raw_template.split("---")
-    template_header = split_template[0]
-    parsed_template_header = YAML.parse(template_header)
-    Log.debug { "parsed_template_header: #{parsed_template_header}" }
-    parsed_template_header
-  end
-
-  #TODO move to helm module
-  def self.helm_chart_template_release_name(helm_chart_or_directory : String, template_file="/tmp/temp_template.yml")
-    Log.info { "helm_chart_template_release_name" }
-    Log.info { "helm_chart_template_release_name helm_chart_or_directory: #{helm_chart_or_directory}" }
-    hth = helm_template_header(helm_chart_or_directory, template_file)
-    Log.info { "helm template (should not be a full path): #{hth}" }
-    hth["NAME"]
-  end
-
   def self.config_source_dir(config_file)
     if File.directory?(config_file)
       config_file
@@ -536,6 +493,15 @@ module CNFManager
       else
         raise "Deployment method not found"
       end
+
+      #Generating manifest from installed CNF
+      #Returns true or false in case when manifest was generated successfully or not
+      manifest_generated_successfully = CNFInstall::Manifest.generate_common_manifest(config, release_name, deployment_namespace)
+
+      if !manifest_generated_successfully
+        stdout_failure "Manifest generation failed. Check CNF definition (helm charts, values, manifests, etc.)"
+        exit 1
+      end
 
       resource_ymls = cnf_workload_resources(nil, config) do |resource|
         resource
@@ -619,7 +585,7 @@ module CNFManager
     end
 
     if match[:found]
-      sleep 120
+      sleep(120.seconds)
       metrics_checkpoints = JaegerManager.unique_services_total
       Log.info { "metrics_checkpoints: #{metrics_checkpoints}" }
       tracing_used = JaegerManager.tracing_used?(baselines, metrics_checkpoints)
@@ -629,7 +595,7 @@ module CNFManager
     end
 
     if ORANMonitor.isCNFaRIC?(config)
-      sleep 30 
+      sleep(30.seconds) 
       e2_found = ORANMonitor.e2_session_established?(capture)
     else
       e2_found = false
@@ -736,6 +702,10 @@ module CNFManager
   def self.sample_cleanup(config_file, force=false, installed_from_manifest=false, verbose=true)
     Log.info { "sample_cleanup" }
     Log.info { "sample_cleanup installed_from_manifest: #{installed_from_manifest}" }
+
+    FileUtils.rm_rf(COMMON_MANIFEST_FILE_PATH)
+    Log.info { "#{COMMON_MANIFEST_FILE_PATH} file was removed." }
+
     config = CNFInstall::Config.parse_cnf_config_from_file(CNFManager.ensure_cnf_testsuite_yml_path(config_file))
     Log.for("verbose").info { "cleanup config: #{config.inspect}" } if verbose
     destination_cnf_dir = config.dynamic.destination_cnf_dir

diff --git a/src/tasks/utils/k8s_tshark.cr b/src/tasks/utils/k8s_tshark.cr
@@ -116,7 +116,7 @@ module K8sTshark
 
         # Some tshark captures were left in zombie states if only kill/kill -9 was invoked.
         ClusterTools.exec_by_node_bg("kill -15 #{@pid}", @node_match.not_nil!)
-        sleep 1
+        sleep(1.seconds)
         ClusterTools.exec_by_node_bg("kill -9 #{@pid}", @node_match.not_nil!)
 
         @pid = nil

diff --git a/src/tasks/utils/timeouts.cr b/src/tasks/utils/timeouts.cr
@@ -20,7 +20,7 @@ def repeat_with_timeout(timeout, errormsg, reset_on_nil=false, delay=2, &block)
     elsif result
       return true
     end
-    sleep delay
+    sleep(delay.seconds)
     Log.for("verbose").info { "Time left: #{timeout - (Time.utc - start_time).to_i} seconds" }
   end
   Log.error { errormsg }

diff --git a/src/tasks/workload/5g_validator.cr b/src/tasks/workload/5g_validator.cr
@@ -115,7 +115,7 @@ end
 
 def smf_up_heartbeat_capture_matches_count(smf_key : String, smf_value : String, command : String)
   K8sTshark::TsharkPacketCapture.begin_capture_by_label(smf_key, smf_value, command) do |capture|
-    sleep 60
+    sleep(60.seconds)
     capture.regex_search(/"pfcp\.msg_type": "(1|2)"/).size
   end
 end
@@ -135,7 +135,7 @@ task "suci_enabled" do |t, args|
       K8sTshark::TsharkPacketCapture.begin_capture_by_label(core_key, core_value, command) do |capture|
         #todo put in prereq
         UERANSIM.install(config)
-        sleep 30.0
+        sleep(30.0.seconds)
         # TODO 5g RAN (only) mobile traffic check ????
         # use suci encyption but don't use a null encryption key
         suci_found = capture.regex_match?(/"nas_5gs.mm.type_id": "1"/) && \

diff --git a/src/tasks/workload/microservice.cr b/src/tasks/workload/microservice.cr
@@ -371,7 +371,7 @@ task "zombie_handled" do |t, args|
       end
     end
 
-    sleep 10.0
+    sleep(10.0.seconds)
 
     task_response = CNFManager.workload_resource_test(args, config, check_containers:false ) do |resource, container, initialized|
       ClusterTools.all_containers_by_resource?(resource, resource[:namespace], only_container_pids:false) do | container_id, container_pid_on_node, node, container_proctree_statuses, container_status| 
@@ -565,7 +565,7 @@ task "sig_term_handled" do |t, args|
               Log.for(t.name).info { "pid_log_names: #{pid_log_names}" }
               #todo 2.3 parse the logs 
               #todo get the log
-              sleep 5
+              sleep(5.seconds)
               sig_term_found = pid_log_names.map do |pid_name|
                 Log.info { "pid_name: #{pid_name}" }
                 resp = File.read("#{pid_name}")