Feature: Group several manifest into one file

- group manifests from several CNFs into one common_manifests.yaml file - add functionality in cleanup.cr to remove this file - depends on PR in helm library: cnf-testsuite/helm#4 - common_manifest functionality: used common_manifest instead of templates - remove methods related to templates Signed-off-by: barmull <[email protected]>
cnti-testcatalog · Sep 27, 2024 · 8bdf96c · 8bdf96c
1 parent bfe48fc
commit 8bdf96c
Show file tree

Hide file tree

Showing 5 changed files with 75 additions and 69 deletions.
diff --git a/shard.lock b/shard.lock
@@ -18,7 +18,7 @@ shards:
 
   find:
     git: https://github.com/cnf-testsuite/find.git
-    version: 0.1.0+git.commit.129096c08d84adb1b741aa8a5ee132d6f2a87e02
+    version: 0.1.0
 
   git:
     git: https://github.com/cnf-testsuite/git.git
@@ -29,19 +29,19 @@ shards:
     version: 0.12.1
 
   helm:
-    git: https://github.com/cnf-testsuite/helm.git
+    git: https://github.com/barmull/helm
     version: 1.0.2
 
   icr:
     git: https://github.com/crystal-community/icr.git
-    version: 0.9.0+git.commit.f62bfcdfbe65ee31c46c3d9951cee08ac2e0bee0
+    version: 0.9.0
 
   k8s_kernel_introspection:
     git: https://github.com/cnf-testsuite/k8s_kernel_introspection.git
     version: 1.0.3
 
   k8s_netstat:
-    git: https://github.com/cnf-testsuite/k8s_netstat.git
+    git: https://github.com/barmull/k8s_netstat
     version: 1.0.1
 
   kernel_introspection:
@@ -62,23 +62,23 @@ shards:
 
   readline:
     git: https://github.com/crystal-lang/crystal-readline.git
-    version: 0.1.1+git.commit.69ecf33d7cad5568d7d19333510cfd9d17cb1bbd
+    version: 0.1.1
 
   release_manager:
     git: https://github.com/cnf-testsuite/release_manager.git
-    version: 0.1.0+git.commit.a1d7b3568d3112f737ab3ff4a7bae69a6b86970a
+    version: 0.1.0
 
   retriable:
     git: https://github.com/sija/retriable.cr.git
     version: 0.2.5
 
   sam:
     git: https://github.com/vulk/sam.cr.git
-    version: 0.4.0+git.commit.4e3b271d31d7fd3c3ca2b0c1ff4943b86dc27021
+    version: 0.4.0
 
   tar:
     git: https://github.com/cnf-testsuite/tar.git
-    version: 0.1.0+git.commit.ae9bbea1402eded7b411368336653e4ad822003a
+    version: 0.1.0
 
   totem:
     git: https://github.com/icyleaf/totem.git

diff --git a/shard.yml b/shard.yml
@@ -60,10 +60,12 @@ dependencies:
     github: cnf-testsuite/k8s_kernel_introspection
     version: ~> 1.0.3
   helm:
-    github: cnf-testsuite/helm
+    github: barmull/helm
+    branch: main
     version: ~> 1.0.1
   k8s_netstat:
-    github: cnf-testsuite/k8s_netstat
+    github: barmull/k8s_netstat
+    branch: main
     version: ~> 1.0.1
   release_manager:
     github: cnf-testsuite/release_manager

diff --git a/src/tasks/cleanup.cr b/src/tasks/cleanup.cr
@@ -40,6 +40,12 @@ task "samples_cleanup" do  |_, args|
     )
     nil
   end
+  # Remove common_manifest.yaml file
+  common_manifest_path = "cnfs/common_manifest.yml"
+  if File.exists?(common_manifest_path)
+    File.delete(common_manifest_path)
+    Log.info { "#{common_manifest_path} file deleted successfully." }
+  end
 end
 
 desc "Cleans up the CNF Test Suite helper tools and containers"

diff --git a/src/tasks/utils/cnf_installation/manifest.cr b/src/tasks/utils/cnf_installation/manifest.cr
@@ -1,15 +1,15 @@
 module CNFInstall
   module Manifest
-    def self.parse_manifest_as_ymls(template_file_name="cnfs/temp_template.yml")
-      Log.info { "parse_manifest_as_ymls template_file_name: #{template_file_name}" }
-      templates = File.read(template_file_name)
-      split_template = templates.split(/(\s|^)---(\s|$)/)
-      ymls = split_template.map { | template |
+    def self.parse_manifest_as_ymls(file_name)
+      Log.info { "parse_manifest_as_ymls file_name: #{file_name}" }
+      file_content = File.read(file_name)
+      split_content = file_content.split(/(\s|^)---(\s|$)/)
+      ymls = split_content.map { | manifest |
         #TODO strip out NOTES
-        YAML.parse(template)
+        YAML.parse(manifest)
         # compact seems to have problems with yaml::any
       }.reject{|x|x==nil}
-      Log.debug { "read_template ymls: #{ymls}" }
+      Log.debug { "ymls:\n #{ymls}" }
       ymls
     end
 
@@ -47,5 +47,42 @@ module CNFInstall
       Log.debug { "manifest_containers: #{manifest_yml}" }
       manifest_yml.dig?("spec", "template", "spec", "containers")
     end
+
+    def self.add_manifest_to_file(release_name : String, manifest : String, destination_file = "cnfs/common_manifest.yml")
+      if File.exists?(destination_file)
+        File.open(destination_file, "a") do |file|
+          file.puts manifest
+          Log.info { "#{release_name} manifest was appended into #{destination_file} file" }
+        end
+      else
+        File.open(destination_file, "w") do |file|
+          file.puts manifest
+          Log.info { "Created #{destination_file} file with #{release_name} manifest." }
+        end
+      end
+    end
+
+    def self.generate_manifest(config, release_name, namespace)
+      install_method = config.dynamic.install_method
+      case install_method[0]
+      when CNFInstall::InstallMethod::ManifestDirectory
+        destination_cnf_dir = config.dynamic.destination_cnf_dir
+        manifest_directory = config.deployments.get_deployment_param(:manifest_directory)
+        list_of_manifests = manifest_file_list( destination_cnf_dir + "/" + manifest_directory )
+        list_of_manifests.each do |manifest_path|
+          manifest = File.read(manifest_path)
+          add_manifest_to_file(release_name: release_name, manifest: manifest)
+        end
+
+      when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory
+        begin
+          generated_manifest = Helm.generate_manifest(release_name, namespace)
+          add_manifest_to_file(release_name: release_name, manifest: generated_manifest)
+        rescue ex : Helm::ManifestGenerationError
+          Log.error { ex.message.colorize(:red) }
+          exit 1
+        end
+      end
+    end
   end
 end
diff --git a/src/tasks/utils/cnf_manager.cr b/src/tasks/utils/cnf_manager.cr
@@ -26,36 +26,20 @@ module CNFManager
 
   def self.cnf_resource_ymls(args, config)
     Log.info { "cnf_resource_ymls" }
-    template_ymls = [] of YAML::Any
-    case config.dynamic.install_method[0]
-    when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory
-      helm_chart_path = CNFInstall::Config.get_helm_chart_path(config)
-      generated_manifest_file_path = CNFInstall::Config.get_manifest_file_path(config)
-      Log.info { "EXPORTED CHART PATH: #{helm_chart_path}" } 
-      Helm.generate_manifest_from_templates(release_name: config.deployments.get_deployment_param(:name),
-                                            helm_chart: helm_chart_path,
-                                            output_file: generated_manifest_file_path,
-                                            namespace: CNFManager.get_deployment_namespace(config),
-                                            helm_values: config.deployments.get_deployment_param(:helm_values))
-      template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(generated_manifest_file_path)
-
-    when CNFInstall::InstallMethod::ManifestDirectory
-      template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list(
-        CNFInstall::Manifest.manifest_file_list(config.dynamic.destination_cnf_dir + "/" + config.deployments.get_deployment_param(:manifest_directory))
-      )
-    end
+    common_manifest_file_path = "cnfs/common_manifest.yml"
+    manifest_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(common_manifest_file_path)
 
-    template_ymls = template_ymls.reject! {|x|
+    manifest_ymls = manifest_ymls.reject! {|x|
       # reject resources that contain the 'helm.sh/hook: test' annotation
       x.dig?("metadata","annotations","helm.sh/hook")
     }
-    Log.debug { "template_ymls: #{template_ymls}" }
-    template_ymls 
+    Log.debug { "manifest_ymls: #{manifest_ymls}" }
+    manifest_ymls 
   end
 
   def self.cnf_resources(args, config, &block)
-    template_ymls = cnf_resource_ymls(args, config)
-    resource_resp = template_ymls.map do | resource |
+    manifest_ymls = cnf_resource_ymls(args, config)
+    resource_resp = manifest_ymls.map do | resource |
       resp = yield resource
       Log.debug { "cnf_workload_resource yield resp: #{resp}" }
       resp
@@ -78,7 +62,7 @@ module CNFManager
   def self.get_deployment_namespace(config)
     install_method = config.dynamic.install_method
     case install_method[0]
-    when CNFInstall::InstallMethod::HelmChart, Helm::InstallMethod::HelmDirectory
+    when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory
       config_namespace = config.deployments.get_deployment_param(:namespace)
       if !config_namespace.empty?
         Log.info { "deployment namespace was set to: #{config_namespace}" }
@@ -96,9 +80,9 @@ module CNFManager
 
   def self.cnf_workload_resources(args, config, &block)
     deployment_namespace = CNFManager.get_deployment_namespace(config)
-    template_ymls = cnf_resource_ymls(args, config)
+    manifest_ymls = cnf_resource_ymls(args, config)
     # call cnf cnf_resources to get unfiltered yml
-    resource_ymls = Helm.all_workload_resources(template_ymls, deployment_namespace)
+    resource_ymls = Helm.all_workload_resources(manifest_ymls, deployment_namespace)
     resource_resp = resource_ymls.map do | resource |
       resp = yield resource
       Log.debug { "cnf_workload_resource yield resp: #{resp}" }
@@ -229,32 +213,6 @@ module CNFManager
     cnf_testsuite_helm_directory.split("/")[-1]
   end
 
-  #TODO move to helm module
-  def self.helm_template_header(helm_chart_or_directory : String, template_file="/tmp/temp_template.yml")
-    Log.info { "helm_template_header" }
-    Log.info { "helm_template_header helm_chart_or_directory: #{helm_chart_or_directory}" }
-    helm = Helm::BinarySingleton.helm
-    # generate helm chart release name
-    # use --dry-run to generate yml file
-    Helm.install("--dry-run --generate-name #{helm_chart_or_directory} > #{template_file}")
-    raw_template = File.read(template_file)
-    Log.debug { "raw_template: #{raw_template}" }
-    split_template = raw_template.split("---")
-    template_header = split_template[0]
-    parsed_template_header = YAML.parse(template_header)
-    Log.debug { "parsed_template_header: #{parsed_template_header}" }
-    parsed_template_header
-  end
-
-  #TODO move to helm module
-  def self.helm_chart_template_release_name(helm_chart_or_directory : String, template_file="/tmp/temp_template.yml")
-    Log.info { "helm_chart_template_release_name" }
-    Log.info { "helm_chart_template_release_name helm_chart_or_directory: #{helm_chart_or_directory}" }
-    hth = helm_template_header(helm_chart_or_directory, template_file)
-    Log.info { "helm template (should not be a full path): #{hth}" }
-    hth["NAME"]
-  end
-
   def self.config_source_dir(config_file)
     if File.directory?(config_file)
       config_file
@@ -536,6 +494,9 @@ module CNFManager
       else
         raise "Deployment method not found"
       end
+
+      #Generating manifest from installed CNF
+      CNFInstall::Manifest.generate_manifest(config, release_name, deployment_namespace)
 
       resource_ymls = cnf_workload_resources(nil, config) do |resource|
         resource