feat: add script to generate helm install command

SumoLogic · Jun 28, 2024 · 2370c99 · 2370c99
1 parent ce9dcc5
commit 2370c99
Show file tree

Hide file tree

Showing 4 changed files with 221 additions and 60 deletions.
diff --git a/scripts/get_image_config_keys.py b/scripts/get_image_config_keys.py
@@ -61,7 +61,6 @@ def extract_keys(dictionary: dict) -> list:
     "tailing-sidecar-operator.kubeRbacProxy.image.tag",
     "opentelemetry-operator.manager.image.repository.tag",
     "opentelemetry-operator.kubeRBACProxy.image.repository.tag",
-    "opentelemetry-operator.testFramework.image.repository.tag",
     "telegraf-operator.image.tag",
     "kube-prometheus-stack.prometheus.prometheusSpec.image.tag",
     "kube-prometheus-stack.prometheus.prometheusSpec.image.sha",
@@ -75,13 +74,7 @@ def extract_keys(dictionary: dict) -> list:
     "metrics-server.image.tag",
 ]
 
-not_needed_image_keys = [
-    "Percentage",
-    "falco",
-    "pullPolicy",
-    "pullSecrets",
-    "imagePullSecrets",
-]
+not_needed_image_keys = ["Percentage", "falco", "pullPolicy", "pullSecrets", "imagePullSecrets", "debug.sumologicMock.image", "otellogswindows", "testFramework"]
 needed_image_keys = ["image", "tag", "repository"]
 
 
@@ -117,6 +110,27 @@ def get_image_keys() -> list:
     return image_keys
 
 
+def generate_components_map(keys) -> list:
+    """Generates image configuration keys for components map used in update_images.py
+
+    Args:
+        keys (list): list of Helm Chart image configuration keys from values.yaml
+    Returns:
+        list: image configuration keys for components map used in update_images.py
+    """
+    unique_root_keys = set()
+    for key in keys:
+        keys_parts = key.split(".")[:-1]
+        new_key = ".".join(keys_parts)
+        unique_root_keys.add(f'"{new_key}": "",')
+
+    return sorted(unique_root_keys)
+
+
 if __name__ == "__main__":
+    print("---------- Helm Chart image configuration keys from values.yaml, used in watches.yaml ---------------")
     image_config_keys = get_image_keys()
     print("\n".join(image_config_keys))
+    print("---------- Image configuration keys for components map used in update_images.py ---------------------")
+    components_map_keys = generate_components_map(image_config_keys)
+    print("\n".join(sorted(components_map_keys)))
diff --git a/scripts/update_images.py b/scripts/update_images.py
@@ -4,21 +4,85 @@
 - bundle/manifests/operator.clusterserviceversion.yaml
 - config/manager/manager.yaml
 - tests/replace_components_images.sh
+- tests/helm_install.sh
 """
 
 import argparse
 import os
 import subprocess
 import yaml
 
+from get_image_config_keys import get_image_keys
+
 RED_HAT_REGISTRY = "registry.connect.redhat.com/sumologic/"
 PUBLIC_ECR_REGISTRY = "public.ecr.aws/sumologic/"
 ENV_PREFIX = "RELATED_IMAGE_"
 CLUSTER_SERVICE_VERSION_PATH = "bundle/manifests/operator.clusterserviceversion.yaml"
 MANAGER_PATH = "config/manager/manager.yaml"
 REPLACE_COMPONENTS_IMAGES_PATH = "tests/replace_components_images.sh"
+HELM_INSTALL_SCRIPT_PATH = "tests/helm_install.sh"
 BASH_HEADER = "#!/usr/bin/env bash\n\n"
 
+HELM_INSTALL_COMMAND_HEADER = """helm install test-openshift sumologic/sumologic \\
+  --set sumologic.accessId="dummy" \\
+  --set sumologic.accessKey="dummy" \\
+  --set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \\
+  --set sumologic.scc.create=true \\
+  --set fluent-bit.securityContext.privileged=true \\
+  --set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \\
+  --set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \\
+  --set fluentd.logs.containers.multiline.enabled=false \\
+  --set metrics-server.enabled=true \\
+  --set metrics-server.apiService.create=false \\
+  --set otelagent.enabled=true \\
+  --set telegraf-operator.enabled=true \\
+  --set falco.enabled=true \\
+  --set tailing-sidecar-operator.enabled=true \\
+  --set opentelemetry-operator.enabled=true \\
+  --version 2.19.1 \\
+  -n sumologic-system \\
+  --create-namespace -f \"${ROOT_DIR}/tests/values.yaml\" \\\n"""
+
+# COMPONENTS_CONFIG_MAP maps helm chart configuration keys into components names
+COMPONENTS_CONFIG_MAP = {
+    "instrumentation.instrumentationJobImage.image": "kubernetes-tools-kubectl",
+    "kube-prometheus-stack.kube-state-metrics.image": "kube-state-metrics",
+    "kube-prometheus-stack.prometheus-node-exporter.image": "node-exporter",
+    "kube-prometheus-stack.prometheus.prometheusSpec.image": "prometheus",
+    "kube-prometheus-stack.prometheusOperator.image": "prometheus-operator",
+    "kube-prometheus-stack.prometheusOperator.prometheusConfigReloader.image": "prometheus-config-reloader",
+    "kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage": "prometheus-config-reloader",
+    "kube-prometheus-stack.prometheusOperator.thanosImage": "thanos",
+    "metadata.image": "sumologic-otel-collector",
+    "metrics-server.image": "metrics-server",
+    "opentelemetry-operator.kubeRBACProxy.image": "kube-rbac-proxy",
+    "opentelemetry-operator.kubeRBACProxy.image.repository": "kube-rbac-proxy",
+    "opentelemetry-operator.manager.autoInstrumentationImage.dotnet": "autoinstrumentation-dotnet",
+    "opentelemetry-operator.manager.autoInstrumentationImage.java": "autoinstrumentation-java",
+    "opentelemetry-operator.manager.autoInstrumentationImage.nodejs": "autoinstrumentation-nodejs",
+    "opentelemetry-operator.manager.autoInstrumentationImage.python": "autoinstrumentation-nodejs",
+    "opentelemetry-operator.manager.collectorImage": "sumologic-otel-collector",
+    "opentelemetry-operator.manager.image": "opentelemetry-operator",
+    "opentelemetry-operator.manager.image.repository": "opentelemetry-operator",
+    "otelcolInstrumentation.statefulset.image": "sumologic-otel-collector",
+    "otelevents.image": "sumologic-otel-collector",
+    "otellogs.daemonset.initContainers.changeowner.image": "busybox",
+    "otellogs.image": "sumologic-otel-collector",
+    "pvcCleaner.job.image": "kubernetes-tools-kubectl",
+    "sumologic.metrics.collector.otelcol.image": "sumologic-otel-collector",
+    "sumologic.metrics.remoteWriteProxy.image": "nginx-unprivileged",
+    "sumologic.otelcolImage": "sumologic-otel-collector",
+    "sumologic.setup.job.image": "kubernetes-setup",
+    "sumologic.setup.job.initContainerImage": "busybox",
+    "tailing-sidecar-operator.kubeRbacProxy.image": "kube-rbac-proxy",
+    "tailing-sidecar-operator.operator.image": "tailing-sidecar-operator",
+    "tailing-sidecar-operator.sidecar.image": "tailing-sidecar",
+    "telegraf-operator.image": "telegraf-operator",
+    "telegraf-operator.image.sidecarImage": "telegraf",
+    "tracesGateway.deployment.image": "sumologic-otel-collector",
+    "tracesSampler.deployment.image": "sumologic-otel-collector",
+}
+
 
 def pairwise(iterable: list) -> list:
     """Transforms list in following way:
@@ -213,6 +277,82 @@ def update_replace_components_images(image_file_path: str, create_new_file: bool
                 new_file.write(f"{cmd}\n")
 
 
+def prepare_components_images_map(file_path: str) -> dict:
+    """Prepares components dict containing information about container images
+
+    Args:
+        file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh
+
+    Returns
+        dict: dict with information about container images
+    """
+    components_images = {}
+    for image_with_tag, image_with_sha256 in pairwise(get_lines(file_path)):
+        component, tag = image_with_tag.removeprefix(RED_HAT_REGISTRY).split(":")
+        sha = image_with_sha256.split(":")[-1]
+        components_images[component] = {"image_with_tag": image_with_tag, "image_with_sha256": image_with_sha256, "tag": tag, "sha": sha}
+    return components_images
+
+
+def update_helm_install(image_file_path: str):
+    """Updates helm install command in tests/helm_install.sh"
+
+    Args:
+        file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh
+    """
+    # pylint: disable=R0912
+    image_config_keys = get_image_keys()
+    components_images = prepare_components_images_map(image_file_path)
+    set_args = []
+
+    for image_config in image_config_keys:
+        image_config_root = ".".join(image_config.split(".")[:-1])
+
+        if image_config_root not in COMPONENTS_CONFIG_MAP:
+            print(f"WARNING: missing key in components map, key: {image_config_root}")
+            continue
+
+        component = COMPONENTS_CONFIG_MAP[image_config_root]
+
+        if component not in components_images:
+            print(f"WARNING: missing key in components_images, key: {component}")
+            set_arg = f"  --set {image_config}='' \\"
+            set_args.append(set_arg)
+            continue
+
+        config = image_config.split(".")[-1]
+        if config == "repository":
+            if image_config_root + ".registry" not in image_config_keys:
+                set_arg = f"  --set {image_config}={PUBLIC_ECR_REGISTRY}{component}@sha256 \\"
+            else:
+                set_arg = f"  --set {image_config}={component}@sha256 \\"
+        elif config == "tag":
+            if image_config_root + ".sha" not in image_config_keys:
+                tag = components_images[component]["sha"]
+            else:
+                tag = components_images[component]["tag"]
+
+            set_arg = f"  --set {image_config}={tag} \\"
+        elif config == "registry":
+            registry = PUBLIC_ECR_REGISTRY[:-1]
+            set_arg = f"  --set {image_config}={registry} \\"
+        elif config == "sha":
+            sha = components_images[component]["sha"]
+            set_arg = f"  --set {image_config}={sha} \\"
+        elif config == "sidecarImage":
+            # special case for telegraf-operator.image.sidecarImage
+            component = component.removesuffix("-operator")
+            sidecar = components_images[component]["image_with_sha256"]
+            set_arg = f"  --set {image_config}={sidecar} \\"
+        else:
+            set_arg = f"  --set {image_config}='' \\"
+        set_args.append(set_arg)
+
+    helm_install_cmd = HELM_INSTALL_COMMAND_HEADER + "\n".join(set_args)
+    # TODO: save to file
+    print(helm_install_cmd)
+
+
 def parse_args():
     """Parses command line arguments"""
     parser = argparse.ArgumentParser()
@@ -246,3 +386,5 @@ def parse_args():
     update_manager(m_path, image_envs_list, args.create_new_file)
 
     update_replace_components_images(args.images_file, args.create_new_file)
+
+    update_helm_install(args.images_file)
diff --git a/tests/deploy_helm_chart.sh b/tests/deploy_helm_chart.sh
@@ -13,58 +13,7 @@ readonly TIME=300
 helm repo add sumologic https://sumologic.github.io/sumologic-kubernetes-collection
 helm repo update
 
-helm install test-openshift sumologic/sumologic \
-  --set sumologic.accessId="dummy" \
-  --set sumologic.accessKey="dummy" \
-  --set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \
-  --set sumologic.scc.create=true \
-  --set fluent-bit.securityContext.privileged=true \
-  --set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \
-  --set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \
-  --set fluentd.logs.containers.multiline.enabled=false \
-  --set metrics-server.enabled=true \
-  --set metrics-server.apiService.create=false \
-  --set otelagent.enabled=true \
-  --set telegraf-operator.enabled=true \
-  --set falco.enabled=true \
-  --set tailing-sidecar-operator.enabled=true \
-  --set opentelemetry-operator.enabled=true \
-  --set fluentd.image.repository=public.ecr.aws/sumologic/kubernetes-fluentd@sha256 \
-  --set fluentd.image.tag=dca1e3854b2e7fd2540c53c8092e3105c6a4bdcce426035a2cd5fed28a8690bf \
-  --set metrics-server.image.registry=public.ecr.aws \
-  --set metrics-server.image.repository=sumologic/metrics-server@sha256 \
-  --set metrics-server.image.tag=c4a41f333bf942fa21c08d0c5b1b6b649f878341886ba59593b73d808ad9d3b0 \
-  --set fluent-bit.image.repository=public.ecr.aws/sumologic/fluent-bit@sha256 \
-  --set fluent-bit.image.tag=a8ead538bbba26e83c9f70137ee9ae6dac922f7844ebd3273d57178b214790ab \
-  --set kube-prometheus-stack.prometheusOperator.image.repository=public.ecr.aws/sumologic/prometheus-operator \
-  --set kube-prometheus-stack.prometheusOperator.image.tag=v0.44.0 \
-  --set kube-prometheus-stack.prometheusOperator.image.sha=27384cfcd3bf32bee7584332b82188a7da5780dfc33f33ea8aa3afd2c10ca948 \
-  --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.repository=public.ecr.aws/sumologic/prometheus-config-reloader \
-  --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.tag=v0.44.0 \
-  --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.sha=7fc45d6c4cfa93363751277d3b761d30b1857be70d2c83568cabc36b19f47914 \
-  --set kube-prometheus-stack.kube-state-metrics.image.repository=public.ecr.aws/sumologic/kube-state-metrics@sha256 \
-  --set kube-prometheus-stack.kube-state-metrics.image.tag=18fb3800783b26db6a131ab846a28186dc0e9c12401e72a12ad99bf11d33186a \
-  --set kube-prometheus-stack.prometheus-node-exporter.image.repository=public.ecr.aws/sumologic/node-exporter@sha256 \
-  --set kube-prometheus-stack.prometheus-node-exporter.image.tag=685b59ebf0ce3c7e32f9de83359fdfc66a5143660f96b82c8ef8964c727bb2e5 \
-  --set kube-prometheus-stack.prometheus.prometheusSpec.image.repository=public.ecr.aws/sumologic/prometheus \
-  --set kube-prometheus-stack.prometheus.prometheusSpec.image.tag=v2.22.1 \
-  --set kube-prometheus-stack.prometheus.prometheusSpec.image.sha=47004a8f74aab2353e8d0ac5725e4206a53305e7766a19b1cbb1dd770c2e8e36 \
-  --set kube-prometheus-stack.prometheus.prometheusSpec.thanos.image=public.ecr.aws/sumologic/thanos@sha256:3f5b6df4e423475f912cb2e48e55e89235c68b473b53e202b246e024307ea965 \
-  --set telegraf-operator.image.repository=public.ecr.aws/sumologic/telegraf-operator-ubi \
-  --set telegraf-operator.image.sidecarImage=public.ecr.aws/sumologic/telegraf@sha256:f9883d1e9e0baf0d326a7c2c5503e011b36bf0bff22d6470c2f6d2b39fdd11fb \
-  --set tailing-sidecar-operator.operator.image.repository=ghcr.io/sumologic/tailing-sidecar-operator@sha256 \
-  --set tailing-sidecar-operator.operator.image.tag=e8b1d815666b69ab3047bc1d81afb51fb03caa99ca382c3f81949488072cd748 \
-  --set tailing-sidecar-operator.sidecar.image.repository=ghcr.io/sumologic/tailing-sidecar@sha256 \
-  --set tailing-sidecar-operator.sidecar.image.tag=3a9a741cbc10298d9be5bd68367883de45ad887e63ce12fedaf4024da807b82f \
-  --set tailing-sidecar-operator.kubeRbacProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \
-  --set tailing-sidecar-operator.kubeRbacProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \
-  --set opentelemetry-operator.manager.image.repository=public.ecr.aws/sumologic/opentelemetry-operator@sha256 \
-  --set opentelemetry-operator.manager.image.tag=b6fc0d5880016a8dc51a371839d0336409ad242f3ef046ca877c5d2c9df7e43e \
-  --set opentelemetry-operator.kubeRBACProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \
-  --set opentelemetry-operator.kubeRBACProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \
-  --version 2.19.1 \
-  -n sumologic-system \
-  --create-namespace -f "${ROOT_DIR}/tests/values.yaml"
+./tests/helm_install.sh
 
 wait_for_collection_resources "${NAMESPACE}" "${TIME}"
 

diff --git a/tests/helm_install.sh b/tests/helm_install.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+readonly ROOT_DIR="$(dirname "$(dirname "${0}")")"
+
+helm install test-openshift sumologic/sumologic \
+  --set sumologic.accessId="dummy" \
+  --set sumologic.accessKey="dummy" \
+  --set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \
+  --set sumologic.scc.create=true \
+  --set fluent-bit.securityContext.privileged=true \
+  --set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \
+  --set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \
+  --set fluentd.logs.containers.multiline.enabled=false \
+  --set metrics-server.enabled=true \
+  --set metrics-server.apiService.create=false \
+  --set otelagent.enabled=true \
+  --set telegraf-operator.enabled=true \
+  --set falco.enabled=true \
+  --set tailing-sidecar-operator.enabled=true \
+  --set opentelemetry-operator.enabled=true \
+  --set fluentd.image.repository=public.ecr.aws/sumologic/kubernetes-fluentd@sha256 \
+  --set fluentd.image.tag=dca1e3854b2e7fd2540c53c8092e3105c6a4bdcce426035a2cd5fed28a8690bf \
+  --set metrics-server.image.registry=public.ecr.aws \
+  --set metrics-server.image.repository=sumologic/metrics-server@sha256 \
+  --set metrics-server.image.tag=c4a41f333bf942fa21c08d0c5b1b6b649f878341886ba59593b73d808ad9d3b0 \
+  --set fluent-bit.image.repository=public.ecr.aws/sumologic/fluent-bit@sha256 \
+  --set fluent-bit.image.tag=a8ead538bbba26e83c9f70137ee9ae6dac922f7844ebd3273d57178b214790ab \
+  --set kube-prometheus-stack.prometheusOperator.image.repository=public.ecr.aws/sumologic/prometheus-operator \
+  --set kube-prometheus-stack.prometheusOperator.image.tag=v0.44.0 \
+  --set kube-prometheus-stack.prometheusOperator.image.sha=27384cfcd3bf32bee7584332b82188a7da5780dfc33f33ea8aa3afd2c10ca948 \
+  --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.repository=public.ecr.aws/sumologic/prometheus-config-reloader \
+  --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.tag=v0.44.0 \
+  --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.sha=7fc45d6c4cfa93363751277d3b761d30b1857be70d2c83568cabc36b19f47914 \
+  --set kube-prometheus-stack.kube-state-metrics.image.repository=public.ecr.aws/sumologic/kube-state-metrics@sha256 \
+  --set kube-prometheus-stack.kube-state-metrics.image.tag=18fb3800783b26db6a131ab846a28186dc0e9c12401e72a12ad99bf11d33186a \
+  --set kube-prometheus-stack.prometheus-node-exporter.image.repository=public.ecr.aws/sumologic/node-exporter@sha256 \
+  --set kube-prometheus-stack.prometheus-node-exporter.image.tag=685b59ebf0ce3c7e32f9de83359fdfc66a5143660f96b82c8ef8964c727bb2e5 \
+  --set kube-prometheus-stack.prometheus.prometheusSpec.image.repository=public.ecr.aws/sumologic/prometheus \
+  --set kube-prometheus-stack.prometheus.prometheusSpec.image.tag=v2.22.1 \
+  --set kube-prometheus-stack.prometheus.prometheusSpec.image.sha=47004a8f74aab2353e8d0ac5725e4206a53305e7766a19b1cbb1dd770c2e8e36 \
+  --set kube-prometheus-stack.prometheus.prometheusSpec.thanos.image=public.ecr.aws/sumologic/thanos@sha256:3f5b6df4e423475f912cb2e48e55e89235c68b473b53e202b246e024307ea965 \
+  --set telegraf-operator.image.repository=public.ecr.aws/sumologic/telegraf-operator-ubi \
+  --set telegraf-operator.image.sidecarImage=public.ecr.aws/sumologic/telegraf@sha256:f9883d1e9e0baf0d326a7c2c5503e011b36bf0bff22d6470c2f6d2b39fdd11fb \
+  --set tailing-sidecar-operator.operator.image.repository=ghcr.io/sumologic/tailing-sidecar-operator@sha256 \
+  --set tailing-sidecar-operator.operator.image.tag=e8b1d815666b69ab3047bc1d81afb51fb03caa99ca382c3f81949488072cd748 \
+  --set tailing-sidecar-operator.sidecar.image.repository=ghcr.io/sumologic/tailing-sidecar@sha256 \
+  --set tailing-sidecar-operator.sidecar.image.tag=3a9a741cbc10298d9be5bd68367883de45ad887e63ce12fedaf4024da807b82f \
+  --set tailing-sidecar-operator.kubeRbacProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \
+  --set tailing-sidecar-operator.kubeRbacProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \
+  --set opentelemetry-operator.manager.image.repository=public.ecr.aws/sumologic/opentelemetry-operator@sha256 \
+  --set opentelemetry-operator.manager.image.tag=b6fc0d5880016a8dc51a371839d0336409ad242f3ef046ca877c5d2c9df7e43e \
+  --set opentelemetry-operator.kubeRBACProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \
+  --set opentelemetry-operator.kubeRBACProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \
+  --version 2.19.1 \
+  -n sumologic-system \
+  --create-namespace -f "${ROOT_DIR}/tests/values.yaml"