diff --git a/scripts/get_image_config_keys.py b/scripts/get_image_config_keys.py old mode 100644 new mode 100755 index 7458dfa..9ae8d67 --- a/scripts/get_image_config_keys.py +++ b/scripts/get_image_config_keys.py @@ -61,7 +61,6 @@ def extract_keys(dictionary: dict) -> list: "tailing-sidecar-operator.kubeRbacProxy.image.tag", "opentelemetry-operator.manager.image.repository.tag", "opentelemetry-operator.kubeRBACProxy.image.repository.tag", - "opentelemetry-operator.testFramework.image.repository.tag", "telegraf-operator.image.tag", "kube-prometheus-stack.prometheus.prometheusSpec.image.tag", "kube-prometheus-stack.prometheus.prometheusSpec.image.sha", @@ -75,13 +74,7 @@ def extract_keys(dictionary: dict) -> list: "metrics-server.image.tag", ] -not_needed_image_keys = [ - "Percentage", - "falco", - "pullPolicy", - "pullSecrets", - "imagePullSecrets", -] +not_needed_image_keys = ["Percentage", "falco", "pullPolicy", "pullSecrets", "imagePullSecrets", "debug.sumologicMock.image", "otellogswindows", "testFramework"] needed_image_keys = ["image", "tag", "repository"] @@ -117,6 +110,27 @@ def get_image_keys() -> list: return image_keys +def generate_components_map(keys) -> list: + """Generates image configuration keys for components map used in update_images.py + + Args: + keys (list): list of Helm Chart image configuration keys from values.yaml + Returns: + list: image configuration keys for components map used in update_images.py + """ + unique_root_keys = set() + for key in keys: + keys_parts = key.split(".")[:-1] + new_key = ".".join(keys_parts) + unique_root_keys.add(f'"{new_key}": "",') + + return sorted(unique_root_keys) + + if __name__ == "__main__": + print("---------- Helm Chart image configuration keys from values.yaml, used in watches.yaml ---------------") image_config_keys = get_image_keys() print("\n".join(image_config_keys)) + print("---------- Image configuration keys for components map used in update_images.py ---------------------") + components_map_keys = generate_components_map(image_config_keys) + print("\n".join(sorted(components_map_keys))) diff --git a/scripts/update_images.py b/scripts/update_images.py index 49cc8c2..e71cbb1 100755 --- a/scripts/update_images.py +++ b/scripts/update_images.py @@ -4,6 +4,7 @@ - bundle/manifests/operator.clusterserviceversion.yaml - config/manager/manager.yaml - tests/replace_components_images.sh +- tests/helm_install.sh """ import argparse @@ -11,14 +12,79 @@ import subprocess import yaml +from get_image_config_keys import get_image_keys + RED_HAT_REGISTRY = "registry.connect.redhat.com/sumologic/" PUBLIC_ECR_REGISTRY = "public.ecr.aws/sumologic/" ENV_PREFIX = "RELATED_IMAGE_" CLUSTER_SERVICE_VERSION_PATH = "bundle/manifests/operator.clusterserviceversion.yaml" MANAGER_PATH = "config/manager/manager.yaml" REPLACE_COMPONENTS_IMAGES_PATH = "tests/replace_components_images.sh" +HELM_INSTALL_SCRIPT_PATH = "tests/helm_install.sh" BASH_HEADER = "#!/usr/bin/env bash\n\n" +HELM_INSTALL_COMMAND_HEADER = """readonly ROOT_DIR="$(dirname "$(dirname "${0}")")" + +helm install test-openshift sumologic/sumologic \\ + --set sumologic.accessId="dummy" \\ + --set sumologic.accessKey="dummy" \\ + --set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \\ + --set sumologic.scc.create=true \\ + --set fluent-bit.securityContext.privileged=true \\ + --set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \\ + --set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \\ + --set fluentd.logs.containers.multiline.enabled=false \\ + --set metrics-server.enabled=true \\ + --set metrics-server.apiService.create=false \\ + --set otelagent.enabled=true \\ + --set telegraf-operator.enabled=true \\ + --set falco.enabled=true \\ + --set tailing-sidecar-operator.enabled=true \\ + --set opentelemetry-operator.enabled=true \\ + --version 2.19.1 \\ + -n sumologic-system \\ + --create-namespace -f "${ROOT_DIR}/tests/values.yaml" \\\n""" + +# COMPONENTS_CONFIG_MAP maps helm chart configuration keys into components names +COMPONENTS_CONFIG_MAP = { + "instrumentation.instrumentationJobImage.image": "kubernetes-tools-kubectl", + "kube-prometheus-stack.kube-state-metrics.image": "kube-state-metrics", + "kube-prometheus-stack.prometheus-node-exporter.image": "node-exporter", + "kube-prometheus-stack.prometheus.prometheusSpec.image": "prometheus", + "kube-prometheus-stack.prometheusOperator.image": "prometheus-operator", + "kube-prometheus-stack.prometheusOperator.prometheusConfigReloader.image": "prometheus-config-reloader", + "kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage": "prometheus-config-reloader", + "kube-prometheus-stack.prometheusOperator.thanosImage": "thanos", + "metadata.image": "sumologic-otel-collector", + "metrics-server.image": "metrics-server", + "opentelemetry-operator.kubeRBACProxy.image": "kube-rbac-proxy", + "opentelemetry-operator.kubeRBACProxy.image.repository": "kube-rbac-proxy", + "opentelemetry-operator.manager.autoInstrumentationImage.dotnet": "autoinstrumentation-dotnet", + "opentelemetry-operator.manager.autoInstrumentationImage.java": "autoinstrumentation-java", + "opentelemetry-operator.manager.autoInstrumentationImage.nodejs": "autoinstrumentation-nodejs", + "opentelemetry-operator.manager.autoInstrumentationImage.python": "autoinstrumentation-nodejs", + "opentelemetry-operator.manager.collectorImage": "sumologic-otel-collector", + "opentelemetry-operator.manager.image": "opentelemetry-operator", + "opentelemetry-operator.manager.image.repository": "opentelemetry-operator", + "otelcolInstrumentation.statefulset.image": "sumologic-otel-collector", + "otelevents.image": "sumologic-otel-collector", + "otellogs.daemonset.initContainers.changeowner.image": "busybox", + "otellogs.image": "sumologic-otel-collector", + "pvcCleaner.job.image": "kubernetes-tools-kubectl", + "sumologic.metrics.collector.otelcol.image": "sumologic-otel-collector", + "sumologic.metrics.remoteWriteProxy.image": "nginx-unprivileged", + "sumologic.otelcolImage": "sumologic-otel-collector", + "sumologic.setup.job.image": "kubernetes-setup", + "sumologic.setup.job.initContainerImage": "busybox", + "tailing-sidecar-operator.kubeRbacProxy.image": "kube-rbac-proxy", + "tailing-sidecar-operator.operator.image": "tailing-sidecar-operator", + "tailing-sidecar-operator.sidecar.image": "tailing-sidecar", + "telegraf-operator.image": "telegraf-operator", + "telegraf-operator.image.sidecarImage": "telegraf", + "tracesGateway.deployment.image": "sumologic-otel-collector", + "tracesSampler.deployment.image": "sumologic-otel-collector", +} + def pairwise(iterable: list) -> list: """Transforms list in following way: @@ -192,7 +258,7 @@ def update_replace_components_images(image_file_path: str, create_new_file: bool Args: file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh - create_new_file (bool): determines whether new yaml should be created or the exiting file should be overwritten + create_new_file (bool): determines whether new file should be created or the exiting file should be overwritten """ cmd_lines = [] for image_with_tag, image_with_sha256 in pairwise(get_lines(image_file_path)): @@ -213,6 +279,85 @@ def update_replace_components_images(image_file_path: str, create_new_file: bool new_file.write(f"{cmd}\n") +def prepare_components_images_map(file_path: str) -> dict: + """Prepares components dict containing information about container images + + Args: + file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh + create_new_file (bool): determines whether new file should be created or the exiting file should be overwritten + + Returns + dict: dict with information about container images + """ + components_images = {} + for image_with_tag, image_with_sha256 in pairwise(get_lines(file_path)): + component, tag = image_with_tag.removeprefix(RED_HAT_REGISTRY).split(":") + sha = image_with_sha256.split(":")[-1] + components_images[component] = {"image_with_tag": image_with_tag, "image_with_sha256": image_with_sha256, "tag": tag, "sha": sha} + return components_images + + +def update_helm_install(image_file_path: str, create_new_file: bool): + """Updates helm install command in tests/helm_install.sh" + + Args: + file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh + """ + # pylint: disable=R0912 + image_config_keys = get_image_keys() + components_images = prepare_components_images_map(image_file_path) + set_args = [] + + for image_config in image_config_keys: + image_config_root = ".".join(image_config.split(".")[:-1]) + + if image_config_root not in COMPONENTS_CONFIG_MAP: + print(f"WARNING: missing key in components map, key: {image_config_root}") + continue + + component = COMPONENTS_CONFIG_MAP[image_config_root] + + if component not in components_images: + print(f"WARNING: missing key in components_images, key: {component}") + set_arg = f" --set {image_config}='' \\" + set_args.append(set_arg) + continue + + config = image_config.split(".")[-1] + if config == "repository": + if image_config_root + ".registry" not in image_config_keys: + set_arg = f" --set {image_config}={PUBLIC_ECR_REGISTRY}{component}@sha256 \\" + else: + set_arg = f" --set {image_config}={component}@sha256 \\" + elif config == "tag": + if image_config_root + ".sha" not in image_config_keys: + tag = components_images[component]["sha"] + else: + tag = components_images[component]["tag"] + + set_arg = f" --set {image_config}={tag} \\" + elif config == "registry": + registry = PUBLIC_ECR_REGISTRY[:-1] + set_arg = f" --set {image_config}={registry} \\" + elif config == "sha": + sha = components_images[component]["sha"] + set_arg = f" --set {image_config}={sha} \\" + elif config == "sidecarImage": + # special case for telegraf-operator.image.sidecarImage + component = component.removesuffix("-operator") + sidecar = components_images[component]["image_with_sha256"] + set_arg = f" --set {image_config}={sidecar} \\" + else: + set_arg = f" --set {image_config}='' \\" + set_args.append(set_arg) + + new_file_path = create_new_file_path(HELM_INSTALL_SCRIPT_PATH, create_new_file, ".sh") + with open(new_file_path, "w", encoding="utf-8") as new_file: + file_content = BASH_HEADER + HELM_INSTALL_COMMAND_HEADER + "\n".join(set_args) + file_content = file_content.removesuffix(" \\") # remove last \ after last helm install argument + new_file.write(f"{file_content}\n") + + def parse_args(): """Parses command line arguments""" parser = argparse.ArgumentParser() @@ -228,7 +373,7 @@ def parse_args(): ) parser.add_argument( "--create-new-file", - help="determines whether new yaml should be created or the exiting file should be overwritten", + help="determines whether new file should be created or the exiting file should be overwritten", default=True, ) return parser.parse_args() @@ -246,3 +391,5 @@ def parse_args(): update_manager(m_path, image_envs_list, args.create_new_file) update_replace_components_images(args.images_file, args.create_new_file) + + update_helm_install(args.images_file, args.create_new_file) diff --git a/tests/deploy_helm_chart.sh b/tests/deploy_helm_chart.sh index a8820ee..751f265 100755 --- a/tests/deploy_helm_chart.sh +++ b/tests/deploy_helm_chart.sh @@ -13,58 +13,7 @@ readonly TIME=300 helm repo add sumologic https://sumologic.github.io/sumologic-kubernetes-collection helm repo update -helm install test-openshift sumologic/sumologic \ - --set sumologic.accessId="dummy" \ - --set sumologic.accessKey="dummy" \ - --set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \ - --set sumologic.scc.create=true \ - --set fluent-bit.securityContext.privileged=true \ - --set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \ - --set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \ - --set fluentd.logs.containers.multiline.enabled=false \ - --set metrics-server.enabled=true \ - --set metrics-server.apiService.create=false \ - --set otelagent.enabled=true \ - --set telegraf-operator.enabled=true \ - --set falco.enabled=true \ - --set tailing-sidecar-operator.enabled=true \ - --set opentelemetry-operator.enabled=true \ - --set fluentd.image.repository=public.ecr.aws/sumologic/kubernetes-fluentd@sha256 \ - --set fluentd.image.tag=dca1e3854b2e7fd2540c53c8092e3105c6a4bdcce426035a2cd5fed28a8690bf \ - --set metrics-server.image.registry=public.ecr.aws \ - --set metrics-server.image.repository=sumologic/metrics-server@sha256 \ - --set metrics-server.image.tag=c4a41f333bf942fa21c08d0c5b1b6b649f878341886ba59593b73d808ad9d3b0 \ - --set fluent-bit.image.repository=public.ecr.aws/sumologic/fluent-bit@sha256 \ - --set fluent-bit.image.tag=a8ead538bbba26e83c9f70137ee9ae6dac922f7844ebd3273d57178b214790ab \ - --set kube-prometheus-stack.prometheusOperator.image.repository=public.ecr.aws/sumologic/prometheus-operator \ - --set kube-prometheus-stack.prometheusOperator.image.tag=v0.44.0 \ - --set kube-prometheus-stack.prometheusOperator.image.sha=27384cfcd3bf32bee7584332b82188a7da5780dfc33f33ea8aa3afd2c10ca948 \ - --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.repository=public.ecr.aws/sumologic/prometheus-config-reloader \ - --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.tag=v0.44.0 \ - --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.sha=7fc45d6c4cfa93363751277d3b761d30b1857be70d2c83568cabc36b19f47914 \ - --set kube-prometheus-stack.kube-state-metrics.image.repository=public.ecr.aws/sumologic/kube-state-metrics@sha256 \ - --set kube-prometheus-stack.kube-state-metrics.image.tag=18fb3800783b26db6a131ab846a28186dc0e9c12401e72a12ad99bf11d33186a \ - --set kube-prometheus-stack.prometheus-node-exporter.image.repository=public.ecr.aws/sumologic/node-exporter@sha256 \ - --set kube-prometheus-stack.prometheus-node-exporter.image.tag=685b59ebf0ce3c7e32f9de83359fdfc66a5143660f96b82c8ef8964c727bb2e5 \ - --set kube-prometheus-stack.prometheus.prometheusSpec.image.repository=public.ecr.aws/sumologic/prometheus \ - --set kube-prometheus-stack.prometheus.prometheusSpec.image.tag=v2.22.1 \ - --set kube-prometheus-stack.prometheus.prometheusSpec.image.sha=47004a8f74aab2353e8d0ac5725e4206a53305e7766a19b1cbb1dd770c2e8e36 \ - --set kube-prometheus-stack.prometheus.prometheusSpec.thanos.image=public.ecr.aws/sumologic/thanos@sha256:3f5b6df4e423475f912cb2e48e55e89235c68b473b53e202b246e024307ea965 \ - --set telegraf-operator.image.repository=public.ecr.aws/sumologic/telegraf-operator-ubi \ - --set telegraf-operator.image.sidecarImage=public.ecr.aws/sumologic/telegraf@sha256:f9883d1e9e0baf0d326a7c2c5503e011b36bf0bff22d6470c2f6d2b39fdd11fb \ - --set tailing-sidecar-operator.operator.image.repository=ghcr.io/sumologic/tailing-sidecar-operator@sha256 \ - --set tailing-sidecar-operator.operator.image.tag=e8b1d815666b69ab3047bc1d81afb51fb03caa99ca382c3f81949488072cd748 \ - --set tailing-sidecar-operator.sidecar.image.repository=ghcr.io/sumologic/tailing-sidecar@sha256 \ - --set tailing-sidecar-operator.sidecar.image.tag=3a9a741cbc10298d9be5bd68367883de45ad887e63ce12fedaf4024da807b82f \ - --set tailing-sidecar-operator.kubeRbacProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \ - --set tailing-sidecar-operator.kubeRbacProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \ - --set opentelemetry-operator.manager.image.repository=public.ecr.aws/sumologic/opentelemetry-operator@sha256 \ - --set opentelemetry-operator.manager.image.tag=b6fc0d5880016a8dc51a371839d0336409ad242f3ef046ca877c5d2c9df7e43e \ - --set opentelemetry-operator.kubeRBACProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \ - --set opentelemetry-operator.kubeRBACProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \ - --version 2.19.1 \ - -n sumologic-system \ - --create-namespace -f "${ROOT_DIR}/tests/values.yaml" +./tests/helm_install.sh wait_for_collection_resources "${NAMESPACE}" "${TIME}" diff --git a/tests/helm_install.sh b/tests/helm_install.sh new file mode 100755 index 0000000..2c186a8 --- /dev/null +++ b/tests/helm_install.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +readonly ROOT_DIR="$(dirname "$(dirname "${0}")")" + +helm install test-openshift sumologic/sumologic \ + --set sumologic.accessId="dummy" \ + --set sumologic.accessKey="dummy" \ + --set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \ + --set sumologic.scc.create=true \ + --set fluent-bit.securityContext.privileged=true \ + --set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \ + --set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \ + --set fluentd.logs.containers.multiline.enabled=false \ + --set metrics-server.enabled=true \ + --set metrics-server.apiService.create=false \ + --set otelagent.enabled=true \ + --set telegraf-operator.enabled=true \ + --set falco.enabled=true \ + --set tailing-sidecar-operator.enabled=true \ + --set opentelemetry-operator.enabled=true \ + --set fluentd.image.repository=public.ecr.aws/sumologic/kubernetes-fluentd@sha256 \ + --set fluentd.image.tag=dca1e3854b2e7fd2540c53c8092e3105c6a4bdcce426035a2cd5fed28a8690bf \ + --set metrics-server.image.registry=public.ecr.aws \ + --set metrics-server.image.repository=sumologic/metrics-server@sha256 \ + --set metrics-server.image.tag=c4a41f333bf942fa21c08d0c5b1b6b649f878341886ba59593b73d808ad9d3b0 \ + --set fluent-bit.image.repository=public.ecr.aws/sumologic/fluent-bit@sha256 \ + --set fluent-bit.image.tag=a8ead538bbba26e83c9f70137ee9ae6dac922f7844ebd3273d57178b214790ab \ + --set kube-prometheus-stack.prometheusOperator.image.repository=public.ecr.aws/sumologic/prometheus-operator \ + --set kube-prometheus-stack.prometheusOperator.image.tag=v0.44.0 \ + --set kube-prometheus-stack.prometheusOperator.image.sha=27384cfcd3bf32bee7584332b82188a7da5780dfc33f33ea8aa3afd2c10ca948 \ + --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.repository=public.ecr.aws/sumologic/prometheus-config-reloader \ + --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.tag=v0.44.0 \ + --set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.sha=7fc45d6c4cfa93363751277d3b761d30b1857be70d2c83568cabc36b19f47914 \ + --set kube-prometheus-stack.kube-state-metrics.image.repository=public.ecr.aws/sumologic/kube-state-metrics@sha256 \ + --set kube-prometheus-stack.kube-state-metrics.image.tag=18fb3800783b26db6a131ab846a28186dc0e9c12401e72a12ad99bf11d33186a \ + --set kube-prometheus-stack.prometheus-node-exporter.image.repository=public.ecr.aws/sumologic/node-exporter@sha256 \ + --set kube-prometheus-stack.prometheus-node-exporter.image.tag=685b59ebf0ce3c7e32f9de83359fdfc66a5143660f96b82c8ef8964c727bb2e5 \ + --set kube-prometheus-stack.prometheus.prometheusSpec.image.repository=public.ecr.aws/sumologic/prometheus \ + --set kube-prometheus-stack.prometheus.prometheusSpec.image.tag=v2.22.1 \ + --set kube-prometheus-stack.prometheus.prometheusSpec.image.sha=47004a8f74aab2353e8d0ac5725e4206a53305e7766a19b1cbb1dd770c2e8e36 \ + --set kube-prometheus-stack.prometheus.prometheusSpec.thanos.image=public.ecr.aws/sumologic/thanos@sha256:3f5b6df4e423475f912cb2e48e55e89235c68b473b53e202b246e024307ea965 \ + --set telegraf-operator.image.repository=public.ecr.aws/sumologic/telegraf-operator-ubi \ + --set telegraf-operator.image.sidecarImage=public.ecr.aws/sumologic/telegraf@sha256:f9883d1e9e0baf0d326a7c2c5503e011b36bf0bff22d6470c2f6d2b39fdd11fb \ + --set tailing-sidecar-operator.operator.image.repository=ghcr.io/sumologic/tailing-sidecar-operator@sha256 \ + --set tailing-sidecar-operator.operator.image.tag=e8b1d815666b69ab3047bc1d81afb51fb03caa99ca382c3f81949488072cd748 \ + --set tailing-sidecar-operator.sidecar.image.repository=ghcr.io/sumologic/tailing-sidecar@sha256 \ + --set tailing-sidecar-operator.sidecar.image.tag=3a9a741cbc10298d9be5bd68367883de45ad887e63ce12fedaf4024da807b82f \ + --set tailing-sidecar-operator.kubeRbacProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \ + --set tailing-sidecar-operator.kubeRbacProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \ + --set opentelemetry-operator.manager.image.repository=public.ecr.aws/sumologic/opentelemetry-operator@sha256 \ + --set opentelemetry-operator.manager.image.tag=b6fc0d5880016a8dc51a371839d0336409ad242f3ef046ca877c5d2c9df7e43e \ + --set opentelemetry-operator.kubeRBACProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \ + --set opentelemetry-operator.kubeRBACProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \ + --version 2.19.1 \ + -n sumologic-system \ + --create-namespace -f "${ROOT_DIR}/tests/values.yaml"