Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add script to generate helm install command #121

Merged
merged 2 commits into from
Jul 1, 2024
Merged

feat: add script to generate helm install command #121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b31e320
feat: add script to generate helm install command
kasia-kujawa Jun 25, 2024
12be38b
fix: change list of image keys not available in values.yaml
kasia-kujawa Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 25 additions & 10 deletions scripts/get_image_config_keys.py
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ def extract_keys(dictionary: dict) -> list:
"tailing-sidecar-operator.operator.image.repository",
"tailing-sidecar-operator.operator.image.tag",
"tailing-sidecar-operator.kubeRbacProxy.image.tag",
"opentelemetry-operator.manager.image.repository.tag",
"opentelemetry-operator.kubeRBACProxy.image.repository.tag",
"opentelemetry-operator.testFramework.image.repository.tag",
"opentelemetry-operator.manager.image.tag",
"opentelemetry-operator.kubeRBACProxy.image.tag",
"opentelemetry-operator.manager.collectorImage.tag",
"telegraf-operator.image.tag",
"kube-prometheus-stack.prometheus.prometheusSpec.image.tag",
"kube-prometheus-stack.prometheus.prometheusSpec.image.sha",
Expand All @@ -75,13 +75,7 @@ def extract_keys(dictionary: dict) -> list:
"metrics-server.image.tag",
]

not_needed_image_keys = [
"Percentage",
"falco",
"pullPolicy",
"pullSecrets",
"imagePullSecrets",
]
not_needed_image_keys = ["Percentage", "falco", "pullPolicy", "pullSecrets", "imagePullSecrets", "debug.sumologicMock.image", "otellogswindows", "testFramework"]
needed_image_keys = ["image", "tag", "repository"]


Expand Down Expand Up @@ -117,6 +111,27 @@ def get_image_keys() -> list:
return image_keys


def generate_components_map(keys) -> list:
"""Generates image configuration keys for components map used in update_images.py

Args:
keys (list): list of Helm Chart image configuration keys from values.yaml
Returns:
list: image configuration keys for components map used in update_images.py
"""
unique_root_keys = set()
for key in keys:
keys_parts = key.split(".")[:-1]
new_key = ".".join(keys_parts)
unique_root_keys.add(f'"{new_key}": "",')

return sorted(unique_root_keys)


if __name__ == "__main__":
print("---------- Helm Chart image configuration keys from values.yaml, used in watches.yaml ---------------")
image_config_keys = get_image_keys()
print("\n".join(image_config_keys))
print("---------- Image configuration keys for components map used in update_images.py ---------------------")
components_map_keys = generate_components_map(image_config_keys)
print("\n".join(sorted(components_map_keys)))
149 changes: 147 additions & 2 deletions scripts/update_images.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,85 @@
- bundle/manifests/operator.clusterserviceversion.yaml
- config/manager/manager.yaml
- tests/replace_components_images.sh
- tests/helm_install.sh
"""

import argparse
import os
import subprocess
import yaml

from get_image_config_keys import get_image_keys

RED_HAT_REGISTRY = "registry.connect.redhat.com/sumologic/"
PUBLIC_ECR_REGISTRY = "public.ecr.aws/sumologic/"
ENV_PREFIX = "RELATED_IMAGE_"
CLUSTER_SERVICE_VERSION_PATH = "bundle/manifests/operator.clusterserviceversion.yaml"
MANAGER_PATH = "config/manager/manager.yaml"
REPLACE_COMPONENTS_IMAGES_PATH = "tests/replace_components_images.sh"
HELM_INSTALL_SCRIPT_PATH = "tests/helm_install.sh"
BASH_HEADER = "#!/usr/bin/env bash\n\n"

HELM_INSTALL_COMMAND_HEADER = """readonly ROOT_DIR="$(dirname "$(dirname "${0}")")"

helm install test-openshift sumologic/sumologic \\
--set sumologic.accessId="dummy" \\
--set sumologic.accessKey="dummy" \\
--set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \\
--set sumologic.scc.create=true \\
--set fluent-bit.securityContext.privileged=true \\
--set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \\
--set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \\
--set fluentd.logs.containers.multiline.enabled=false \\
--set metrics-server.enabled=true \\
--set metrics-server.apiService.create=false \\
--set otelagent.enabled=true \\
--set telegraf-operator.enabled=true \\
--set falco.enabled=true \\
--set tailing-sidecar-operator.enabled=true \\
--set opentelemetry-operator.enabled=true \\
--version 2.19.1 \\
-n sumologic-system \\
--create-namespace -f "${ROOT_DIR}/tests/values.yaml" \\\n"""

# COMPONENTS_CONFIG_MAP maps helm chart configuration keys into components names
COMPONENTS_CONFIG_MAP = {
"instrumentation.instrumentationJobImage.image": "kubernetes-tools-kubectl",
"kube-prometheus-stack.kube-state-metrics.image": "kube-state-metrics",
"kube-prometheus-stack.prometheus-node-exporter.image": "node-exporter",
"kube-prometheus-stack.prometheus.prometheusSpec.image": "prometheus",
"kube-prometheus-stack.prometheusOperator.image": "prometheus-operator",
"kube-prometheus-stack.prometheusOperator.prometheusConfigReloader.image": "prometheus-config-reloader",
"kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage": "prometheus-config-reloader",
"kube-prometheus-stack.prometheusOperator.thanosImage": "thanos",
"metadata.image": "sumologic-otel-collector",
"metrics-server.image": "metrics-server",
"opentelemetry-operator.kubeRBACProxy.image": "kube-rbac-proxy",
"opentelemetry-operator.manager.autoInstrumentationImage.dotnet": "autoinstrumentation-dotnet",
"opentelemetry-operator.manager.autoInstrumentationImage.java": "autoinstrumentation-java",
"opentelemetry-operator.manager.autoInstrumentationImage.nodejs": "autoinstrumentation-nodejs",
"opentelemetry-operator.manager.autoInstrumentationImage.python": "autoinstrumentation-nodejs",
"opentelemetry-operator.manager.collectorImage": "sumologic-otel-collector",
"opentelemetry-operator.manager.image": "opentelemetry-operator",
"otelcolInstrumentation.statefulset.image": "sumologic-otel-collector",
"otelevents.image": "sumologic-otel-collector",
"otellogs.daemonset.initContainers.changeowner.image": "busybox",
"otellogs.image": "sumologic-otel-collector",
"pvcCleaner.job.image": "kubernetes-tools-kubectl",
"sumologic.metrics.collector.otelcol.image": "sumologic-otel-collector",
"sumologic.metrics.remoteWriteProxy.image": "nginx-unprivileged",
"sumologic.otelcolImage": "sumologic-otel-collector",
"sumologic.setup.job.image": "kubernetes-setup",
"sumologic.setup.job.initContainerImage": "busybox",
"tailing-sidecar-operator.kubeRbacProxy.image": "kube-rbac-proxy",
"tailing-sidecar-operator.operator.image": "tailing-sidecar-operator",
"tailing-sidecar-operator.sidecar.image": "tailing-sidecar",
"telegraf-operator.image": "telegraf-operator",
"telegraf-operator.image.sidecarImage": "telegraf",
"tracesGateway.deployment.image": "sumologic-otel-collector",
"tracesSampler.deployment.image": "sumologic-otel-collector",
}


def pairwise(iterable: list) -> list:
"""Transforms list in following way:
Expand Down Expand Up @@ -192,7 +256,7 @@ def update_replace_components_images(image_file_path: str, create_new_file: bool

Args:
file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh
create_new_file (bool): determines whether new yaml should be created or the exiting file should be overwritten
create_new_file (bool): determines whether new file should be created or the exiting file should be overwritten
"""
cmd_lines = []
for image_with_tag, image_with_sha256 in pairwise(get_lines(image_file_path)):
Expand All @@ -213,6 +277,85 @@ def update_replace_components_images(image_file_path: str, create_new_file: bool
new_file.write(f"{cmd}\n")


def prepare_components_images_map(file_path: str) -> dict:
"""Prepares components dict containing information about container images

Args:
file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh
create_new_file (bool): determines whether new file should be created or the exiting file should be overwritten

Returns
dict: dict with information about container images
"""
components_images = {}
for image_with_tag, image_with_sha256 in pairwise(get_lines(file_path)):
component, tag = image_with_tag.removeprefix(RED_HAT_REGISTRY).split(":")
sha = image_with_sha256.split(":")[-1]
components_images[component] = {"image_with_tag": image_with_tag, "image_with_sha256": image_with_sha256, "tag": tag, "sha": sha}
return components_images


def update_helm_install(image_file_path: str, create_new_file: bool):
"""Updates helm install command in tests/helm_install.sh"

Args:
file_path (str): path to the output of get_images_sha256.sh, see: https://github.com/SumoLogic/sumologic-openshift-images/blob/main/scripts/get_images_sha256.sh
"""
# pylint: disable=R0912,R0914
image_config_keys = get_image_keys()
components_images = prepare_components_images_map(image_file_path)
set_args = []

for image_config in image_config_keys:
image_config_root = ".".join(image_config.split(".")[:-1])

if image_config_root not in COMPONENTS_CONFIG_MAP:
print(f"WARNING: missing key in components map, key: {image_config_root}")
continue

component = COMPONENTS_CONFIG_MAP[image_config_root]

if component not in components_images:
print(f"WARNING: missing key in components_images, key: {component}")
set_arg = f" --set {image_config}='' \\"
set_args.append(set_arg)
continue

config = image_config.split(".")[-1]
if config == "repository":
if image_config_root + ".registry" not in image_config_keys:
set_arg = f" --set {image_config}={PUBLIC_ECR_REGISTRY}{component}@sha256 \\"
else:
set_arg = f" --set {image_config}={component}@sha256 \\"
elif config == "tag":
if image_config_root + ".sha" not in image_config_keys:
tag = components_images[component]["sha"]
else:
tag = components_images[component]["tag"]

set_arg = f" --set {image_config}={tag} \\"
elif config == "registry":
registry = PUBLIC_ECR_REGISTRY[:-1]
set_arg = f" --set {image_config}={registry} \\"
elif config == "sha":
sha = components_images[component]["sha"]
set_arg = f" --set {image_config}={sha} \\"
elif config == "sidecarImage":
# special case for telegraf-operator.image.sidecarImage
component = component.removesuffix("-operator")
sidecar = components_images[component]["image_with_sha256"]
set_arg = f" --set {image_config}={sidecar} \\"
else:
set_arg = f" --set {image_config}='' \\"
set_args.append(set_arg)

new_file_path = create_new_file_path(HELM_INSTALL_SCRIPT_PATH, create_new_file, ".sh")
with open(new_file_path, "w", encoding="utf-8") as new_file:
file_content = BASH_HEADER + HELM_INSTALL_COMMAND_HEADER + "\n".join(set_args)
file_content = file_content.removesuffix(" \\") # remove last \ after last helm install argument
new_file.write(f"{file_content}\n")


def parse_args():
"""Parses command line arguments"""
parser = argparse.ArgumentParser()
Expand All @@ -228,7 +371,7 @@ def parse_args():
)
parser.add_argument(
"--create-new-file",
help="determines whether new yaml should be created or the exiting file should be overwritten",
help="determines whether new file should be created or the exiting file should be overwritten",
default=True,
)
return parser.parse_args()
Expand All @@ -246,3 +389,5 @@ def parse_args():
update_manager(m_path, image_envs_list, args.create_new_file)

update_replace_components_images(args.images_file, args.create_new_file)

update_helm_install(args.images_file, args.create_new_file)
53 changes: 1 addition & 52 deletions tests/deploy_helm_chart.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,58 +13,7 @@ readonly TIME=300
helm repo add sumologic https://sumologic.github.io/sumologic-kubernetes-collection
helm repo update

helm install test-openshift sumologic/sumologic \
--set sumologic.accessId="dummy" \
--set sumologic.accessKey="dummy" \
--set sumologic.endpoint="http://receiver-mock.receiver-mock:3000/terraform/api/" \
--set sumologic.scc.create=true \
--set fluent-bit.securityContext.privileged=true \
--set kube-prometheus-stack.prometheus-node-exporter.service.port=9200 \
--set kube-prometheus-stack.prometheus-node-exporter.service.targetPort=9200 \
--set fluentd.logs.containers.multiline.enabled=false \
--set metrics-server.enabled=true \
--set metrics-server.apiService.create=false \
--set otelagent.enabled=true \
--set telegraf-operator.enabled=true \
--set falco.enabled=true \
--set tailing-sidecar-operator.enabled=true \
--set opentelemetry-operator.enabled=true \
--set fluentd.image.repository=public.ecr.aws/sumologic/kubernetes-fluentd@sha256 \
--set fluentd.image.tag=dca1e3854b2e7fd2540c53c8092e3105c6a4bdcce426035a2cd5fed28a8690bf \
--set metrics-server.image.registry=public.ecr.aws \
--set metrics-server.image.repository=sumologic/metrics-server@sha256 \
--set metrics-server.image.tag=c4a41f333bf942fa21c08d0c5b1b6b649f878341886ba59593b73d808ad9d3b0 \
--set fluent-bit.image.repository=public.ecr.aws/sumologic/fluent-bit@sha256 \
--set fluent-bit.image.tag=a8ead538bbba26e83c9f70137ee9ae6dac922f7844ebd3273d57178b214790ab \
--set kube-prometheus-stack.prometheusOperator.image.repository=public.ecr.aws/sumologic/prometheus-operator \
--set kube-prometheus-stack.prometheusOperator.image.tag=v0.44.0 \
--set kube-prometheus-stack.prometheusOperator.image.sha=27384cfcd3bf32bee7584332b82188a7da5780dfc33f33ea8aa3afd2c10ca948 \
--set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.repository=public.ecr.aws/sumologic/prometheus-config-reloader \
--set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.tag=v0.44.0 \
--set kube-prometheus-stack.prometheusOperator.prometheusConfigReloaderImage.sha=7fc45d6c4cfa93363751277d3b761d30b1857be70d2c83568cabc36b19f47914 \
--set kube-prometheus-stack.kube-state-metrics.image.repository=public.ecr.aws/sumologic/kube-state-metrics@sha256 \
--set kube-prometheus-stack.kube-state-metrics.image.tag=18fb3800783b26db6a131ab846a28186dc0e9c12401e72a12ad99bf11d33186a \
--set kube-prometheus-stack.prometheus-node-exporter.image.repository=public.ecr.aws/sumologic/node-exporter@sha256 \
--set kube-prometheus-stack.prometheus-node-exporter.image.tag=685b59ebf0ce3c7e32f9de83359fdfc66a5143660f96b82c8ef8964c727bb2e5 \
--set kube-prometheus-stack.prometheus.prometheusSpec.image.repository=public.ecr.aws/sumologic/prometheus \
--set kube-prometheus-stack.prometheus.prometheusSpec.image.tag=v2.22.1 \
--set kube-prometheus-stack.prometheus.prometheusSpec.image.sha=47004a8f74aab2353e8d0ac5725e4206a53305e7766a19b1cbb1dd770c2e8e36 \
--set kube-prometheus-stack.prometheus.prometheusSpec.thanos.image=public.ecr.aws/sumologic/thanos@sha256:3f5b6df4e423475f912cb2e48e55e89235c68b473b53e202b246e024307ea965 \
--set telegraf-operator.image.repository=public.ecr.aws/sumologic/telegraf-operator-ubi \
--set telegraf-operator.image.sidecarImage=public.ecr.aws/sumologic/telegraf@sha256:f9883d1e9e0baf0d326a7c2c5503e011b36bf0bff22d6470c2f6d2b39fdd11fb \
--set tailing-sidecar-operator.operator.image.repository=ghcr.io/sumologic/tailing-sidecar-operator@sha256 \
--set tailing-sidecar-operator.operator.image.tag=e8b1d815666b69ab3047bc1d81afb51fb03caa99ca382c3f81949488072cd748 \
--set tailing-sidecar-operator.sidecar.image.repository=ghcr.io/sumologic/tailing-sidecar@sha256 \
--set tailing-sidecar-operator.sidecar.image.tag=3a9a741cbc10298d9be5bd68367883de45ad887e63ce12fedaf4024da807b82f \
--set tailing-sidecar-operator.kubeRbacProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \
--set tailing-sidecar-operator.kubeRbacProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \
--set opentelemetry-operator.manager.image.repository=public.ecr.aws/sumologic/opentelemetry-operator@sha256 \
--set opentelemetry-operator.manager.image.tag=b6fc0d5880016a8dc51a371839d0336409ad242f3ef046ca877c5d2c9df7e43e \
--set opentelemetry-operator.kubeRBACProxy.image.repository=public.ecr.aws/sumologic/kube-rbac-proxy@sha256 \
--set opentelemetry-operator.kubeRBACProxy.image.tag=d47164d94803408071d68227798eadd3692d6705a6bcd55ccb31b025fc36d1d8 \
--version 2.19.1 \
-n sumologic-system \
--create-namespace -f "${ROOT_DIR}/tests/values.yaml"
./tests/helm_install.sh

wait_for_collection_resources "${NAMESPACE}" "${TIME}"

Expand Down
Loading
Loading