feat: add schema to helm chart

Signed-off-by: Dominik Rosiek <[email protected]>

ci/check_configuration_keys.py

@@ -12,21 +12,13 @@
 
 DESCRIPTION = 'This program verifies if all configuration from values.yaml has been documented.'
 SKIP_DEFAULTS = {
-    'kube-prometheus-stack.enabled',
-    'kube-prometheus-stack.global.imagePullSecrets',
-    'metadata.logs.autoscaling.targetMemoryUtilizationPercentage',
-    'metadata.logs.podDisruptionBudget',
     'metadata.logs.statefulset.extraEnvVars',
     'metadata.logs.statefulset.extraVolumeMounts',
     'metadata.logs.statefulset.extraVolumes',
     'metadata.logs.statefulset.extraPorts',
-    'metadata.metrics.podDisruptionBudget',
-    'metadata.metrics.autoscaling.targetMemoryUtilizationPercentage',
     'metadata.metrics.statefulset.extraEnvVars',
     'metadata.metrics.statefulset.extraVolumeMounts',
     'metadata.metrics.statefulset.extraVolumes',
-    'metadata.persistence.storageClass',
-    'otelcolInstrumentation.statefulset.priorityClassName',
     'otelcolInstrumentation.statefulset.extraEnvVars',
     'otelcolInstrumentation.statefulset.extraVolumeMounts',
     'otelcolInstrumentation.statefulset.extraVolumes',
@@ -37,12 +29,8 @@
     'tracesSampler.deployment.extraVolumeMounts',
     'tracesSampler.deployment.extraVolumes',
     'sumologic.setup.job.tolerations',
-    'sumologic.setup.job.pullSecrets',
-    'sumologic.pullSecrets',
-    'sumologic.setup.force',
-    'sumologic.setup.debug',
-    'metrics-server.image.pullSecrets',
-    'sumologic.events.sourceCategory',
+    'kube-prometheus-stack.prometheus-node-exporter.resources',
+    'kube-prometheus-stack.prometheusOperator.resources',
 }
 
 def main(values_path: str, readme_path: str, full_diff=False) -> None:
@@ -241,6 +229,9 @@ def compare_values(readme: dict, values_keys: list[str], values: dict) -> dict:
             if compare_keys(this_key, other_key):
                 other_value = get_value(this_key, values)
                 if this_value != other_value:
+                    if this_value.replace("\\\\", "\\").replace("\\|", "|") == other_value:
+                        # yaml contains both `'` and `"` strings and readme is always `"` string
+                        continue
 
                     # Skip configuration linked to values.yaml
                     if this_value == 'See [values.yaml]':
@@ -270,7 +261,7 @@ def get_value(key: str, dictionary: dict) -> str:
         value = value[subkey]
 
     if isinstance(value, str):
-        return value
+        return value.replace("\n", "\\n")
 
     return json.dumps(value)
 

ci/generate-schema.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import re
+import sys
+
+import yaml
+from yaml.loader import SafeLoader
+
+DESCRIPTION = 'This program generates JSON schema from README.md table'
+
+def values_to_dictionary(path: str) -> dict:
+    """Reads given path as values.yaml and returns it as dict
+
+    Args:
+        path (str): path to the value.yaml
+
+    Returns:
+        dict: values.yaml as dict
+    """
+    with open(path, encoding='utf-8') as file:
+        values_yaml = file.read()
+        values_yaml = re.sub(r'(\[\]|\{\})\n(\s+# )', r'\n\2', values_yaml, flags=re.M)
+        values_yaml = re.sub(r'^(\s+)# ', r'\1', values_yaml, flags=re.M)
+        return yaml.load(values_yaml, Loader=SafeLoader)
+
+def set_properties(values):
+    properties = {
+        'type': '',
+        # 'required': [],
+        # 'properties': {},
+        # 'default': '',
+        'description': '',
+    }
+
+    if isinstance(values, dict):
+        properties['type'] = 'object'
+        properties['properties'] = {}
+        for key in values.keys():
+            properties['properties'][key] = set_properties(values[key])
+    else:
+        properties['default'] = values
+        if isinstance(values, bool):
+            properties['type'] = 'boolean'
+        elif isinstance(values, int):
+            properties['type'] = 'integer'
+        elif isinstance(values, (list, set)):
+            properties['type'] = 'array'
+        elif isinstance(values, str):
+            properties['type'] = 'string'
+        else:
+            properties['type'] = 'string'
+            if not properties['default']:
+                properties['default'] = ""
+
+    return properties
+
+def extract_description_from_readme(path: str) -> dict:
+    """Reads given path as README.md and returns dict in the following form:
+
+    ```
+    {
+        configuration_key: configuration_default
+    }
+    ```
+
+    Args:
+        path (str): path to the README.md
+
+    Returns:
+        dict: {configuration_key: configuration_default,...}
+    """
+    with open(path, encoding='utf-8') as file:
+        readme = file.readlines()
+
+    keys = {}
+
+    for line in readme:
+        match = re.match(
+            r'^\|\s+`(?P<key>.*?)`\s+\|\s+(?P<description>.*?)\s+\|\s+(?P<value>.*?)\s+\|$',
+            line)
+        if match and match.group('key'):
+            description = match.group('description').strip('`').strip('"')
+            keys[match.group('key')] = description
+
+    return keys
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        prog = sys.argv[0],
+        description = DESCRIPTION)
+    parser.add_argument('--values', required=True)
+    parser.add_argument('--readme', required=True)
+    parser.add_argument('--output', required=True)
+    parser.add_argument('--full-diff', required=False, action='store_true')
+    args = parser.parse_args()
+
+    values = values_to_dictionary(args.values)
+
+    output = {
+        "$schema": "http://json-schema.org/schema#",
+        "type": "object",
+        "properties": {},
+    }
+
+    for key in values:
+        output['properties'][key] = set_properties(values[key])
+
+    descriptions = extract_description_from_readme(args.readme)
+    for key, description in descriptions.items():
+        a = output['properties']
+        subkeys = key.split(".")
+        for i in range(0, len(subkeys)-1):
+            a = a[subkeys[i]]['properties']
+        a[subkeys[-1]]['description'] = description
+    with open(args.output, "w") as f:
+        f.write(json.dumps(output, indent=2))

ci/generate_readme.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import re
+import sys
+import os
+
+import yaml
+from yaml.loader import SafeLoader
+
+DESCRIPTION = "test"
+HEADER = """# Configuration
+
+To see all available configuration for our sub-charts, please refer to their documentation.
+
+- [Falco](https://github.com/falcosecurity/charts/tree/master/falco#configuration) - All Falco properties should be prefixed with `falco.`
+  in our values.yaml to override a property not listed below.
+- [Kube-Prometheus-Stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#configuration) - All
+  Kube Prometheus Stack properties should be prefixed with `kube-prometheus-stack.` in our values.yaml to override a property not listed
+  below.
+- [Fluent Bit](https://github.com/fluent/helm-charts/blob/main/charts/fluent-bit/values.yaml) - All Fluent Bit properties should be prefixed
+  with `fluent-bit.` in our values.yaml to override a property not listed below.
+- [Metrics Server](https://github.com/bitnami/charts/tree/master/bitnami/metrics-server/#parameters) - All Metrics Server properties should
+  be prefixed with `metrics-server.` in our values.yaml to override a property not listed below.
+- [Tailing Sidecar Operator](https://github.com/SumoLogic/tailing-sidecar/tree/main/helm/tailing-sidecar-operator#configuration) - All
+  Tailing Sidecar Operator properties should be prefixed with `tailing-sidecar-operator` in our values.yaml to override a property not
+  listed below.
+- [OpenTelemetry Operator](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-operator#opentelemetry-operator-helm-chart) -
+  All OpenTelemetry Operator properties should be prefixed with `opentelemetry-operator` in our values.yaml to override a property listed
+  below.
+
+The following table lists the configurable parameters of the Sumo Logic chart and their default values.
+
+| Parameter | Description | Default |
+| --- | --- | --- |"""
+
+FOOTER = """
+[values.yaml]: values.yaml"""
+
+def build_default(data):
+  return_value = {}
+  if 'properties' in data:
+    for key in data['properties']:
+      return_value[key] = build_default(data['properties'][key])
+    return return_value
+  elif 'items' in data:
+    return [item['default'] for item in data['items']]
+  else:
+    return data['default']
+
+def get_description(prefix, data):
+    return_value = []
+    prefix = prefix.strip('.')
+    description = data["description"] if 'description' in data else ""
+    built_default = None
+
+    if 'properties' in data:
+      if not description:
+        for key in data['properties']:
+          if prefix == "":
+            pref = key
+          else:
+            if "." in key:
+              pref = f"{prefix}[{key}]"
+            else:
+              pref = f"{prefix}.{key}"
+          return_value += get_description(pref, data['properties'][key])
+        return return_value
+      else:
+        built_default = build_default(data)
+
+    if 'items' in data:
+        built_default = build_default(data)
+
+    default = json.dumps(built_default if built_default is not None else data['default']).strip('"').replace("|", "\|")
+    if len(default) > 180:
+      default = "See [values.yaml]"
+
+    if default == "":
+      default = "Nil"
+    return_value.append(f'| `{prefix}` | {data["description"]} | `{default}` |')
+
+    return return_value
+
+def main(schema, directory):
+    readme = [HEADER]
+    with open(schema) as f:
+        data = json.loads(f.read())
+        readme += get_description("", data)
+    readme.append(FOOTER)
+
+    readme = "\n".join(readme)
+
+    with open(os.path.join(directory, "README.md"), "w") as f:
+      f.write(readme)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        prog = sys.argv[0],
+        description = DESCRIPTION)
+    parser.add_argument('--schema', required=True)
+    parser.add_argument('--dir', required=True)
+    parser.add_argument('--full-diff', required=False, action='store_true')
+    args = parser.parse_args()
+
+    main(args.schema, args.dir)

ci/generate_values.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import re
+import sys
+import os
+
+import yaml
+from yaml.loader import SafeLoader
+
+DESCRIPTION = "test"
+
+
+def get_values(indent, data):
+    return_value = []
+    if 'properties' in data:
+      for key, value in data['properties'].items():
+        commented = ''
+        if 'comment' in value:
+          for line in value['comment'].split('\n'):
+            if not line.strip():
+              return_value.append(f"{indent}##")
+            else:
+              return_value.append(f"{indent}## {line}")
+        if 'commented' in value and value['commented']:
+          commented = '# '
+        if 'properties' in value:
+          return_value.append(f"{indent}{commented}{key}:")
+        elif 'items' in value:
+          return_value.append(f"{indent}{commented}{key}:")
+          for item in value['items']:
+            commented = ''
+            if 'commented' in item and item['commented']:
+              commented = '# '
+            if 'comment' in item:
+              for line in item['comment'].split('\n'):
+                if '#' in indent:
+                  return_value.append(f"{indent.replace('# ', '##')} {line.rstrip()}")
+                else:
+                  return_value.append(f"{indent}## {line.rstrip()}")
+            dumped = yaml.dump(item['default']).strip()
+            first = True
+            for line in dumped.split("\n"):
+              if first:
+                return_value.append(f"{indent}{commented}- {line}")
+                first = False
+                continue
+              return_value.append(f"{indent}{commented}  {line}")
+        else:
+          dumped = yaml.dump({key: value['default']}).strip()
+          for line in dumped.split("\n"):
+            if not line.strip():
+              return_value.append(f"{indent}{commented.rstrip()}")
+            else:
+              return_value.append(f"{indent}{commented}{line.rstrip()}")
+        if 'example' in value:
+          dumped = yaml.dump({key: value['example']}).strip()
+          for line in dumped.split("\n")[1:]:
+            if not line.strip():
+              return_value.append(f"{indent}#")
+            else:
+              return_value.append(f"{indent}# {line}")
+        return_value += get_values(f"{indent}{commented}  ", data['properties'][key])
+    return return_value
+
+def main(schema, directory):
+    with open(schema) as f:
+        data = json.loads(f.read())
+        values = ['## Sumo Logic Kubernetes Collection configuration file',
+'## All the comments start with two or more # characters'] + get_values('', data)
+
+    print('\n'.join(values))
+
+    # with open(os.path.join(directory, "_values.yaml"), "w") as f:
+    #   f.write(yaml.dump(values))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        prog = sys.argv[0],
+        description = DESCRIPTION)
+    parser.add_argument('--schema', required=True)
+    parser.add_argument('--dir', required=True)
+    parser.add_argument('--full-diff', required=False, action='store_true')
+    args = parser.parse_args()
+
+    main(args.schema, args.dir)