feat: chart install script (#576)

feat: chart install script

the install scripts for using the chart would run repetitive with the
boilerplate, so this tries to get it in one script

bin/chart-install-meta.yaml

@@ -0,0 +1,137 @@
+# This files generally tries to match the naming conventions for a
+# kustomization.yaml with `helmCharts`, but has some mostly necessary
+# deviations.
+#
+# Field names matching `helmCharts` from a kustomization.yaml:
+# ------------------------------------------------------------------------------
+#
+# These values have the same name and function as in a `helmCharts` stanza.
+#
+# - repo: URL for the repo for the chart
+# - name: name of the chart to install from the repo
+# - namespace: The namespace to install in.
+#     - This will often, but only incidentally, match the `releaseName`.
+# - releaseName: identical to a `helmCharts` entry
+#      - However, since we install with a script, actually shows up as a release
+#        with `helm list`.
+#      - This will often, but only incidentally, match the `name`.
+#
+# Field names matching `helmCharts` from a kustomization.yaml, optional:
+# ------------------------------------------------------------------------------
+#
+# - version: Specify a chart version.
+#
+# Slightly different from `helmCharts` in a `kustomization.yaml`:
+# ------------------------------------------------------------------------------
+#
+# This shows slight differences between a `helmCharts` stanza and what you put
+# here.
+#
+# - valuesFiles: This includes all values files listed in order to apply them.
+#     - A separate `valuesFile` field seems unnecessary, so this should
+#       frequently have `values.yaml` as the first, and often the only, list
+#       entry.
+#     - Notably, subsumes the functionality of `additionalValuesFiles` from
+#       a `helmConfig`, as it simply applies all specified values files in
+#       order.
+#
+# - first line as _Genestack_ component name
+#    - `helmCharts` takes a list and has a `name` field that refers to the
+#       name of the chart from the repo. For the purpose of identifying the
+#       _Genestack_ component, the identification for each entry will
+#       frequently duplicate the `name` as used to identify a chart in the
+#       repo, but may or may not differ, e.g., `prometheus:` refers to
+#       Prometheus proper as a _Genestack_ component, but has `name`
+#       `kube-prometheus-stack` as the chart name from the repo, while
+#       `prometheus-pushgateway:` ALSO has `name: prometheus-pushgateway` as
+#       overall top-level identifier matching the `name.
+#
+# Required here but not in a stanza of `helmCharts`
+# ------------------------------------------------------------------------------
+#
+# This doesn't appear in a `helmCharts` stanza of a `kustomization.yaml`, but
+# you have to supply it here.
+#
+# - repoName: name for the repo as shown by `helm repo list`.
+#    - Using `kustomization.yaml` doesn't create a repo shown by
+#      `helm repo list` so this needed adding.
+#
+# Not required here and not in a stanza of `helmCharts`
+# ------------------------------------------------------------------------------
+#
+# - timeout: amount of time allowed for `helm install` to complete
+#    - optional because it defaults to 10m
+#    - no equivalent exists in a `helmCharts` stanza
+# - overlayDir: name of directory under
+#   /etc/genestack/kustomize/<SERVICE_NAME>/ for base or overlays.
+#    - optional, defaults to 'overlay'
+#
+# Notable differences
+# ------------------------------------------------------------------------------
+# - includeCRDs: not used
+#     - This often gets used in `helmCharts` in kustomization.yaml, but
+#       invoking `helm` directly installs CRDs by default unless you
+#       specify `--skip-crds`, so doesn't appear here.
+# - additionalValuesFiles: not used
+#     - As mentioned above, this list just goes under `valuesFiles` here, where
+#       all items of the list get applied in order.
+
+prometheus:
+  repoName: prometheus-community
+  releaseName: kube-prometheus-stack
+  name: kube-prometheus-stack
+  repo: https://prometheus-community.github.io/helm-charts
+  namespace: prometheus
+  valuesFiles:
+    - prometheus-helm-overrides.yaml
+    - alerting_rules.yaml
+    - alertmanager_config.yaml
+
+prometheus-pushgateway:
+  repoName: prometheus-community
+  repo: https://prometheus-community.github.io/helm-charts
+  name: prometheus-pushgateway
+  releaseName: prometheus-pushgateway
+  namespace: prometheus
+  valuesFiles:
+    - values.yaml
+
+prometheus-blackbox-exporter:
+  name: prometheus-blackbox-exporter
+  repoName: prometheus-community
+  repo: https://prometheus-community.github.io/helm-charts
+  releaseName: prometheus-blackbox-exporter
+  namespace: prometheus
+  includeCRDs: true
+  valuesFiles:
+    - values.yaml
+    - probe_targets.yaml
+
+prometheus-rabbitmq-exporter:
+  name: prometheus-rabbitmq-exporter
+  namespace: openstack
+  repoName: prometheus-community
+  repo: https://prometheus-community.github.io/helm-charts
+  releaseName: prometheus-rabbitmq-exporter
+  version: 1.11.0
+  valuesFiles:
+    - values.yaml
+
+prometheus-mysql-exporter:
+  name: prometheus-mysql-exporter
+  repoName: prometheus-community
+  repo: https://prometheus-community.github.io/helm-charts
+  releaseName: prometheus-mysql-exporter
+  namespace: openstack
+  valuesFiles:
+    - values.yaml
+
+prometheus-postgres-exporter:
+  name: prometheus-postgres-exporter
+  repoName: prometheus-community
+  repo: https://prometheus-community.github.io/helm-charts
+  releaseName: prometheus-postgres-exporter
+  namespace: openstack
+  version: 6.0.0
+  valuesFiles:
+    - values.yaml

bin/install-chart.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+CHART="$1"
+shift # preserve "$@" for --post-renderer-args
+
+GENESTACK_DIR="${GENESTACK_DIR:-/opt/genestack}"
+CHART_META_FILE=\
+"${CHART_META_FILE:-$GENESTACK_DIR/bin/chart-install-meta.yaml}"
+GENESTACK_OVERLAY_DIR="${GENESTACK_OVERLAY_DIR:-/etc/genestack}"
+GENESTACK_CHART_DIR=\
+"${GENESTACK_CHART_DIR:-$GENESTACK_DIR/base-helm-configs/$CHART}"
+GENESTACK_CHART_OVERLAY_DIR=\
+"${GENESTACK_CHART_OVERLAY_DIR:-$GENESTACK_OVERLAY_DIR/helm-configs/$CHART}"
+# This Python needs PyYAML, which the normal Genestack venv will have.
+YAML_PARSER_PYTHON="${YAML_PARSER_PYTHON:-$(which python)}"
+YAML_PARSER_PY="${YAML_PARSER_PY:-$GENESTACK_DIR/bin/yamlparse.py}"
+YAML_PARSER_CMD="${YAML_PARSER_CMD:-$YAML_PARSER_PYTHON $YAML_PARSER_PY}"
+
+IFS= readarray -t chart_values < <($YAML_PARSER_CMD "$CHART_META_FILE" "$CHART")
+
+values_length=${#chart_values[@]}
+
+# This for loop takes values from the YAML file `namespace: prometheus` and
+# sets variables like NAMESPACE=prometheus for use by the script.
+for (( i=0; i<values_length; i+=2 ))
+do
+    var_name="$(echo "${chart_values[i]}" | tr '[:lower:]' '[:upper:]')"
+    declare "$var_name=${chart_values[i+1]}"
+done
+
+required_vars=("NAMESPACE" "NAME" "REPONAME" "RELEASENAME" "REPO")
+for var in "${required_vars[@]}"; do
+    if [[ -z "${!var}" ]]; then
+        echo "ERROR: Required variable $var is missing in the YAML file."
+        exit 1
+    fi
+done
+
+# Though it probably wouldn't make any difference for all of the
+# $GENESTACK_OVERLAY_DIR files to come last, this takes care to fully preserve
+# the order
+echo "Including overrides in order:"
+if [[ "$VALUESFILES" == "" ]]
+then
+    echo "WARNING: no values files specified. Check valuesFiles in the YAML file for $CHART"
+fi
+values_args=()
+set -o noglob # Prevent glob expansions in $VALUESFILES
+for BASE_FILENAME in $VALUESFILES
+do
+    for DIR in "$GENESTACK_CHART_DIR" "$GENESTACK_CHART_OVERLAY_DIR"
+    do
+        ABSOLUTE_PATH="$DIR/$BASE_FILENAME"
+        if [[ -e "$ABSOLUTE_PATH" ]]
+        then
+            echo "    $ABSOLUTE_PATH"
+            values_args+=("--values" "$ABSOLUTE_PATH")
+        fi
+    done
+done
+echo
+
+if [[ "$TIMEOUT" != "" ]]
+then
+    timeout_args+=("--timeout" "$TIMEOUT")
+fi
+# Run script as ECHO_TEST=true install-chart.sh to see the commands that would
+# run formatted as a Python list to clearly distinguish whitespace from separate
+# arguments
+run_or_test_print () {
+    if [[ "$ECHO_TEST" == "true" ]]
+    then
+        $YAML_PARSER_PYTHON -c 'import sys; print(sys.argv[1:])' "$@"
+    else
+        "$@"
+    fi
+}
+
+render_or_install=()
+if [[ "$RENDER_ONLY" == "true" ]]
+then
+    render_or_install+=("template")
+else
+    render_or_install+=("upgrade" "--install")
+fi
+
+version_args=()
+if [[ "$VERSION" != "" ]]
+then
+    version_args+=("--version" "$VERSION")
+fi
+
+run_or_test_print helm repo add "$REPONAME" "$REPO"
+run_or_test_print helm repo update
+run_or_test_print helm \
+    "${render_or_install[@]}" "$RELEASENAME" "$REPONAME/$NAME" \
+    --create-namespace --namespace="$NAMESPACE" \
+    --timeout "${TIMEOUT:-10m}" \
+    "${version_args[@]}" "${timeout_args[@]}" \
+    "${values_args[@]}" \
+    --post-renderer "$GENESTACK_OVERLAY_DIR/kustomize/kustomize.sh" \
+    --post-renderer-args "$CHART/${OVERLAYDIR:-overlay}" "$@"

bin/yamlparse.py

@@ -0,0 +1,30 @@
+# This works with install-chart.sh to get the information to run the script.
+# It doesn't work very well as any kind of stand-alone utility.
+
+import sys
+import yaml
+
+if len(sys.argv) < 3:
+    print(f"Usage: {sys.argv[0]} <filename> <chart>")
+    sys.exit(1)
+
+filename = sys.argv[1]
+chart = sys.argv[2]
+
+try:
+    with open(filename, "r") as conf_file:
+        conf_yaml = yaml.safe_load(conf_file)
+except Exception as e:
+    print(f"Error parsing YAML file {filename}: {e}")
+    sys.exit(1)
+
+if chart not in conf_yaml:
+    print(f"No chart '{chart}' in file {filename}")
+    sys.exit(1)
+
+for key, value in conf_yaml[chart].items():
+    print(f"{key}")
+    if not isinstance(value, list):
+        print(f"{value}")
+    else:
+        print(*value)