From 2f84e98c3a18f4c5afdaaf96f585bdbdabb66d1e Mon Sep 17 00:00:00 2001 From: phillip-toohill Date: Mon, 26 Feb 2024 14:58:29 -0600 Subject: [PATCH] Monitoring: adding openstack metrics exporter --- ...lm-prometheus-openstack-api-exporter.yaml} | 2 +- ...rometheus-openstack-metrics-exporter.yaml} | 25 +- .gitmodules | 3 + docs/openstack-metrics-exporter.md | 24 ++ .../openstack-exporter-helm-overrides.yaml | 230 ------------------ .../openstack-metrics-exporter/clouds-yaml | 11 + ...stack-metrics-exporter-helm-overrides.yaml | 89 +++++++ .../aio/kustomization.yaml | 2 +- .../base/hpa-openstack-api-exporter.yaml | 4 +- .../base/kustomization.yaml | 3 - .../aio/kustomization.yaml | 2 +- .../base/hpa-openstack-metrics-exporter.yaml} | 4 +- .../base/kustomization.yaml | 3 + kustomize/prometheus/kustomization.yaml | 2 - kustomize/prometheus/ns-prometheus.yaml | 7 - submodules/openstack-exporter | 1 + 16 files changed, 148 insertions(+), 264 deletions(-) rename .github/workflows/{helm-openstack-api-exporter.yaml => helm-prometheus-openstack-api-exporter.yaml} (97%) rename .github/workflows/{helm-openstack-exporter.yaml => helm-prometheus-openstack-metrics-exporter.yaml} (57%) create mode 100644 docs/openstack-metrics-exporter.md delete mode 100644 helm-configs/monitoring/openstack-exporter/openstack-exporter-helm-overrides.yaml create mode 100644 helm-configs/monitoring/openstack-metrics-exporter/clouds-yaml create mode 100644 helm-configs/monitoring/openstack-metrics-exporter/openstack-metrics-exporter-helm-overrides.yaml delete mode 100644 kustomize/monitoring/openstack-exporter/base/kustomization.yaml rename kustomize/monitoring/{openstack-exporter => openstack-metrics-exporter}/aio/kustomization.yaml (81%) rename kustomize/monitoring/{openstack-exporter/base/hpa-openstack-exporter.yaml => openstack-metrics-exporter/base/hpa-openstack-metrics-exporter.yaml} (77%) create mode 100644 kustomize/monitoring/openstack-metrics-exporter/base/kustomization.yaml delete mode 100644 kustomize/prometheus/ns-prometheus.yaml create mode 160000 submodules/openstack-exporter diff --git a/.github/workflows/helm-openstack-api-exporter.yaml b/.github/workflows/helm-prometheus-openstack-api-exporter.yaml similarity index 97% rename from .github/workflows/helm-openstack-api-exporter.yaml rename to .github/workflows/helm-prometheus-openstack-api-exporter.yaml index ef2c067a..69d754f9 100644 --- a/.github/workflows/helm-openstack-api-exporter.yaml +++ b/.github/workflows/helm-prometheus-openstack-api-exporter.yaml @@ -1,4 +1,4 @@ -name: Helm GitHub Actions for openstack-exporter +name: Helm GitHub Actions for openstack-api-exporter on: pull_request: diff --git a/.github/workflows/helm-openstack-exporter.yaml b/.github/workflows/helm-prometheus-openstack-metrics-exporter.yaml similarity index 57% rename from .github/workflows/helm-openstack-exporter.yaml rename to .github/workflows/helm-prometheus-openstack-metrics-exporter.yaml index 9c402706..640e8d6b 100644 --- a/.github/workflows/helm-openstack-exporter.yaml +++ b/.github/workflows/helm-prometheus-openstack-metrics-exporter.yaml @@ -1,11 +1,11 @@ -name: Helm GitHub Actions for openstack-exporter +name: Helm GitHub Actions for openstack-metrics-exporter on: pull_request: paths: - - helm-configs/monitoring/openstack-exporter/** - - kustomize/monitoring/openstack-exporter/** - - .github/workflows/helm-openstack-exporter.yaml + - helm-configs/monitoring/openstack-metrics-exporter/** + - kustomize/monitoring/openstack-metrics-exporter/** + - .github/workflows/helm-openstack-metrics-exporter.yaml jobs: helm: strategy: @@ -30,24 +30,19 @@ jobs: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x ./kubectl fi - - name: Pull OSH submodules + - name: Pull Openstack Exporter submodules run: | - git submodule update --init submodules/openstack-helm - git submodule update --init submodules/openstack-helm-infra - - name: Make OSH submodule - run: | - cd submodules/openstack-helm-infra - make prometheus-openstack-exporter + git submodule update --init submodules/openstack-exporter - name: Run Helm Template run: | - cd submodules/openstack-helm-infra - ${{ steps.helm.outputs.helm-path }} template prometheus-openstack-exporter ./prometheus-openstack-exporter \ + cd submodules/openstack-exporter/charts/ + ${{ steps.helm.outputs.helm-path }} template os-metrics ./prometheus-openstack-exporter \ --namespace=openstack \ --wait \ --timeout 120m \ - -f ${{ github.workspace }}/helm-configs/monitoring/openstack-exporter/openstack-exporter-helm-overrides.yaml \ + -f ${{ github.workspace }}/helm-configs/monitoring/openstack-metrics-exporter/openstack-metrics-exporter-helm-overrides.yaml \ --post-renderer ${{ github.workspace }}/kustomize/kustomize.sh \ - --post-renderer-args monitoring/openstack-exporter/${{ matrix.overlays }} > /tmp/rendered.yaml + --post-renderer-args monitoring/openstack-metrics-exporter/${{ matrix.overlays }} > /tmp/rendered.yaml - name: Return helm Build uses: actions/upload-artifact@v2 with: diff --git a/.gitmodules b/.gitmodules index 92b0db6b..8fd61a81 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "submodules/mariadb-operator"] path = submodules/mariadb-operator url = https://github.com/mariadb-operator/mariadb-operator +[submodule "submodules/openstack-exporter"] + path = submodules/openstack-exporter + url = https://github.com/openstack-exporter/helm-charts diff --git a/docs/openstack-metrics-exporter.md b/docs/openstack-metrics-exporter.md new file mode 100644 index 00000000..fed12da8 --- /dev/null +++ b/docs/openstack-metrics-exporter.md @@ -0,0 +1,24 @@ +## Openstack MetricsExporter + +We are using Prometheus for monitoring and metrics collection backend along with the openstack exporter to gather openstack resource metrics +For more information see: https://prometheus.io and https://github.com/openstack-exporter/openstack-exporter + +## Deploy the Prometheus Openstack Exporter + +### Create clouds-yaml secret +Modify genestack/helm-configs/monitoring/openstack-metrics-exporter/clouds-yaml with the appropriate settings and create the secret. + +```shell +kubectl create secret generic clouds-yaml-secret --from-file /opt/genestack/helm-configs/monitoring/openstack-metrics-exporter/clouds-yaml +``` + +### Install openstack-metrics-exporter helm chart +```shell +cd /opt/genestack/submodules/openstack-exporter/helm-charts/charts + +helm upgrade --install os-metrics ./prometheus-openstack-exporter \ + --namespace=openstack \ + --timeout 15m \ + -f /opt/genestack/helm-configs/monitoring/openstack-metrics-exporter/openstack-metrics-exporter-helm-overrides.yaml \ + --set clouds_yaml_config="$(kubectl get secret clouds-yaml-secret -o jsonpath='{.data.clouds-yaml}' | base64 -d)" +``` diff --git a/helm-configs/monitoring/openstack-exporter/openstack-exporter-helm-overrides.yaml b/helm-configs/monitoring/openstack-exporter/openstack-exporter-helm-overrides.yaml deleted file mode 100644 index 0c6b06c7..00000000 --- a/helm-configs/monitoring/openstack-exporter/openstack-exporter-helm-overrides.yaml +++ /dev/null @@ -1,230 +0,0 @@ - -images: - tags: - prometheus_openstack_exporter: docker.io/openstackhelm/prometheus-openstack-exporter:ubuntu_bionic-20240210 - dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 - image_repo_sync: docker.io/library/docker:17.07.0 - ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy - pull_policy: IfNotPresent - local_registry: - active: false - exclude: - - dep_check - - image_repo_sync - -labels: - openstack_exporter: - node_selector_key: openstack-control-plane - node_selector_value: enabled - job: - node_selector_key: openstack-control-plane - node_selector_value: enabled - -pod: - security_context: - exporter: - pod: - runAsUser: 65534 - container: - openstack_metrics_exporter: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - ks_user: - pod: - runAsUser: 65534 - container: - prometheus_openstack_exporter_ks_user: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - affinity: - anti: - type: - default: preferredDuringSchedulingIgnoredDuringExecution - topologyKey: - default: kubernetes.io/hostname - mounts: - prometheus_openstack_exporter: - prometheus_openstack_exporter: - init_container: null - replicas: - prometheus_openstack_exporter: 1 - lifecycle: - upgrades: - deployments: - revision_history: 3 - pod_replacement_strategy: RollingUpdate - rolling_update: - max_unavailable: 1 - max_surge: 3 - termination_grace_period: - prometheus_openstack_exporter: - timeout: 30 - resources: - enabled: false - prometheus_openstack_exporter: - requests: - memory: "64Mi" - cpu: "100m" - limits: - memory: "4096Mi" - jobs: - image_repo_sync: - requests: - memory: "64Mi" - cpu: "100m" - limits: - memory: "4096Mi" - ks_user: - requests: - memory: "64Mi" - cpu: "100m" - limits: - memory: "4096Mi" -dependencies: - dynamic: - common: - local_image_registry: - jobs: - - prometheus-openstack-exporter-image-repo-sync - services: - - endpoint: node - service: local_image_registry - static: - image_repo_sync: - services: - - endpoint: internal - service: local_image_registry - ks_user: - services: - - endpoint: internal - service: identity - prometheus_openstack_exporter: - jobs: - - prometheus-openstack-exporter-ks-user - services: - - endpoint: internal - service: identity - -conf: - prometheus_openstack_exporter: - OS_POLLING_INTERVAL: 30 - TIMEOUT_SECONDS: 20 - OS_RETRIES: 1 - -secrets: - identity: - admin: prometheus-openstack-exporter-keystone-admin - user: prometheus-openstack-exporter-keystone-user - oci_image_registry: - prometheus-openstack-exporter: prometheus-openstack-exporter-oci-image-registry-key - tls: - identity: - api: - # This name should be same as in keystone. Keystone - # secret will be used in these charts - # - public: keystone-tls-public - internal: keystone-tls-public - - -endpoints: - cluster_domain_suffix: cluster.local - local_image_registry: - name: docker-registry - namespace: docker-registry - hosts: - default: localhost - internal: docker-registry - node: localhost - host_fqdn_override: - default: null - port: - registry: - node: 5000 - oci_image_registry: - name: oci-image-registry - namespace: oci-image-registry - auth: - enabled: false - prometheus-openstack-exporter: - username: prometheus-openstack-exporter - password: password - hosts: - default: localhost - host_fqdn_override: - default: null - port: - registry: - default: null - prometheus_openstack_exporter: - namespace: null - hosts: - default: openstack-metrics - host_fqdn_override: - default: null - path: - default: null - scheme: - default: 'http' - port: - exporter: - default: 9103 - identity: - name: keystone - auth: - admin: - region_name: RegionOne - username: admin - password: password - project_name: admin - user_domain_name: default - project_domain_name: default - user: - role: admin - region_name: RegionOne - username: prometheus-openstack-exporter - password: password - project_name: service - user_domain_name: default - project_domain_name: default - hosts: - default: keystone - internal: keystone-api - host_fqdn_override: - default: null - path: - default: /v3 - scheme: - default: 'http' - port: - api: - default: 80 - internal: 5000 - -monitoring: - prometheus: - enabled: true - openstack_exporter: - scrape: true - -network: - openstack_metrics_exporter: - port: 9103 - -network_policy: - prometheus-openstack-exporter: - ingress: - - {} - egress: - - {} - -manifests: - certificates: false - configmap_bin: true - deployment: true - job_image_repo_sync: true - job_ks_user: true - network_policy: false - secret_keystone: true - secret_registry: true - service: true diff --git a/helm-configs/monitoring/openstack-metrics-exporter/clouds-yaml b/helm-configs/monitoring/openstack-metrics-exporter/clouds-yaml new file mode 100644 index 00000000..6f70ee7c --- /dev/null +++ b/helm-configs/monitoring/openstack-metrics-exporter/clouds-yaml @@ -0,0 +1,11 @@ +clouds.yaml: | + clouds: + default: + region_name: RegionOne + auth: + username: admin + password: + project_name: admin + project_domain_name: default + user_domain_name: default + auth_url: 'http://keystone-api.openstack.svc.cluster.local:5000/v3' diff --git a/helm-configs/monitoring/openstack-metrics-exporter/openstack-metrics-exporter-helm-overrides.yaml b/helm-configs/monitoring/openstack-metrics-exporter/openstack-metrics-exporter-helm-overrides.yaml new file mode 100644 index 00000000..0854e377 --- /dev/null +++ b/helm-configs/monitoring/openstack-metrics-exporter/openstack-metrics-exporter-helm-overrides.yaml @@ -0,0 +1,89 @@ +--- +# Choose endpoint type (this will override the value of clouds.yaml) +endpoint_type: public +# Ignored when multicloud.enabled: true +cloud: default + +replicaCount: 1 + +image: + repository: ghcr.io/openstack-exporter/openstack-exporter + tag: 1.7.0 + pullPolicy: Always + +serviceMonitor: + interval: 3m + scrapeTimeout: 30s + +multicloud: + # Enable multicloud (i.e. /probe?... targets) + # Doc: https://github.com/openstack-exporter/openstack-exporter#description + enabled: false + # Monitor the exporter's internal metrics (i.e. /metrics) + selfmonitor: false + # List of clouds to scrape, and the services to scrape from them + clouds: + - name: default + services: + - volume + - image + - identity + - compute + - network + - placement + - load-balancer + - object-store + #- container-infra + #- dns + #- baremetal + - gnocchi + #- database + - orchestration + + # - name: cloud1 + # services: + # - volume + # - name: cloud2 + # services: + # - volume + # - image + # - identity + # - compute + # - network + # - placement # NB this needs the openstack-exporter release > 1.4.0 + +# Add extra args to the exporter. +# Doc: https://github.com/openstack-exporter/openstack-exporter#command-line-options +#extraArgs: +# - --disable-service.baremetal +# - --disable-service.container-infra +# - --disable-service.object-store + +# Add extra environment variables +# extraEnvs: +# "ENV_VAR": "value" + +# Add extra labels +#commonLabels: +# prometheus.io/scrape: "true" +# prometheus.io/monitor: "true" + +# Generate a secret for clouds.yaml +# Doc: https://github.com/openstack-exporter/openstack-exporter#openstack-configuration +clouds_yaml_config: | + clouds.yaml: | + clouds: + default: + region_name: RegionOne + auth: + username: admin + password: + project_name: admin + project_domain_name: default + user_domain_name: default + auth_url: 'http://keystone-api.openstack.svc.cluster.local:5000/v3' +# In case of multicloud.enabled +# cloud1: +# ... +# cloud2: +# ... diff --git a/kustomize/monitoring/openstack-api-exporter/aio/kustomization.yaml b/kustomize/monitoring/openstack-api-exporter/aio/kustomization.yaml index 1b804cba..bdf62461 100644 --- a/kustomize/monitoring/openstack-api-exporter/aio/kustomization.yaml +++ b/kustomize/monitoring/openstack-api-exporter/aio/kustomization.yaml @@ -4,7 +4,7 @@ bases: patches: - target: kind: HorizontalPodAutoscaler - name: prometheus-openstack-exporter + name: os-api-prometheus-openstack-exporter patch: |- - op: replace path: /spec/minReplicas diff --git a/kustomize/monitoring/openstack-api-exporter/base/hpa-openstack-api-exporter.yaml b/kustomize/monitoring/openstack-api-exporter/base/hpa-openstack-api-exporter.yaml index dcaaf8db..946c2140 100644 --- a/kustomize/monitoring/openstack-api-exporter/base/hpa-openstack-api-exporter.yaml +++ b/kustomize/monitoring/openstack-api-exporter/base/hpa-openstack-api-exporter.yaml @@ -1,7 +1,7 @@ apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: prometheus-openstack-exporter + name: os-api-prometheus-openstack-exporter namespace: openstack spec: maxReplicas: 3 @@ -16,4 +16,4 @@ spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: prometheus-openstack-exporter + name: os-api-prometheus-openstack-exporter diff --git a/kustomize/monitoring/openstack-exporter/base/kustomization.yaml b/kustomize/monitoring/openstack-exporter/base/kustomization.yaml deleted file mode 100644 index df73c543..00000000 --- a/kustomize/monitoring/openstack-exporter/base/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - all.yaml - - hpa-openstack-exporter.yaml diff --git a/kustomize/monitoring/openstack-exporter/aio/kustomization.yaml b/kustomize/monitoring/openstack-metrics-exporter/aio/kustomization.yaml similarity index 81% rename from kustomize/monitoring/openstack-exporter/aio/kustomization.yaml rename to kustomize/monitoring/openstack-metrics-exporter/aio/kustomization.yaml index 1b804cba..38ff2cde 100644 --- a/kustomize/monitoring/openstack-exporter/aio/kustomization.yaml +++ b/kustomize/monitoring/openstack-metrics-exporter/aio/kustomization.yaml @@ -4,7 +4,7 @@ bases: patches: - target: kind: HorizontalPodAutoscaler - name: prometheus-openstack-exporter + name: os-metrics-prometheus-openstack-exporter patch: |- - op: replace path: /spec/minReplicas diff --git a/kustomize/monitoring/openstack-exporter/base/hpa-openstack-exporter.yaml b/kustomize/monitoring/openstack-metrics-exporter/base/hpa-openstack-metrics-exporter.yaml similarity index 77% rename from kustomize/monitoring/openstack-exporter/base/hpa-openstack-exporter.yaml rename to kustomize/monitoring/openstack-metrics-exporter/base/hpa-openstack-metrics-exporter.yaml index dcaaf8db..1287db78 100644 --- a/kustomize/monitoring/openstack-exporter/base/hpa-openstack-exporter.yaml +++ b/kustomize/monitoring/openstack-metrics-exporter/base/hpa-openstack-metrics-exporter.yaml @@ -1,7 +1,7 @@ apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: prometheus-openstack-exporter + name: os-metrics-prometheus-openstack-exporter namespace: openstack spec: maxReplicas: 3 @@ -16,4 +16,4 @@ spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: prometheus-openstack-exporter + name: os-metrics-prometheus-openstack-exporter diff --git a/kustomize/monitoring/openstack-metrics-exporter/base/kustomization.yaml b/kustomize/monitoring/openstack-metrics-exporter/base/kustomization.yaml new file mode 100644 index 00000000..215a38fe --- /dev/null +++ b/kustomize/monitoring/openstack-metrics-exporter/base/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - hpa-openstack-metrics-exporter.yaml + - all.yaml diff --git a/kustomize/prometheus/kustomization.yaml b/kustomize/prometheus/kustomization.yaml index 51ac4341..88b5289f 100644 --- a/kustomize/prometheus/kustomization.yaml +++ b/kustomize/prometheus/kustomization.yaml @@ -1,5 +1,3 @@ -resources: - - ns-prometheus.yaml helmCharts: - name: kube-prometheus-stack diff --git a/kustomize/prometheus/ns-prometheus.yaml b/kustomize/prometheus/ns-prometheus.yaml deleted file mode 100644 index 968d863f..00000000 --- a/kustomize/prometheus/ns-prometheus.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - kubernetes.io/metadata.name: prometheus - name: prometheus - name: prometheus diff --git a/submodules/openstack-exporter b/submodules/openstack-exporter new file mode 160000 index 00000000..fe694218 --- /dev/null +++ b/submodules/openstack-exporter @@ -0,0 +1 @@ +Subproject commit fe6942189cbc5d836716235ff20e89f9d1dd5ed4