Merge branch 'gardener-logging-stack' into gardener-logging-test

metal-stack · Nov 11, 2024 · 5d6e173 · 5d6e173
2 parents 2312417 + dfb4d5a
commit 5d6e173
Show file tree

Hide file tree

Showing 49 changed files with 707 additions and 107 deletions.
diff --git a/control-plane/roles/auditing-meili/defaults/main/main.yaml b/control-plane/roles/auditing-meili/defaults/main/main.yaml
@@ -17,6 +17,7 @@ auditing_meili_backup_restore_sidecar_backup_cron_schedule: "0 * * * *"
 auditing_meili_backup_restore_sidecar_log_level: debug
 auditing_meili_backup_restore_sidecar_object_prefix: "{{ auditing_meili_name }}-{{ metal_control_plane_stage_name }}"
 auditing_meili_backup_restore_sidecar_object_max_keep:
+auditing_meili_backup_restore_sidecar_encryption_key:
 
 auditing_meili_backup_restore_sidecar_gcp_bucket_name:
 auditing_meili_backup_restore_sidecar_gcp_backup_location:

diff --git a/control-plane/roles/auditing-meili/tasks/main.yaml b/control-plane/roles/auditing-meili/tasks/main.yaml
@@ -41,3 +41,4 @@
     meilisearch_backup_restore_sidecar_gcp_serviceaccount_json: "{{ auditing_meili_backup_restore_sidecar_gcp_serviceaccount_json }}"
     meilisearch_resources: "{{ auditing_meili_resources }}"
     meilisearch_backup_restore_sidecar_object_max_keep: "{{ auditing_meili_backup_restore_sidecar_object_max_keep }}"
+    meilisearch_backup_restore_sidecar_encryption_key: "{{ auditing_meili_backup_restore_sidecar_encryption_key }}"
diff --git a/control-plane/roles/gardener/README.md b/control-plane/roles/gardener/README.md
@@ -38,6 +38,7 @@ Check out the Gardener project for further documentation on [gardener.cloud](htt
 | gardener_kube_api_server_kubeconfig                    |           | The kubeconfig for the Gardener Kubernetes API (virtual garden apiserver)                                                                                                                                   |
 | gardener_kube_apiserver_kubeconfig_path                |           | The acts on multiple Kubernetes APIs, this is where it puts the kubeconfig of the Gardener Kubernetes API                                                                                                   |
 | gardener_local_tmp_dir                                 |           | The acts on multiple Kubernetes APIs, this is a local folder in the deployment container to store the kubeconfigs (ephemeral)                                                                               |
+| gardener_logging_enabled                               |           | Specifies whether the logging Gardener logging stack should be activated in the Gardenlet                                                                                                                   |
 
 ### Virtual Garden
 

diff --git a/control-plane/roles/gardener/defaults/main/gardener.yaml b/control-plane/roles/gardener/defaults/main/gardener.yaml
@@ -110,3 +110,5 @@ gardener_shooted_seed_rollout_delay_minutes:
 gardener_kube_api_server_kubeconfig: "{{ 'garden-kube-apiserver' | kubeconfig_from_cert(gardener_kube_api_server_ca, gardener_kube_api_server_client_cert, gardener_kube_api_server_client_key, prepend_https=true) }}"
 gardener_kube_apiserver_kubeconfig_path: "{{ gardener_local_tmp_dir }}/garden-kube-apiserver-kubeconfig"
 gardener_local_tmp_dir: "{{ playbook_dir }}/.ansible/tmp"
+
+gardener_logging_enabled: false
diff --git a/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml b/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml
@@ -1,11 +1,10 @@
 ---
-apiVersion: core.gardener.cloud/v1beta1
+apiVersion: core.gardener.cloud/v1
 kind: ControllerDeployment
 metadata:
   name: provider-metal
-type: helm
-providerConfig:
-  chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/gardener-extension-provider-metal/' + gardener_extension_provider_metal_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}"
+helm:
+  rawChart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/gardener-extension-provider-metal/' + gardener_extension_provider_metal_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].helm.rawChart }}"
   values:
     image:
       repository: "{{ gardener_extension_provider_metal_image_name }}"

diff --git a/control-plane/roles/gardener/templates/gardenlet-values.j2 b/control-plane/roles/gardener/templates/gardenlet-values.j2
@@ -22,6 +22,11 @@ config:
       # allow setting shoot ignore annotation:
       respectSyncPeriodOverwrite: {{ gardener_gardenlet_shoot_respect_sync_period_overwrite }}
 
+{% if gardener_logging_enabled %}
+  logging:
+    enabled: true
+{% endif %}
+
   seedConfig:
     apiVersion: core.gardener.cloud/v1beta1
     kind: Seed
@@ -78,4 +83,4 @@ imageVectorOverwrite: |
 {% if gardener_component_image_vector_overwrite %}
 componentImageVectorOverwrites: |
     {{ gardener_component_image_vector_overwrite | to_yaml | indent(width=4, first=false) }}
-{% endif %}
+{% endif %}
diff --git a/control-plane/roles/gardener/templates/managed-seed.j2 b/control-plane/roles/gardener/templates/managed-seed.j2
@@ -40,8 +40,11 @@ spec:
               visible: {{ gardener_shooted_seed.visible | default(true) }}
             shootDNS:
               enabled: true
+{% if gardener_logging_enabled %}
       logging:
         enabled: true
+{% endif %}
+
     deployment:
       image:
         pullPolicy: IfNotPresent

diff --git a/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml b/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml
@@ -1,11 +1,10 @@
 ---
-apiVersion: core.gardener.cloud/v1beta1
+apiVersion: core.gardener.cloud/v1
 kind: ControllerDeployment
 metadata:
   name: networking-cilium
-type: helm
-providerConfig:
-  chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/' + gardener_networking_cilium_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}"
+helm:
+  rawChart: "{{ (lookup('url', 'https://raw.githubusercontent.com/' + gardener_networking_cilium_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].helm.rawChart }}"
   values:
     image:
       repository: "{{ gardener_networking_cilium_image_name }}"
@@ -15,4 +14,4 @@ providerConfig:
     imageVectorOverwrite: |
       images:
         {{ gardener_extension_networking_cilium_image_vector_overwrite | to_nice_yaml(indent=2) | indent(width=8, first=false) }}
-{% endif %}
+{% endif %}
diff --git a/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml b/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml
@@ -1,11 +1,10 @@
 ---
-apiVersion: core.gardener.cloud/v1beta1
+apiVersion: core.gardener.cloud/v1
 kind: ControllerDeployment
 metadata:
   name: os-metal
-type: helm
-providerConfig:
-  chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/os-metal-extension/' + gardener_os_controller_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}"
+helm:
+  rawChart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/os-metal-extension/' + gardener_os_controller_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].helm.rawChart }}"
   values:
     image:
       repository: "{{ gardener_os_controller_image_name }}"

diff --git a/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml b/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml
@@ -1,11 +1,10 @@
 ---
-apiVersion: core.gardener.cloud/v1beta1
+apiVersion: core.gardener.cloud/v1
 kind: ControllerDeployment
 metadata:
   name: provider-gcp
-type: helm
-providerConfig:
-  chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-provider-gcp/' + gardener_extension_provider_gcp_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}"
+helm:
+  rawChart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-provider-gcp/' + gardener_extension_provider_gcp_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].helm.rawChart }}"
   values:
     image:
       tag: {{ gardener_extension_provider_gcp_image_tag }}
diff --git a/control-plane/roles/headscale/defaults/main/db.yaml b/control-plane/roles/headscale/defaults/main/db.yaml
@@ -12,6 +12,7 @@ headscale_db_backup_restore_sidecar_provider: local
 headscale_db_backup_restore_sidecar_backup_cron_schedule: "0 0 * * *"
 headscale_db_backup_restore_sidecar_log_level: debug
 headscale_db_backup_restore_sidecar_object_prefix: "{{ headscale_db_name }}"
+headscale_db_backup_restore_sidecar_encryption_key:
 
 headscale_db_backup_restore_sidecar_gcp_bucket_name:
 headscale_db_backup_restore_sidecar_gcp_backup_location:

diff --git a/control-plane/roles/headscale/tasks/main.yaml b/control-plane/roles/headscale/tasks/main.yaml
@@ -50,6 +50,7 @@
     postgres_backup_restore_sidecar_gcp_backup_location: "{{ headscale_db_backup_restore_sidecar_gcp_backup_location }}"
     postgres_backup_restore_sidecar_gcp_project_id: "{{ headscale_db_backup_restore_sidecar_gcp_project_id }}"
     postgres_backup_restore_sidecar_gcp_serviceaccount_json: "{{ headscale_db_backup_restore_sidecar_gcp_serviceaccount_json }}"
+    postgres_backup_restore_sidecar_encryption_key: "{{ headscale_db_backup_restore_sidecar_encryption_key }}"
     postgres_resources: "{{ headscale_db_resources }}"
 
 - name: Deploy headscale

diff --git a/control-plane/roles/ipam-db/defaults/main/main.yaml b/control-plane/roles/ipam-db/defaults/main/main.yaml
@@ -17,6 +17,7 @@ ipam_db_backup_restore_sidecar_backup_cron_schedule: "*/3 * * * *"
 ipam_db_backup_restore_sidecar_log_level: debug
 ipam_db_backup_restore_sidecar_object_prefix: "{{ ipam_db_name }}-{{ metal_control_plane_stage_name }}"
 ipam_db_backup_restore_sidecar_object_max_keep:
+ipam_db_backup_restore_sidecar_encryption_key:
 
 ipam_db_backup_restore_sidecar_gcp_bucket_name:
 ipam_db_backup_restore_sidecar_gcp_backup_location:

diff --git a/control-plane/roles/ipam-db/tasks/main.yaml b/control-plane/roles/ipam-db/tasks/main.yaml
@@ -42,4 +42,4 @@
     postgres_backup_restore_sidecar_gcp_serviceaccount_json: "{{ ipam_db_backup_restore_sidecar_gcp_serviceaccount_json }}"
     postgres_resources: "{{ ipam_db_resources }}"
     postgres_backup_restore_sidecar_object_max_keep: "{{ ipam_db_backup_restore_sidecar_object_max_keep }}"
-
+    postgres_backup_restore_sidecar_encryption_key: "{{ ipam_db_backup_restore_sidecar_encryption_key }}"
diff --git a/control-plane/roles/masterdata-db/defaults/main/main.yaml b/control-plane/roles/masterdata-db/defaults/main/main.yaml
@@ -17,6 +17,7 @@ masterdata_db_backup_restore_sidecar_backup_cron_schedule: "*/3 * * * *"
 masterdata_db_backup_restore_sidecar_log_level: debug
 masterdata_db_backup_restore_sidecar_object_prefix: "{{ masterdata_db_name }}-{{ metal_control_plane_stage_name }}"
 masterdata_db_backup_restore_sidecar_object_max_keep:
+masterdata_db_backup_restore_sidecar_encryption_key:
 
 masterdata_db_backup_restore_sidecar_gcp_bucket_name:
 masterdata_db_backup_restore_sidecar_gcp_backup_location:

diff --git a/control-plane/roles/masterdata-db/tasks/main.yaml b/control-plane/roles/masterdata-db/tasks/main.yaml
@@ -42,3 +42,4 @@
     postgres_backup_restore_sidecar_gcp_serviceaccount_json: "{{ masterdata_db_backup_restore_sidecar_gcp_serviceaccount_json }}"
     postgres_resources: "{{ masterdata_db_resources }}"
     postgres_backup_restore_sidecar_object_max_keep: "{{ masterdata_db_backup_restore_sidecar_object_max_keep }}"
+    postgres_backup_restore_sidecar_encryption_key: "{{ masterdata_db_backup_restore_sidecar_encryption_key }}"
diff --git a/control-plane/roles/meili-backup-restore/README.md b/control-plane/roles/meili-backup-restore/README.md
@@ -8,27 +8,29 @@ This role uses variables from [control-plane-defaults](/control-plane). So, make
 
 You can look up all the default values of this role [here](defaults/main/main.yaml).
 
-| Name                                                       | Mandatory | Description                                                             |
-| ---------------------------------------------------------- | --------- | ----------------------------------------------------------------------- |
-| meilisearch_image_name                                     | yes       | Image version of the meilisearch                                        |
-| meilisearch_image_tag                                      | yes       | Image tag of the meilisearch                                            |
-| meilisearch_registry_auth_enabled                          |           | Enables registry authentication                                         |
-| meilisearch_registry_auth                                  |           | The dockerconfigjson content used for registry authentication           |
-| meilisearch_image_pull_policy                              |           | Image pull policy (defaults to IfNotPresent)                            |
-| meilisearch_name                                           |           | The name of the meilisearch instance                                    |
-| meilisearch_namespace                                      |           | The deployment's target namespace                                       |
-| meilisearch_storage_size                                   |           | The size of the PVC                                                     |
-| meilisearch_storage_class                                  |           | The storage class of the PVC                                            |
-| meilisearch_api_key                                        |           | The api key for meilisearch                                             |
-| meilisearch_environment                                    |           | Sets the environment configuration for meilisearch                      |
-| meilisearch_no_analytics                                   |           | Sets the no analytics configuration for meilisearch                     |
-| meilisearch_backup_restore_sidecar_image_name              | yes       | Image version of the backup-restore-sidecar                             |
-| meilisearch_backup_restore_sidecar_image_tag               | yes       | Image tag of the backup-restore-sidecar                                 |
-| meilisearch_backup_restore_sidecar_provider                |           | The backup provider                                                     |
-| meilisearch_backup_restore_sidecar_backup_cron_schedule    |           | The backup cron schedule                                                |
-| meilisearch_backup_restore_sidecar_log_level               |           | The log level of the sidecar                                            |
-| meilisearch_backup_restore_sidecar_gcp_bucket_name         |           | Bucket name of the GCP bucket                                           |
-| meilisearch_backup_restore_sidecar_gcp_backup_location     |           | Location of the GCP bucket                                              |
-| meilisearch_backup_restore_sidecar_gcp_project_id          |           | GCP project name                                                        |
-| meilisearch_backup_restore_sidecar_gcp_serviceaccount_json |           | GCP Serviceaccount JSON string (service account requires bucket access) |
-| meilisearch_resources                                      |           | The kubernetes resources for the actual meilisearch container           |
+| Name                                                       | Mandatory | Description                                                                                                       |
+| ---------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------------- |
+| meilisearch_image_name                                     | yes       | Image version of the meilisearch                                                                                  |
+| meilisearch_image_tag                                      | yes       | Image tag of the meilisearch                                                                                      |
+| meilisearch_registry_auth_enabled                          |           | Enables registry authentication                                                                                   |
+| meilisearch_registry_auth                                  |           | The dockerconfigjson content used for registry authentication                                                     |
+| meilisearch_image_pull_policy                              |           | Image pull policy (defaults to IfNotPresent)                                                                      |
+| meilisearch_name                                           |           | The name of the meilisearch instance                                                                              |
+| meilisearch_namespace                                      |           | The deployment's target namespace                                                                                 |
+| meilisearch_storage_size                                   |           | The size of the PVC                                                                                               |
+| meilisearch_storage_class                                  |           | The storage class of the PVC                                                                                      |
+| meilisearch_api_key                                        |           | The api key for meilisearch                                                                                       |
+| meilisearch_environment                                    |           | Sets the environment configuration for meilisearch                                                                |
+| meilisearch_no_analytics                                   |           | Sets the no analytics configuration for meilisearch                                                               |
+| meilisearch_backup_restore_sidecar_image_name              | yes       | Image version of the backup-restore-sidecar                                                                       |
+| meilisearch_backup_restore_sidecar_image_tag               | yes       | Image tag of the backup-restore-sidecar                                                                           |
+| meilisearch_backup_restore_sidecar_provider                |           | The backup provider                                                                                               |
+| meilisearch_backup_restore_sidecar_backup_cron_schedule    |           | The backup cron schedule                                                                                          |
+| meilisearch_backup_restore_sidecar_log_level               |           | The log level of the sidecar                                                                                      |
+| meilisearch_backup_restore_sidecar_gcp_bucket_name         |           | Bucket name of the GCP bucket                                                                                     |
+| meilisearch_backup_restore_sidecar_gcp_backup_location     |           | Location of the GCP bucket                                                                                        |
+| meilisearch_backup_restore_sidecar_gcp_project_id          |           | GCP project name                                                                                                  |
+| meilisearch_backup_restore_sidecar_gcp_serviceaccount_json |           | GCP Serviceaccount JSON string (service account requires bucket access)                                           |
+| meilisearch_resources                                      |           | The kubernetes resources for the actual meilisearch container                                                     |
+| meilisearch_backup_restore_sidecar_object_max_keep         |           | The number of objects to keep at the cloud provider bucket                                                        |
+| meilisearch_backup_restore_sidecar_encryption_key          |           | An optional encryption key to AES-encrypt the backups before uploading them to the backup provider (length == 32) |
diff --git a/control-plane/roles/meili-backup-restore/defaults/main/main.yaml b/control-plane/roles/meili-backup-restore/defaults/main/main.yaml
@@ -17,6 +17,7 @@ meilisearch_backup_restore_sidecar_backup_cron_schedule: "0 * * * *"
 meilisearch_backup_restore_sidecar_log_level: debug
 meilisearch_backup_restore_sidecar_object_prefix: "{{ meilisearch_name }}-{{ metal_control_plane_stage_name }}"
 meilisearch_backup_restore_sidecar_object_max_keep:
+meilisearch_backup_restore_sidecar_encryption_key:
 
 meilisearch_backup_restore_sidecar_gcp_bucket_name:
 meilisearch_backup_restore_sidecar_gcp_backup_location:

diff --git a/control-plane/roles/meili-backup-restore/tasks/main.yml b/control-plane/roles/meili-backup-restore/tasks/main.yml
@@ -11,6 +11,7 @@
       - meilisearch_image_tag is defined
       - meilisearch_backup_restore_sidecar_image_name is defined
       - meilisearch_backup_restore_sidecar_image_tag is defined
+      - meilisearch_backup_restore_sidecar_encryption_key is none or meilisearch_backup_restore_sidecar_encryption_key | length == 32
 
 - name: Deploy meilisearch (backup-restore)
   k8s:

diff --git a/control-plane/roles/meili-backup-restore/templates/meilisearch.yaml b/control-plane/roles/meili-backup-restore/templates/meilisearch.yaml
@@ -231,6 +231,9 @@ data:
     compression-method: targz
 {% if meilisearch_backup_restore_sidecar_object_max_keep %}
     object-max-keep: {{ meilisearch_backup_restore_sidecar_object_max_keep }}
+{% endif %}
+{% if meilisearch_backup_restore_sidecar_encryption_key %}
+    encryption-key: {{ meilisearch_backup_restore_sidecar_encryption_key }}
 {% endif %}
     post-exec-cmds:
       - meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files

diff --git a/control-plane/roles/metal-db/defaults/main/main.yaml b/control-plane/roles/metal-db/defaults/main/main.yaml
@@ -14,6 +14,7 @@ metal_db_backup_restore_sidecar_backup_cron_schedule: "*/3 * * * *"
 metal_db_backup_restore_sidecar_log_level: debug
 
 metal_db_backup_restore_sidecar_object_max_keep:
+metal_db_backup_restore_sidecar_encryption_key:
 
 metal_db_backup_restore_sidecar_gcp_bucket_name:
 metal_db_backup_restore_sidecar_gcp_backup_location: