feat(gardener): network isolation #235 (#259)

control-plane/README.md

@@ -32,6 +32,7 @@ The `control-plane-defaults` folder contains defaults that are used by multiple
 | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
 | [headscale](roles/headscale)                               | Deploys headscale                                                                                                               |
 | [ipam-db](roles/ipam-db)                                   | Deploys a database for the [IPAM](https://github.com/metal-stack/go-ipam) of the metal-api                                      |
+| [isolated-clusters](roles/isolated-clusters)               | Deploys services for isolated clusters                                                                                          |
 | [masterdata-db](roles/masterdata-db)                       | Deploys a database for the masterdata-api                                                                                       |
 | [metal](roles/metal)                                       | Deploys all metal-stack components of the metal-control-plane via Helm                                                          |
 | [metal-db](roles/metal-db)                                 | Deploys a database for the metal-api                                                                                            |

control-plane/roles/isolated-clusters/README.md

@@ -0,0 +1,42 @@
+# isolated clusters
+
+Contains roles for deploying addtional services for the isolated cluster feature as described [here](https://docs.metal-stack.io/stable/overview/isolated-kubernetes/).
+
+It contains the services:
+
+- oci-mirror
+- Registry
+- DNS
+- NTP
+
+A cert-manager is expected to be deployed by the user beforehand.
+
+## Variables
+
+The `control-plane-defaults` folder contains defaults that are used by multiple roles in the control-plane directory. You can look up all the default values [here](control-plane-defaults/main.yaml).
+
+| Name                                                             | Mandatory | Description                                                                                      |
+| ---------------------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------ |
+| isolated_clusters_virtual_garden_kubeconfig                      |           | The kubeconfig to access the virutal garden as a string value.                                              |
+| isolated_clusters_ntp_image_name                                 |           | The image name of the ntp service for the partition.                                             |
+| isolated_clusters_ntp_image_tag                                  | yes       | The tag or version of the ntp service container image.                                           |
+| isolated_clusters_ntp_namespace                                  |           | The namespace to deploy the ntp server to.                                                       |
+| isolated_clusters_dns_image_name                                 |           | The image name of the dns service for the partition.                                             |
+| isolated_clusters_dns_image_tag                                  | yes       | The tag or version of the dns service container image.                                           |
+| isolated_clusters_dns_namespace                                  |           | The namespace to deploy the dns server to.                                                       |
+| isolated_clusters_ingress_controller_namespace                   |           | The namespace where the ingress controller should be deployed to.                                |
+| isolated_clusters_ingress_controller_chart_version               | yes       | The version of the ingress controller chart.                                                     |
+| isolated_clusters_ingress_controller_chroot                      |           | Indicates if the image should have a changed root.                                               |
+| isolated_clusters_ingress_controller_load_balancer_ip            | yes       | The load balancer source ip of the ingress controller.                                           |
+| isolated_clusters_ingress_controller_load_balancer_source_ranges | yes       | The load balancer source ranges of the ingress controller.                                       |
+| isolated_clusters_registry_image_name                            |           | The image name of the registry service for the partition.                                        |
+| isolated_clusters_registry_image_tag                             | yes       | The tag or version of the registry service container image.                                      |
+| isolated_clusters_registry_namespace                             |           | The namespace for the registry used for isolated clusters.                                       |
+| isolated_clusters_registry_oci_mirror_image_name                 |           | The OCI mirror image of the registry used for isolated clusters.                                 |
+| isolated_clusters_registry_oci_mirror_image_tag                  | yes       | The OCI mirror version of the registry used for isolated clusters.                               |
+| isolated_clusters_registry_oci_mirror_config                     |           | Contains a mapping of source and destination images for specific versions.                       |
+| isolated_clusters_registry_storage_size                          |           | The storage size of the registry used for isolated clusters.                                     |
+| isolated_clusters_registry_storage_class_name                    |           | The storageClassName of the registry used for isolated clusters.                                 |
+| isolated_clusters_registry_ingress_fqdn                          | yes       | The full name of the registry used for isolated clusters.                                        |
+| isolated_clusters_registry_ingress_annotations                   |           | Optional ingress annotations for the registry used for isolated clusters.                        |
+| isolated_clusters_partition_services_cluster                     |           | The cluster to deploy the services like ntp, dns, ingress, cert manager and the OCI registry to. |

control-plane/roles/isolated-clusters/defaults/main/control-plane-defaults

@@ -0,0 +1 @@
+../../../../control-plane-defaults/

control-plane/roles/isolated-clusters/defaults/main/global-defaults

@@ -0,0 +1 @@
+../../../../../defaults

control-plane/roles/isolated-clusters/defaults/main/images.yaml

@@ -0,0 +1,131 @@
+---
+isolated_clusters_registry_oci_mirror_config:
+  images:
+    - source: docker.io/fluent/fluentd
+      destination: http://registry:5000/fluent/fluentd
+      match:
+        semver: ">= v1.12"
+    - source: quay.io/calico/cni
+      destination: http://registry:5000/calico/cni
+      match:
+        semver: ">= v3.25.0"
+    - source: quay.io/calico/node
+      destination: http://registry:5000/calico/node
+      match:
+        semver: ">= v3.25.0"
+    - source: docker.lightbitslabs.com/lightos-csi/lb-csi-plugin
+      destination: http://registry:5000/lightos-csi/lb-csi-plugin
+      match:
+        semver: ">= 1.14.0"
+    - source: docker.lightbitslabs.com/lightos-csi/lb-nvme-discovery-client
+      destination: http://registry:5000/lightos-csi/lb-nvme-discovery-client
+      match:
+        semver: ">= 1.14.0"
+    - source: eu.gcr.io/gardener-project/hyperkube
+      destination: http://registry:5000/gardener-project/hyperkube
+      match:
+        semver: ">= v1.27.8"
+    - source: europe-docker.pkg.dev/gardener-project/releases/hyperkube
+      destination: http://registry:5000/gardener-project/releases/hyperkube
+      match:
+        semver: ">= v1.27.10"
+    - source: eu.gcr.io/gardener-project/3rd/alpine
+      destination: http://registry:5000/gardener-project/3rd/alpine
+      match:
+        semver: ">= 3.15.8"
+    - source: eu.gcr.io/gardener-project/3rd/coredns/coredns
+      destination: http://registry:5000/gardener-project/3rd/coredns/coredns
+      match:
+        semver: ">= 1.10.0"
+    - source: eu.gcr.io/gardener-project/3rd/envoyproxy/envoy-distroless
+      destination: http://registry:5000/gardener-project/3rd/envoyproxy/envoy-distroless
+      match:
+        semver: ">= v1.24.1"
+    - source: eu.gcr.io/gardener-project/gardener/apiserver-proxy
+      destination: http://registry:5000/gardener-project/gardener/apiserver-proxy
+      match:
+        semver: ">= v0.12.0"
+    - source: eu.gcr.io/gardener-project/gardener/vpn-shoot-client
+      destination: http://registry:5000/gardener-project/gardener/vpn-shoot-client
+      match:
+        semver: ">= 0.16.0"
+    - source: ghcr.io/metal-stack/metallb-health-sidecar
+      destination: http://registry:5000/metal-stack/metallb-health-sidecar
+      match:
+        semver: ">= v0.1.1"
+    - source: quay.io/metallb/controller
+      destination: http://registry:5000/metallb/controller
+      match:
+        semver: ">= v0.10.3"
+    - source: quay.io/metallb/speaker
+      destination: http://registry:5000/metallb/speaker
+      match:
+        semver: ">= v0.10.3"
+    - source: quay.io/prometheus/blackbox-exporter
+      destination: http://registry:5000/prometheus/blackbox-exporter
+      match:
+        semver: ">= v0.23.0"
+    - source: quay.io/prometheus/node-exporter
+      destination: http://registry:5000/prometheus/node-exporter
+      match:
+        semver: ">= v1.5.0"
+    - source: registry.k8s.io/pause
+      destination: http://registry:5000/pause
+      match:
+        semver: ">= 3.6"
+    - source: registry.k8s.io/cpa/cpvpa
+      destination: http://registry:5000/cpa/cpvpa
+      match:
+        semver: ">= v0.8.4"
+    - source: registry.k8s.io/kube-proxy
+      destination: http://registry:5000/kube-proxy
+      match:
+        semver: ">= v1.27.8"
+    - source: registry.k8s.io/metrics-server/metrics-server
+      destination: http://registry:5000/metrics-server/metrics-server
+      match:
+        semver: ">= v0.6.3"
+    - source: registry.k8s.io/node-problem-detector/node-problem-detector
+      destination: http://registry:5000/node-problem-detector/node-problem-detector
+      match:
+        semver: ">= v0.8.13"
+    - source: registry.k8s.io/sig-storage/csi-attacher
+      destination: http://registry:5000/sig-storage/csi-attacher
+      match:
+        semver: ">= v4.4.0"
+    - source: registry.k8s.io/sig-storage/csi-node-driver-registrar
+      destination: http://registry:5000/sig-storage/csi-node-driver-registrar
+      match:
+        semver: ">= v2.9.0"
+    - source: registry.k8s.io/sig-storage/csi-provisioner
+      destination: http://registry:5000/sig-storage/csi-provisioner
+      match:
+        semver: ">= v3.6.0"
+    - source: registry.k8s.io/sig-storage/csi-resizer
+      destination: http://registry:5000/sig-storage/csi-resizer
+      match:
+        semver: ">= v1.9.0"
+    - source: registry.k8s.io/sig-storage/csi-snapshotter
+      destination: http://registry:5000/sig-storage/csi-snapshotter
+      match:
+        semver: ">= v6.3.0"
+    - source: registry.k8s.io/sig-storage/snapshot-controller
+      destination: http://registry:5000/sig-storage/snapshot-controller
+      match:
+        semver: ">= v6.3.0"
+    - source: r.metal-stack.io/csi-lvm-controller
+      destination: http://registry:5000/csi-lvm-controller
+      match:
+        semver: ">= v0.7.0"
+    - source: r.metal-stack.io/csi-lvm-provisioner
+      destination: http://registry:5000/csi-lvm-provisioner
+      match:
+        semver: ">= v0.7.0"
+    - source: r.metal-stack.io/droptailer
+      destination: http://registry:5000/droptailer
+      match:
+        semver: ">= v0.2.12"
+    - source: r.metal-stack.io/node-init
+      destination: http://registry:5000/node-init
+      match:
+        semver: ">= v0.1.4"

control-plane/roles/isolated-clusters/defaults/main/main.yaml

@@ -0,0 +1,31 @@
+---
+isolated_clusters_virtual_garden_kubeconfig: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace='garden', resource_name='garden-kubeconfig-for-admin').get('data', {}).get('kubeconfig') | b64decode }}"
+
+isolated_clusters_ntp_image_name: ghcr.io/mwennrich/chrony
+isolated_clusters_ntp_image_tag:
+isolated_clusters_ntp_namespace: ntp-service
+
+isolated_clusters_dns_image_name: coredns/coredns
+isolated_clusters_dns_image_tag:
+isolated_clusters_dns_namespace: dns-service
+
+isolated_clusters_ingress_controller_namespace: ingress
+isolated_clusters_ingress_controller_chart_version:
+isolated_clusters_ingress_controller_chroot: true
+isolated_clusters_ingress_controller_load_balancer_ip:
+isolated_clusters_ingress_controller_load_balancer_source_ranges:
+
+isolated_clusters_group_label: "{{ metal_control_plane_stage_name }}"
+isolated_clusters_partition_services_cluster: []
+  # - name: somename
+  #   kubeconfig: "{{ isolated_clusters_virtual_garden_kubeconfig | shoot_admin_kubeconfig('garden-...', 'somename') | from_yaml }}"
+
+isolated_clusters_registry_image_name: docker.io/registry
+isolated_clusters_registry_image_tag:
+isolated_clusters_registry_namespace: registry
+isolated_clusters_registry_oci_mirror_image_name: "ghcr.io/metal-stack/oci-mirror"
+isolated_clusters_registry_oci_mirror_image_tag:
+isolated_clusters_registry_storage_size: "100Gi"
+isolated_clusters_registry_storage_class_name:
+isolated_clusters_registry_ingress_fqdn:
+isolated_clusters_registry_ingress_annotations: {}

control-plane/roles/isolated-clusters/tasks/dns.yaml

@@ -0,0 +1,23 @@
+---
+- name: Create namespace {{ isolated_clusters_dns_namespace }}
+  k8s:
+    definition:
+      api_version: v1
+      kind: Namespace
+      metadata:
+        name: "{{ isolated_clusters_dns_namespace }}"
+        labels:
+          app.kubernetes.io/part-of: "{{ isolated_clusters_group_label }}"
+    kubeconfig: "{{ cluster.kubeconfig }}"
+    apply: yes
+
+- name: Deploy DNS service
+  k8s:
+    definition: "{{ lookup('template', 'dns/' + item) }}"
+    namespace: "{{ isolated_clusters_dns_namespace }}"
+    kubeconfig: "{{ cluster.kubeconfig }}"
+    apply: yes
+  loop:
+    - coredns-configmap.yaml
+    - coredns-deployment.yaml
+    - coredns-service.yaml

control-plane/roles/isolated-clusters/tasks/ingress.yaml

@@ -0,0 +1,22 @@
+---
+- name: Create namespace for ingress-controller
+  k8s:
+    definition:
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: "{{ isolated_clusters_ingress_controller_namespace }}"
+        labels:
+          app.kubernetes.io/part-of: "{{ isolated_clusters_group_label }}"
+    apply: yes
+    kubeconfig: "{{ cluster.kubeconfig }}"
+
+- name: Deploy ingress nginx controller
+  kubernetes.core.helm:
+    name: ingress-nginx
+    chart_repo_url: https://kubernetes.github.io/ingress-nginx
+    chart_ref: ingress-nginx
+    chart_version: "{{ isolated_clusters_ingress_controller_chart_version }}"
+    namespace: "{{ isolated_clusters_ingress_controller_namespace }}"
+    values: "{{ lookup('template', 'ingress/values.yaml') | from_yaml }}"
+    kubeconfig: "{{ cluster.kubeconfig }}"

control-plane/roles/isolated-clusters/tasks/main.yaml

@@ -0,0 +1,21 @@
+---
+- name: Check mandatory variables for this role are set
+  assert:
+    fail_msg: "not all mandatory variables given, check role documentation"
+    quiet: yes
+    that:
+      - isolated_clusters_ntp_image_tag is not none
+      - isolated_clusters_dns_image_tag is not none
+      - isolated_clusters_registry_image_tag is not none
+      - isolated_clusters_registry_oci_mirror_image_tag is not none
+      - isolated_clusters_ingress_controller_chart_version is not none
+      - isolated_clusters_ingress_controller_load_balancer_ip is not none
+      - isolated_clusters_ingress_controller_load_balancer_source_ranges is not none
+      - isolated_clusters_registry_ingress_fqdn is not none
+
+- name: Loop over clusters to install services
+  include_tasks: services.yaml
+  loop: "{{ isolated_clusters_partition_services_cluster }}"
+  loop_control:
+    loop_var: cluster
+    label: "{{ cluster.name }}"

control-plane/roles/isolated-clusters/tasks/ntp.yaml

@@ -0,0 +1,23 @@
+---
+- name: Create namespace {{ isolated_clusters_ntp_namespace }}
+  k8s:
+    definition:
+      api_version: v1
+      kind: Namespace
+      metadata:
+        name: "{{ isolated_clusters_ntp_namespace }}"
+        labels:
+          app.kubernetes.io/part-of: "{{ isolated_clusters_group_label }}"
+    kubeconfig: "{{ cluster.kubeconfig }}"
+    apply: yes
+
+- name: Deploy NTP service
+  k8s:
+    definition: "{{ lookup('template', 'ntp/' + item) }}"
+    namespace: "{{ isolated_clusters_ntp_namespace }}"
+    kubeconfig: "{{ cluster.kubeconfig }}"
+    apply: yes
+  loop:
+    - chrony-configmap.yaml
+    - chrony-deployment.yaml
+    - chrony-service.yaml

control-plane/roles/isolated-clusters/tasks/registry.yaml

@@ -0,0 +1,24 @@
+---
+- name: Create namespace {{ isolated_clusters_registry_namespace }}
+  k8s:
+    definition:
+      api_version: v1
+      kind: Namespace
+      metadata:
+        name: "{{ isolated_clusters_registry_namespace }}"
+        labels:
+          app.kubernetes.io/part-of: "{{ isolated_clusters_group_label }}"
+    kubeconfig: "{{ cluster.kubeconfig }}"
+    apply: yes
+
+- name: Deploy registry
+  k8s:
+    definition: "{{ lookup('template', 'registry/' + item) }}"
+    namespace: "{{ isolated_clusters_registry_namespace }}"
+    kubeconfig: "{{ cluster.kubeconfig }}"
+    apply: yes
+  loop:
+    - registry-sts.yaml
+    - registry-service.yaml
+    - registry-ingress.yaml
+    - registry-oci-mirror.yaml

control-plane/roles/isolated-clusters/tasks/services.yaml

@@ -0,0 +1,14 @@
+---
+# called with loop_var cluster
+
+- name: Install nginx-ingress
+  import_tasks: ingress.yaml
+
+- name: Install ntp
+  import_tasks: ntp.yaml
+
+- name: Install dns
+  import_tasks: dns.yaml
+
+- name: Install deployment
+  import_tasks: registry.yaml

control-plane/roles/isolated-clusters/templates/dns/coredns-configmap.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  labels:
+    app.kubernetes.io/name: coredns
+    app.kubernetes.io/component: dns
+    app.kubernetes.io/part-of: isolated-clusters
+    app.kubernetes.io/managed-by: ansible
+    app.kubernetes.io/part-of: "{{ isolated_clusters_group_label }}"
+data:
+  Corefile: |
+    .:8053 {
+      errors
+      log . {
+          class error
+      }
+      health {
+          lameduck 15s
+      }
+      ready
+      prometheus 0.0.0.0:9153
+      forward . 8.8.8.8 8.8.4.4
+      cache {
+              success  39936
+              denial   39936
+              prefetch 2000
+      }
+      loop
+      reload
+      template ANY AAAA {
+        rcode NOERROR
+      }
+    }