Modifications to get flatcar working

Temporary changes for now just to get Flatcar build back to a working state. Will need to sort out whether a conditional is needed in kubeadm-control-plane.
azimuth-cloud · Oct 12, 2023 · 81247e5 · 81247e5
1 parent b643ff3
commit 81247e5
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -24,3 +24,61 @@ Currently, the following charts are available:
 | --- | --- |
 | [cluster-addons](./charts/cluster-addons) | Deploys addons into a Kubernetes cluster, e.g. CNI. |
 | [openstack-cluster](./charts/openstack-cluster) | Deploys a Kubernetes cluster on an OpenStack cloud. |
+
+
+## Flatcar
+
+To deploy clusters which use Ignition such as Flatcar, you will need to override the following settings in your local `values.yaml`:
+
+```yaml
+ignitionBasedOS: true 
+
+controlPlane.kubeadmConfigSpec.initConfiguration.nodeRegistration.name: ${COREOS_OPENSTACK_HOSTNAME}
+controlPlane.kubeadmConfigSpec.clusterConfiguration.joinConfiguration.nodeRegistration.name: ${COREOS_OPENSTACK_HOSTNAME}
+controlPlane.kubeadmConfigSpec.clusterConfiguration.preKubeadmCommands:
+  - export COREOS_OPENSTACK_HOSTNAME=${COREOS_OPENSTACK_HOSTNAME%.*}
+  - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
+  - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
+controlPlane.kubeadmConfigSpec.clusterConfiguration.format: ignition
+controlPlane.kubeadmConfigSpec.clusterConfiguration.ignition:
+  containerLinuxConfig:
+    additionalConfig: |
+      systemd:
+        units:
+        - name: [email protected]
+          enabled: true
+        - name: kubeadm.service
+          enabled: true
+          dropins:
+          - name: 10-flatcar.conf
+            contents: |
+              [Unit]
+              Requires=containerd.service coreos-metadata.service
+              After=containerd.service coreos-metadata.service
+              [Service]
+              EnvironmentFile=/run/metadata/flatcar
+
+nodeGroupDefaults.kubeadmConfigSpec.format: ignition
+nodeGroupDefaults.kubeadmConfigSpec.ignition:
+  containerLinuxConfig:
+    additionalConfig: |
+      systemd:
+        units:
+        - name: [email protected]
+          enabled: true
+        - name: kubeadm.service
+          enabled: true
+          dropins:
+          - name: 10-flatcar.conf
+            contents: |
+              [Unit]
+              Requires=containerd.service coreos-metadata.service
+              After=containerd.service coreos-metadata.service
+              [Service]
+              EnvironmentFile=/run/metadata/flatcar
+nodeGroupDefaults.kubeadmConfigSpec.joinConfiguration.nodeRegistration.name: ${COREOS_OPENSTACK_HOSTNAME}
+nodeGroupDefaults.kubeadmConfigSpec.preKubeadmCommands:
+  - export COREOS_OPENSTACK_HOSTNAME=${COREOS_OPENSTACK_HOSTNAME%.*}
+  - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
+  - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
+```
diff --git a/charts/openstack-cluster/templates/_helpers.tpl b/charts/openstack-cluster/templates/_helpers.tpl
@@ -203,10 +203,11 @@ files:
       # This file is created by the capi-helm-chart to ensure that its parent directory exists
     owner: root:root
     permissions: "0644"
-  - path: /etc/containerd/config.toml
+  - path: /etc/containerd/config.d/containerd-certs.toml
     content: |
-      [plugins."io.containerd.grpc.v1.cri".registry]
-      config_path = "/etc/containerd/certs.d"
+      [plugins]
+        [plugins."io.containerd.grpc.v1.cri".registry]
+        config_path = "/etc/containerd/certs.d"
     owner: root:root
     permissions: "0644"
     append: true

diff --git a/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml b/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml
@@ -21,6 +21,22 @@ preKubeadmCommands:
 {{- end }}
 {{- end }}
 
+{{- define "openstack-cluster.controlplane.kubeadmConfigSpec.ignitionKubeProxyConfiguration" -}}
+{{- with .kubeProxyConfiguration }}
+files:
+  - path: /etc/kube-proxy-configuration.yaml
+    content: |
+      ---
+      apiVersion: kubeproxy.config.k8s.io/v1alpha1
+      kind: KubeProxyConfiguration
+      {{- toYaml . | nindent 6 }}
+    owner: root:root
+    permissions: "0644"
+preKubeadmCommands:
+  - cat /etc/kube-proxy-configuration.yaml >> /run/kubeadm.yml
+{{- end }}
+{{- end }}
+
 ---
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 kind: KubeadmControlPlane
@@ -47,6 +63,22 @@ spec:
     nodeDrainTimeout: {{ .Values.controlPlane.nodeDrainTimeout }}
     nodeVolumeDetachTimeout: {{ .Values.controlPlane.nodeVolumeDetachTimeout }}
     nodeDeletionTimeout: {{ .Values.controlPlane.nodeDeletionTimeout }}
+  {{- if .Values.ignitionBasedOS }}
+  kubeadmConfigSpec: {{
+    omit
+      (
+        list
+          (include "openstack-cluster.controlplane.kubeadmConfigSpec.nodeLabels" . | fromYaml)
+          (include "openstack-cluster.kubeadmConfigSpec" (list . .Values.controlPlane.kubeadmConfigSpec) | fromYaml) 
+          (include "openstack-cluster.controlplane.kubeadmConfigSpec.ignitionKubeProxyConfiguration" .Values.controlPlane.kubeadmConfigSpec | fromYaml) |
+        include "openstack-cluster.mergeConcatMany" |
+        fromYaml
+      )
+      "kubeProxyConfiguration" |
+    toYaml |
+    nindent 4
+  }}
+  {{- else }}
   kubeadmConfigSpec: {{
     omit
       (
@@ -61,3 +93,4 @@ spec:
     toYaml |
     nindent 4
   }}
+  {{- end }}
diff --git a/charts/openstack-cluster/values.yaml b/charts/openstack-cluster/values.yaml
@@ -115,6 +115,11 @@ apiServer:
   # The port to use for the API server
   port: 6443
 
+
+# WARN(travis): Set ignition based OS
+# ignitionBasedOS:
+
+
 # Settings for the control plane
 controlPlane:
   # The failure domains to use for control plane nodes
@@ -151,8 +156,6 @@ controlPlane:
     # The volume availability zone to use
     # If not specified, the machine availability zone is used
     # availabilityZone:
-  # The ID of the server group to use for control plane machines
-  serverGroupId:
   # Labels to apply to the node objects in Kubernetes that correspond to control plane machines
   nodeLabels:
     # my.company.org/label: value
@@ -176,7 +179,7 @@ controlPlane:
   kubeadmConfigSpec:
     initConfiguration:
       nodeRegistration:
-        name: '{{ local_hostname }}'
+        name: ${COREOS_OPENSTACK_HOSTNAME}
         kubeletExtraArgs:
           cloud-provider: external
     # As well as enabling an external cloud provider, we set the bind addresses for the
@@ -258,10 +261,8 @@ nodeGroupDefaults:
     # The volume availability zone to use
     # If not specified, the machine availability zone is used
     # availabilityZone:
-  # The ID of the server group to use for machines in the node group
-  serverGroupId:
   # Labels to apply to the node objects in Kubernetes that correspond to machines in the node group
-  # By default, nodes get the label "capi.stackhpc.com/node-group=<node group name>"
+  # By default, nodes that are part of a node group get the label "capi.stackhpc.com/node-group=<node group name>"
   nodeLabels:
     # my.company.org/label: value
   # The time to wait for a node to finish draining before it can be removed