From 738039c7e69ec5b8d117192c49a27d9efa906f11 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Tue, 11 Oct 2022 17:27:54 +0300 Subject: [PATCH] Working calico ipip configuration (#27) --- apis/v1beta1/microk8sconfig_types.go | 14 ++++++++----- bootstrap-components.yaml | 20 +++++++++++++++---- ...trap.cluster.x-k8s.io_microk8sconfigs.yaml | 7 +++++-- ...ster.x-k8s.io_microk8sconfigtemplates.yaml | 7 +++++-- controllers/cloudinit/cloudinit_test.go | 1 + controllers/cloudinit/controlplane_init.go | 9 ++++++--- controllers/cloudinit/controlplane_join.go | 6 ++++++ controllers/cloudinit/utils.go | 17 ++++++++++++++++ controllers/microk8sconfig_controller.go | 7 +++++-- examples/aws-capi-quickstart.yaml | 3 ++- templates/cluster-template-aws.yaml | 1 + 11 files changed, 73 insertions(+), 19 deletions(-) diff --git a/apis/v1beta1/microk8sconfig_types.go b/apis/v1beta1/microk8sconfig_types.go index 1cc5183..7b258c5 100644 --- a/apis/v1beta1/microk8sconfig_types.go +++ b/apis/v1beta1/microk8sconfig_types.go @@ -31,12 +31,12 @@ import ( type ClusterConfiguration struct { metav1.TypeMeta `json:",inline"` - // cluster agent port (25000) and dqlite port (19001) set to use calico port 179 and etcd port 2380 respectively + // cluster agent port (25000) and dqlite port (19001) set to use ports 30000 and 2379 respectively // The default ports of cluster agent and dqlite are blocked by security groups and as a temporary // workaround we reuse the etcd and calico ports that are open in the infra providers because kubeadm uses those. // PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) - // to 179 and 2380. The default ports are blocked via security groups in several infra providers. + // to 30000 and 2379. The default ports are blocked via security groups in several infra providers. // +kubebuilder:default:=true // +optional PortCompatibilityRemap bool `json:"portCompatibilityRemap,omitempty"` @@ -94,6 +94,10 @@ type InitConfiguration struct { // List of addons to be enabled upon cluster creation // +optional Addons []string `json:"addons,omitempty"` + + // The optional IPinIP configuration + // +optional + IPinIP bool `json:"IPinIP,omitempty"` } // MicroK8sConfigSpec defines the desired state of MicroK8sConfig @@ -130,9 +134,9 @@ type MicroK8sConfigStatus struct { Conditions clusterv1.Conditions `json:"conditions,omitempty"` } -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status -//+kubebuilder:storageversion +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion // MicroK8sConfig is the Schema for the microk8sconfigs API type MicroK8sConfig struct { metav1.TypeMeta `json:",inline"` diff --git a/bootstrap-components.yaml b/bootstrap-components.yaml index 8dd7d0c..81476c4 100644 --- a/bootstrap-components.yaml +++ b/bootstrap-components.yaml @@ -44,11 +44,14 @@ spec: type: string portCompatibilityRemap: default: true - description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 179 and 2380. The default ports are blocked via security groups in several infra providers. + description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 30000 and 2379. The default ports are blocked via security groups in several infra providers. type: boolean type: object initConfiguration: properties: + IPinIP: + description: The optional IPinIP configuration + type: boolean addons: description: List of addons to be enabled upon cluster creation items: @@ -204,11 +207,14 @@ spec: type: string portCompatibilityRemap: default: true - description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 179 and 2380. The default ports are blocked via security groups in several infra providers. + description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 30000 and 2379. The default ports are blocked via security groups in several infra providers. type: boolean type: object initConfiguration: properties: + IPinIP: + description: The optional IPinIP configuration + type: boolean addons: description: List of addons to be enabled upon cluster creation items: @@ -486,11 +492,14 @@ spec: type: string portCompatibilityRemap: default: true - description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 179 and 2380. The default ports are blocked via security groups in several infra providers. + description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 30000 and 2379. The default ports are blocked via security groups in several infra providers. type: boolean type: object initConfiguration: properties: + IPinIP: + description: The optional IPinIP configuration + type: boolean addons: description: List of addons to be enabled upon cluster creation items: @@ -646,11 +655,14 @@ spec: type: string portCompatibilityRemap: default: true - description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 179 and 2380. The default ports are blocked via security groups in several infra providers. + description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) to 30000 and 2379. The default ports are blocked via security groups in several infra providers. type: boolean type: object initConfiguration: properties: + IPinIP: + description: The optional IPinIP configuration + type: boolean addons: description: List of addons to be enabled upon cluster creation items: diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml index 4b879c7..be762bd 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml @@ -56,13 +56,16 @@ spec: portCompatibilityRemap: default: true description: PortCompatibilityRemap switches the default ports - used by cluster agent (25000) and dqlite (19001) to 179 and - 2380. The default ports are blocked via security groups in several + used by cluster agent (25000) and dqlite (19001) to 30000 and + 2379. The default ports are blocked via security groups in several infra providers. type: boolean type: object initConfiguration: properties: + IPinIP: + description: The optional IPinIP configuration + type: boolean addons: description: List of addons to be enabled upon cluster creation items: diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml index 3a1f8b4..062c3e8 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml @@ -66,12 +66,15 @@ spec: default: true description: PortCompatibilityRemap switches the default ports used by cluster agent (25000) and dqlite (19001) - to 179 and 2380. The default ports are blocked via security - groups in several infra providers. + to 30000 and 2379. The default ports are blocked via + security groups in several infra providers. type: boolean type: object initConfiguration: properties: + IPinIP: + description: The optional IPinIP configuration + type: boolean addons: description: List of addons to be enabled upon cluster creation diff --git a/controllers/cloudinit/cloudinit_test.go b/controllers/cloudinit/cloudinit_test.go index f6efa85..c05fcaf 100644 --- a/controllers/cloudinit/cloudinit_test.go +++ b/controllers/cloudinit/cloudinit_test.go @@ -32,6 +32,7 @@ func TestNewInitControlPlaneCommands(t *testing.T) { JoinTokenTTLInSecs: 56789, Version: "v1.23.3", Addons: addons, + IPinIP: true, } out, err := NewInitControlPlane(cpinput) diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index 9c6b94e..1eb4fdf 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -51,6 +51,7 @@ runcmd: - sudo echo Version {{.Version}} - sudo sh -c "while ! snap install microk8s --classic {{.Version}} ; do sleep 10 ; echo 'Retry snap installation'; done" - sudo microk8s status --wait-ready +- sudo echo "--service-node-port-range=30001-32767" >> /var/snap/microk8s/current/args/kube-apiserver - sudo microk8s refresh-certs /var/tmp - sudo sleep 30 - sudo microk8s stop @@ -75,9 +76,7 @@ runcmd: - sudo microk8s status --wait-ready - sudo microk8s add-node --token-ttl {{.JoinTokenTTLInSecs}} --token {{.JoinToken}} - sudo microk8s.kubectl delete svc kubernetes -- sudo microk8s.kubectl delete -f /var/snap/microk8s/current/args/cni-network/cni.yaml -- sudo sleep 5 -- sudo microk8s.kubectl apply -f /var/snap/microk8s/current/args/cni-network/cni.yaml +{{.IPinIPSection}} - sudo sh -c "for a in {{.Addons}} ; do echo 'Enabling ' \$a ; microk8s enable \$a ; sleep 10; microk8s status --wait-ready ; done" - sudo sleep 15 ` @@ -99,6 +98,7 @@ type ControlPlaneInput struct { HTTPProxy *string NoProxy *string Addons []string + IPinIP bool } // NewInitControlPlane returns the user data string to be used on a controlplane instance. @@ -135,6 +135,9 @@ func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) { proxyCommands := generateProxyCommands(input.HTTPSProxy, input.HTTPProxy, input.NoProxy) cloudinitStr = strings.Replace(cloudinitStr, "{{.ProxySection}}", proxyCommands, -1) + ipinipCommands := generateIPinIPCommands(input.IPinIP) + cloudinitStr = strings.Replace(cloudinitStr, "{{.IPinIPSection}}", ipinipCommands, -1) + addr := net.ParseIP(input.ControlPlaneEndpoint) if addr != nil { input.ControlPlaneEndpointType = "IP" diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index d275d30..4be0789 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -34,6 +34,8 @@ runcmd: - sudo echo Version {{.Version}} - sudo sh -c "while ! snap install microk8s --classic {{.Version}}; do sleep 10 ; echo 'Retry snap installation'; done" - sudo microk8s status --wait-ready +- sudo echo "--service-node-port-range=30001-32767" >> /var/snap/microk8s/current/args/kube-apiserver +{{.IPinIPSection}} - sudo microk8s stop - sudo sed -i 's/25000/{{.PortOfNodeToJoin}}/' /var/snap/microk8s/current/args/cluster-agent - sudo grep Address /var/snap/microk8s/current/var/kubernetes/backend/info.yaml > /var/tmp/port-update.yaml @@ -79,6 +81,7 @@ type ControlPlaneJoinInput struct { HTTPSProxy *string HTTPProxy *string NoProxy *string + IPinIP bool } // NewJoinControlPlane returns the user data string to be used on a new control plane instance. @@ -99,6 +102,9 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) ([]byte, error) { proxyCommands := generateProxyCommands(input.HTTPSProxy, input.HTTPProxy, input.NoProxy) cloudinitStr := strings.Replace(controlPlaneJoinCloudInit, "{{.ProxySection}}", proxyCommands, -1) + ipinipCommands := generateIPinIPCommands(input.IPinIP) + cloudinitStr = strings.Replace(cloudinitStr, "{{.IPinIPSection}}", ipinipCommands, -1) + userData, err := generate("JoinControlplane", cloudinitStr, input) if err != nil { return nil, errors.Wrapf(err, "failed to generate user data for machine joining control plane") diff --git a/controllers/cloudinit/utils.go b/controllers/cloudinit/utils.go index 4def837..07c870f 100644 --- a/controllers/cloudinit/utils.go +++ b/controllers/cloudinit/utils.go @@ -76,3 +76,20 @@ func generateProxyCommands(https *string, http *string, noproxy *string) string return proxyCommands.String() } + +func generateIPinIPCommands(ipinip bool) string { + if ipinip { + var commands bytes.Buffer + commands.WriteString("- sudo microk8s.kubectl delete ippools --all\n") + commands.WriteString("- sudo microk8s.kubectl delete -f /var/snap/microk8s/current/args/cni-network/cni.yaml\n") + commands.WriteString("- sudo sleep 5\n") + commands.WriteString("- sudo sed 's/CALICO_IPV4POOL_VXLAN/CALICO_IPV4POOL_IPIP/' -i /var/snap/microk8s/current/args/cni-network/cni.yaml\n") + commands.WriteString("- sudo sed 's/ \"vxlan\"/ \"bird\"/' -i /var/snap/microk8s/current/args/cni-network/cni.yaml\n") + commands.WriteString("- sudo sed -i '/- -felix-ready/p;s/-felix-ready/-bird-ready/' /var/snap/microk8s/current/args/cni-network/cni.yaml\n") + commands.WriteString("- sudo sed -i '/- -felix-live/p;s/-felix-live/-bird-live/' /var/snap/microk8s/current/args/cni-network/cni.yaml\n") + commands.WriteString("- sudo microk8s.kubectl apply -f /var/snap/microk8s/current/args/cni-network/cni.yaml\n") + return commands.String() + } else { + return "- sudo echo \"VXLAN is to be used\"" + } +} diff --git a/controllers/microk8sconfig_controller.go b/controllers/microk8sconfig_controller.go index 35867e6..44f4c76 100644 --- a/controllers/microk8sconfig_controller.go +++ b/controllers/microk8sconfig_controller.go @@ -85,10 +85,10 @@ type Scope struct { const ( defaultDqlitePort string = "19001" - remappedDqlitePort string = "2380" + remappedDqlitePort string = "2379" defaultClusterAgentPort string = "25000" - remappedClusterAgentPort string = "179" + remappedClusterAgentPort string = "30000" ) //+kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=microk8sconfigs,verbs=get;list;watch;create;update;patch;delete @@ -306,6 +306,7 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte controlPlaneInput.HTTPSProxy = microk8sConfig.Spec.InitConfiguration.HTTPSProxy controlPlaneInput.HTTPProxy = microk8sConfig.Spec.InitConfiguration.HTTPProxy controlPlaneInput.NoProxy = microk8sConfig.Spec.InitConfiguration.NoProxy + controlPlaneInput.IPinIP = microk8sConfig.Spec.InitConfiguration.IPinIP if microk8sConfig.Spec.InitConfiguration.JoinTokenTTLInSecs == 0 { // set by default to 10 years @@ -396,6 +397,8 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con controlPlaneInput.HTTPSProxy = microk8sConfig.Spec.InitConfiguration.HTTPSProxy controlPlaneInput.HTTPProxy = microk8sConfig.Spec.InitConfiguration.HTTPProxy controlPlaneInput.NoProxy = microk8sConfig.Spec.InitConfiguration.NoProxy + controlPlaneInput.IPinIP = microk8sConfig.Spec.InitConfiguration.IPinIP + if microk8sConfig.Spec.InitConfiguration.JoinTokenTTLInSecs == 0 { // set by default to 10 years controlPlaneInput.JoinTokenTTLInSecs = 315569260 diff --git a/examples/aws-capi-quickstart.yaml b/examples/aws-capi-quickstart.yaml index a082500..bd25ff4 100644 --- a/examples/aws-capi-quickstart.yaml +++ b/examples/aws-capi-quickstart.yaml @@ -30,6 +30,7 @@ metadata: spec: controlPlaneConfig: initConfiguration: + IPinIP: true addons: - dns - ingress @@ -67,7 +68,7 @@ spec: template: spec: clusterName: capi-aws - version: v1.23.0 + version: v1.23.0 bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-aws.yaml b/templates/cluster-template-aws.yaml index a1f2b02..f54aeda 100644 --- a/templates/cluster-template-aws.yaml +++ b/templates/cluster-template-aws.yaml @@ -32,6 +32,7 @@ spec: controlPlaneConfig: initConfiguration: joinTokenTTLInSecs: 9000 + IPinIP: true addons: - dns - ingress