From f7948c8747360ac8b7dbcd32c9fa1bec7183a140 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe Date: Tue, 25 Oct 2022 12:49:49 +0530 Subject: [PATCH] [WIP] PCP-22: Boostrap and CAPI changes Minor changes PCP-22: CAPI changes Code cleanup Added takeOverCluster in ClusterSpec Added takeOverCluster in ClusterSpec Spectro generated changes Update cluster_types.go Update kubeadmconfig_controller.go Update controller.go Update kubeadm_control_plane_webhook.go Added takeOverCluster in Annotations Update kubeadm_control_plane_webhook.go --- api/v1beta1/common_types.go | 3 ++ .../config/default/manager_image_patch.yaml | 2 +- .../controllers/kubeadmconfig_controller.go | 8 +++- config/default/manager_image_patch.yaml | 2 +- .../v1beta1/kubeadm_control_plane_webhook.go | 47 +++++++++---------- .../config/default/manager_image_patch.yaml | 2 +- .../internal/controllers/controller.go | 2 +- .../cluster/cluster_controller_phases.go | 5 +- spectro/generated/bootstrap-base.yaml | 2 +- spectro/generated/bootstrap-global.yaml | 2 +- spectro/generated/controlplane-base.yaml | 2 +- spectro/generated/controlplane-global.yaml | 2 +- spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 2 +- util/annotations/helpers.go | 5 ++ util/secret/certificates.go | 3 ++ 16 files changed, 54 insertions(+), 37 deletions(-) diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 113c4e95f25a..de2fc4115e53 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -136,6 +136,9 @@ const ( // instead of being a source of truth for eventual consistency. // This annotation can be used to inform MachinePool status during in-progress scaling scenarios. ReplicasManagedByAnnotation = "cluster.x-k8s.io/replicas-managed-by" + + // TakeOverCluster is the label used to mark the nodes that run on takeover-cluster instances. + TakeOverCluster = "cluster.x-k8s.io/takeover-cluster" ) const ( diff --git a/bootstrap/kubeadm/config/default/manager_image_patch.yaml b/bootstrap/kubeadm/config/default/manager_image_patch.yaml index 810a74d02471..1a8b1833dd06 100644 --- a/bootstrap/kubeadm/config/default/manager_image_patch.yaml +++ b/bootstrap/kubeadm/config/default/manager_image_patch.yaml @@ -7,5 +7,5 @@ spec: template: spec: containers: - - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:20220805 + - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:dev name: manager diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index d46246b453c6..2e0f364a9cdd 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -277,8 +277,12 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } + // TODO: PCP-22 check (annotation to skip handleClusterNotInitialized and go for join ) + // how to make this condition true for new cluster as kubeadm cluster is already initialized // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. - if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { + log.Info("TESTING... skip handleClusterNotInitialized and push cluster for join") + + if !annotations.IsTakeOverCluster(cluster.GetObjectMeta()) && !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { return r.handleClusterNotInitialized(ctx, scope) } @@ -296,10 +300,12 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques // it's a control plane join if configOwner.IsControlPlaneMachine() { + log.Info("TESTING.... joinControlplane") return r.joinControlplane(ctx, scope) } // It's a worker join + log.Info("TESTING.... It's a worker join") return r.joinWorker(ctx, scope) } diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 11b1230a111c..d68596266d58 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -7,5 +7,5 @@ spec: template: spec: containers: - - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:20220805 + - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:dev name: manager diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go index e8f4a68367cb..7ab75392716e 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go @@ -115,7 +115,6 @@ const ( initConfiguration = "initConfiguration" joinConfiguration = "joinConfiguration" nodeRegistration = "nodeRegistration" - skipPhases = "skipPhases" patches = "patches" directory = "directory" preKubeadmCommands = "preKubeadmCommands" @@ -127,7 +126,6 @@ const ( scheduler = "scheduler" ntp = "ntp" ignition = "ignition" - diskSetup = "diskSetup" ) const minimumCertificatesExpiryDays = 7 @@ -138,29 +136,28 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error { // For example, {"spec", "*"} will allow any path under "spec" to change. allowedPaths := [][]string{ {"metadata", "*"}, - {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageRepository"}, - {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageTag"}, - {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "extraArgs", "*"}, - {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageRepository"}, - {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageTag"}, - {spec, kubeadmConfigSpec, clusterConfiguration, "imageRepository"}, - {spec, kubeadmConfigSpec, clusterConfiguration, apiServer, "*"}, - {spec, kubeadmConfigSpec, clusterConfiguration, controllerManager, "*"}, - {spec, kubeadmConfigSpec, clusterConfiguration, scheduler, "*"}, - {spec, kubeadmConfigSpec, initConfiguration, nodeRegistration, "*"}, - {spec, kubeadmConfigSpec, initConfiguration, patches, directory}, - {spec, kubeadmConfigSpec, initConfiguration, skipPhases}, - {spec, kubeadmConfigSpec, joinConfiguration, nodeRegistration, "*"}, - {spec, kubeadmConfigSpec, joinConfiguration, patches, directory}, - {spec, kubeadmConfigSpec, joinConfiguration, skipPhases}, - {spec, kubeadmConfigSpec, preKubeadmCommands}, - {spec, kubeadmConfigSpec, postKubeadmCommands}, - {spec, kubeadmConfigSpec, files}, - {spec, kubeadmConfigSpec, "verbosity"}, - {spec, kubeadmConfigSpec, users}, - {spec, kubeadmConfigSpec, ntp, "*"}, - {spec, kubeadmConfigSpec, ignition, "*"}, - {spec, kubeadmConfigSpec, diskSetup, "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageRepository"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "imageTag"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "etcd", "local", "extraArgs", "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageRepository"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageTag"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, "imageRepository"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, apiServer, "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, controllerManager, "*"}, + // {spec, kubeadmConfigSpec, clusterConfiguration, scheduler, "*"}, + // {spec, kubeadmConfigSpec, initConfiguration, nodeRegistration, "*"}, + // {spec, kubeadmConfigSpec, initConfiguration, patches, directory}, + // {spec, kubeadmConfigSpec, joinConfiguration, nodeRegistration, "*"}, + // {spec, kubeadmConfigSpec, joinConfiguration, patches, directory}, + // {spec, kubeadmConfigSpec, preKubeadmCommands}, + // {spec, kubeadmConfigSpec, postKubeadmCommands}, + // {spec, kubeadmConfigSpec, files}, + // {spec, kubeadmConfigSpec, "verbosity"}, + // {spec, kubeadmConfigSpec, users}, + // {spec, kubeadmConfigSpec, ntp, "*"}, + // {spec, kubeadmConfigSpec, ignition, "*"}, + // allow all fields to be modified + {spec, kubeadmConfigSpec, "*"}, {spec, "machineTemplate", "metadata", "*"}, {spec, "machineTemplate", "infrastructureRef", "apiVersion"}, {spec, "machineTemplate", "infrastructureRef", "name"}, diff --git a/controlplane/kubeadm/config/default/manager_image_patch.yaml b/controlplane/kubeadm/config/default/manager_image_patch.yaml index 5c278272e3dd..7fb394e94feb 100644 --- a/controlplane/kubeadm/config/default/manager_image_patch.yaml +++ b/controlplane/kubeadm/config/default/manager_image_patch.yaml @@ -7,5 +7,5 @@ spec: template: spec: containers: - - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:20220805 + - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:dev name: manager diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 05d69b52b00b..1177bdd11e43 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -568,7 +568,7 @@ func (r *KubeadmControlPlaneReconciler) reconcileEtcdMembers(ctx context.Context log := ctrl.LoggerFrom(ctx) // If etcd is not managed by KCP this is a no-op. - if !controlPlane.IsEtcdManaged() { + if annotations.IsTakeOverCluster(controlPlane.Cluster.GetObjectMeta()) || !controlPlane.IsEtcdManaged() { return ctrl.Result{}, nil } diff --git a/internal/controllers/cluster/cluster_controller_phases.go b/internal/controllers/cluster/cluster_controller_phases.go index 741a423cb244..1fb739d6e6d0 100644 --- a/internal/controllers/cluster/cluster_controller_phases.go +++ b/internal/controllers/cluster/cluster_controller_phases.go @@ -245,7 +245,10 @@ func (r *Reconciler) reconcileControlPlane(ctx context.Context, cluster *cluster if err != nil { return ctrl.Result{}, err } - if initialized { + + // TODO: PCP-22 set controlPlaneInitializedCondition to true for takeOver cluster + // as CP are already initialized in existing cluster + if annotations.IsTakeOverCluster(cluster.GetObjectMeta()) || initialized { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) } else { conditions.MarkFalse(cluster, clusterv1.ControlPlaneInitializedCondition, clusterv1.WaitingForControlPlaneProviderInitializedReason, clusterv1.ConditionSeverityInfo, "Waiting for control plane provider to indicate the control plane has been initialized") diff --git a/spectro/generated/bootstrap-base.yaml b/spectro/generated/bootstrap-base.yaml index 558d42de5217..efe4f4239c69 100644 --- a/spectro/generated/bootstrap-base.yaml +++ b/spectro/generated/bootstrap-base.yaml @@ -26,7 +26,7 @@ spec: - --bootstrap-token-ttl=${KUBEADM_BOOTSTRAP_TOKEN_TTL:=15m} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/bootstrap-global.yaml b/spectro/generated/bootstrap-global.yaml index 90de902d0bb1..22afe88f7a10 100644 --- a/spectro/generated/bootstrap-global.yaml +++ b/spectro/generated/bootstrap-global.yaml @@ -6241,7 +6241,7 @@ spec: - --bootstrap-token-ttl=${KUBEADM_BOOTSTRAP_TOKEN_TTL:=15m} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-bootstrap-controller-amd64:dev imagePullPolicy: Always livenessProbe: httpGet: diff --git a/spectro/generated/controlplane-base.yaml b/spectro/generated/controlplane-base.yaml index 5cd9f6869892..ae2c22f687e0 100644 --- a/spectro/generated/controlplane-base.yaml +++ b/spectro/generated/controlplane-base.yaml @@ -25,7 +25,7 @@ spec: - --feature-gates=ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/controlplane-global.yaml b/spectro/generated/controlplane-global.yaml index 6adf5b04b649..b0d272291e92 100644 --- a/spectro/generated/controlplane-global.yaml +++ b/spectro/generated/controlplane-global.yaml @@ -6209,7 +6209,7 @@ spec: - --feature-gates=ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/kubeadm-control-plane-controller-amd64:dev imagePullPolicy: Always livenessProbe: httpGet: diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index 3096a97a49fd..0e468767b44b 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -25,7 +25,7 @@ spec: - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},ClusterResourceSet=${EXP_CLUSTER_RESOURCE_SET:=false},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index 28ab7d47c1fb..1d40fe5793aa 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -9646,7 +9646,7 @@ spec: - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},ClusterResourceSet=${EXP_CLUSTER_RESOURCE_SET:=false},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false} command: - /manager - image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:20220805 + image: gcr.io/spectro-dev-public/release/cluster-api-controller-amd64:dev imagePullPolicy: Always livenessProbe: httpGet: diff --git a/util/annotations/helpers.go b/util/annotations/helpers.go index 0ec9ef9388ac..072dbd119e1c 100644 --- a/util/annotations/helpers.go +++ b/util/annotations/helpers.go @@ -38,6 +38,11 @@ func IsExternallyManaged(o metav1.Object) bool { return hasAnnotation(o, clusterv1.ManagedByAnnotation) } +// IsTakeOverCluster returns true if the object has the `managed-by` annotation. +func IsTakeOverCluster(o metav1.Object) bool { + return hasAnnotation(o, clusterv1.TakeOverCluster) +} + // HasPaused returns true if the object has the `paused` annotation. func HasPaused(o metav1.Object) bool { return hasAnnotation(o, clusterv1.PausedAnnotation) diff --git a/util/secret/certificates.go b/util/secret/certificates.go index e3aadfbc7a37..c017f5ea6f9c 100644 --- a/util/secret/certificates.go +++ b/util/secret/certificates.go @@ -24,6 +24,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" + "fmt" "math/big" "path/filepath" "strings" @@ -238,6 +239,7 @@ func (c Certificates) EnsureAllExist() error { func (c Certificates) Generate() error { for _, certificate := range c { if certificate.KeyPair == nil { + fmt.Println("TESTING.... Generate new certificates") err := certificate.Generate() if err != nil { return err @@ -268,6 +270,7 @@ func (c Certificates) LookupOrGenerate(ctx context.Context, ctrlclient client.Cl return err } + fmt.Println("TESTING.... Generate new certificates if that don't exist") // Generate the certificates that don't exist if err := c.Generate(); err != nil { return err