Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ensure user modifications on managed resources are reverted on reconcile #35

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

Ensure user modifications on managed resources are reverted on reconcile #35

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
10fe3ae
Update kubebuilder, controller-gen
majst01 Feb 7, 2022
369041b
Ensure user modifications on managed resources are reverted on reconcile
Gerrit91 Feb 8, 2022
a93011a
Retrieve resource version.
Gerrit91 Feb 8, 2022
080f0a5
Different attempt.
Gerrit91 Feb 8, 2022
0d76157
Better logging.
Gerrit91 Feb 8, 2022
5cc6c44
Merge branch 'master' into update-resources
Gerrit91 Feb 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 121 additions & 98 deletions controllers/resources.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

const (
Expand Down Expand Up @@ -801,21 +800,18 @@ func (r *DurosReconciler) deployStorageClassSecret(ctx context.Context, log logr

storageClassSecret := corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: storageClassCredentialsRef, Namespace: namespace},
}

op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, &storageClassSecret, func() error {
storageClassSecret.Type = "kubernetes.io/lb-csi"
storageClassSecret.Data = map[string][]byte{
Type: "kubernetes.io/lb-csi",
Data: map[string][]byte{
"jwt": []byte(token),
}
},
}

return nil
})
err = r.createOrUpdate(ctx, &storageClassSecret)
if err != nil {
return err
}

log.Info("storageclasssecret", "name", storageClassCredentialsRef, "operation", op)
log.Info("storageclasssecret", "name", storageClassCredentialsRef)

return nil
}
Expand All @@ -837,88 +833,93 @@ func (r *DurosReconciler) deployCSI(ctx context.Context, projectID string, scs [
snapshotsSupported := false
switch gkv.Version {
case "v1":
csiDriver := &storage.CSIDriver{ObjectMeta: metav1.ObjectMeta{Name: provisioner}}
op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, csiDriver, func() error {
csiDriver.Spec = storage.CSIDriverSpec{
csiDriver := &storage.CSIDriver{
ObjectMeta: metav1.ObjectMeta{Name: provisioner},
Spec: storage.CSIDriverSpec{
AttachRequired: pointer.Bool(true),
PodInfoOnMount: pointer.Bool(true),
}
return nil
})
},
}
err := r.createOrUpdate(ctx, csiDriver)
if err != nil {
return err
}
log.Info("csidriver", "name", csiDriver.Name, "operation", op)
snapshotsSupported = true
case "v1beta1":
csiDriver := &storagev1beta1.CSIDriver{ObjectMeta: metav1.ObjectMeta{Name: provisioner}}
op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, csiDriver, func() error {
csiDriver.Spec = storagev1beta1.CSIDriverSpec{
csiDriver := &storagev1beta1.CSIDriver{
ObjectMeta: metav1.ObjectMeta{Name: provisioner},
Spec: storagev1beta1.CSIDriverSpec{
AttachRequired: pointer.Bool(true),
PodInfoOnMount: pointer.Bool(true),
}
return nil
})
},
}
err := r.createOrUpdate(ctx, csiDriver)
if err != nil {
return err
}
log.Info("csidriver", "name", csiDriver.Name, "operation", op)
default:
err := fmt.Errorf("unsupported csi driver version:%s", gkv.Version)
log.Error(err, "no csi plugin deployment possible")
return err
}

for i := range psps {
psp := psps[i]
obj := &policy.PodSecurityPolicy{ObjectMeta: metav1.ObjectMeta{Name: psp.Name}}
op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, obj, func() error {
obj.Spec = psp.Spec
return nil
})
psp := psps[i].DeepCopy()
obj := &policy.PodSecurityPolicy{
ObjectMeta: metav1.ObjectMeta{Name: psp.Name},
Spec: psp.Spec,
}
err := r.createOrUpdate(ctx, obj)
if err != nil {
return err
}
log.Info("psp", "name", psp.Name, "operation", op)
}

for i := range serviceAccounts {
sa := serviceAccounts[i]
obj := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: sa.Name, Namespace: sa.Namespace}}
op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, obj, func() error {
return nil
})
sa := serviceAccounts[i].DeepCopy()
obj := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: sa.Name, Namespace: sa.Namespace},
}
err := r.createOrUpdate(ctx, obj)
if err != nil {
return err
}
log.Info("serviceaccount", "name", sa.Name, "operation", op)
}

for i := range clusterRoles {
cr := clusterRoles[i]
obj := &rbac.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: cr.Name, Namespace: cr.Namespace}}
op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, obj, func() error {
obj.Rules = cr.Rules
return nil
})
cr := clusterRoles[i].DeepCopy()
obj := &rbac.ClusterRole{ObjectMeta: metav1.ObjectMeta{
Name: cr.Name, Namespace: cr.Namespace},
Rules: cr.Rules,
}
err := r.createOrUpdate(ctx, obj)
if err != nil {
return err
}
log.Info("clusterrole", "name", cr.Name, "operation", op)
}

for i := range clusterRoleBindings {
crb := clusterRoleBindings[i]
obj := &rbac.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: crb.Name, Namespace: crb.Namespace}}
op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, obj, func() error {
obj.Subjects = crb.Subjects
obj.RoleRef = crb.RoleRef
return nil
})
crb := clusterRoleBindings[i].DeepCopy()
obj := &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{Name: crb.Name, Namespace: crb.Namespace},
Subjects: crb.Subjects,
RoleRef: crb.RoleRef,
}
err := r.createOrUpdate(ctx, obj)
if err != nil {
return err
}
log.Info("clusterrolebindinding", "name", crb.Name, "operation", op)
}

controllerRoleLabels := map[string]string{"app": "lb-csi-plugin", "role": "controller"}
containers := []corev1.Container{
*csiPluginContainer.DeepCopy(),
*csiProvisionerContainer.DeepCopy(),
*csiAttacherContainer.DeepCopy(),
*csiResizerContainer.DeepCopy(),
}
if snapshotsSupported {
containers = append(containers, snapshotControllerContainer, csiSnapshotterContainer)
}

sts := &apps.StatefulSet{
Expand All @@ -927,21 +928,7 @@ func (r *DurosReconciler) deployCSI(ctx context.Context, projectID string, scs [
Namespace: namespace,
Labels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"},
},
}
op, err := controllerutil.CreateOrUpdate(ctx, r.Shoot, sts, func() error {

controllerRoleLabels := map[string]string{"app": "lb-csi-plugin", "role": "controller"}
containers := []corev1.Container{
csiPluginContainer,
csiProvisionerContainer,
csiAttacherContainer,
csiResizerContainer,
}
if snapshotsSupported {
containers = append(containers, snapshotControllerContainer, csiSnapshotterContainer)
}

sts.Spec = apps.StatefulSetSpec{
Spec: apps.StatefulSetSpec{
Selector: &metav1.LabelSelector{MatchLabels: controllerRoleLabels},
ServiceName: "lb-csi-ctrl-svc",
Replicas: pointer.Int32(1),
Expand All @@ -952,43 +939,38 @@ func (r *DurosReconciler) deployCSI(ctx context.Context, projectID string, scs [
ServiceAccountName: ctrlServiceAccount.Name,
PriorityClassName: "system-cluster-critical",
Volumes: []corev1.Volume{
socketDirVolume,
*socketDirVolume.DeepCopy(),
},
},
},
}
return nil
})
},
}

err = r.createOrUpdate(ctx, sts)
if err != nil {
return err
}
log.Info("statefulset", "name", sts.Name, "operation", op)

ds := &apps.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: csiNodeDaemonSet.Name, Namespace: csiNodeDaemonSet.Namespace}}
op, err = controllerutil.CreateOrUpdate(ctx, r.Shoot, ds, func() error {
ds.Spec = csiNodeDaemonSet.Spec
return nil
})
ds := csiNodeDaemonSet.DeepCopy()
err = r.createOrUpdate(ctx, ds)
if err != nil {
return err
}
log.Info("daemonset", "name", csiNodeDaemonSet.Name, "operation", op)

for i := range scs {
sc := scs[i]

annotations := map[string]string{
"storageclass.kubernetes.io/is-default-class": strconv.FormatBool(sc.Default),
metaltag.ClusterDescription: "DO NOT EDIT - This resource is managed by duros-controller. Any modifications are discarded and the resource is returned to the original state.",
}

obj := &storage.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: sc.Name}}
op, err = controllerutil.CreateOrUpdate(ctx, r.Shoot, obj, func() error {
obj.ObjectMeta.Annotations = annotations
obj.Provisioner = provisioner
obj.AllowVolumeExpansion = pointer.Bool(true)
obj.Parameters = map[string]string{
sc := scs[i].DeepCopy()

obj := &storage.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: sc.Name,
Annotations: map[string]string{
"storageclass.kubernetes.io/is-default-class": strconv.FormatBool(sc.Default),
metaltag.ClusterDescription: "DO NOT EDIT - This resource is managed by duros-controller. Any modifications are discarded and the resource is returned to the original state.",
},
},
Provisioner: provisioner,
AllowVolumeExpansion: pointer.Bool(true),
Parameters: map[string]string{
"mgmt-scheme": "grpcs",
"compression": "disabled",
"mgmt-endpoint": r.Endpoints.String(),
Expand All @@ -1004,17 +986,17 @@ func (r *DurosReconciler) deployCSI(ctx context.Context, projectID string, scs [
"csi.storage.k8s.io/provisioner-secret-namespace": namespace,
"csi.storage.k8s.io/controller-expand-secret-name": storageClassCredentialsRef,
"csi.storage.k8s.io/controller-expand-secret-namespace": namespace,
}
},
}

if sc.Compression {
obj.Parameters["compression"] = "enabled"
}
return nil
})
if sc.Compression {
obj.Parameters["compression"] = "enabled"
}

err = r.createOrUpdate(ctx, obj)
if err != nil {
return err
}
log.Info("storageclass", "name", sc.Name, "operation", op)
}

return nil
Expand Down Expand Up @@ -1046,3 +1028,44 @@ func (r *DurosReconciler) deleteResourceWithWait(ctx context.Context, log logr.L
return false, err
})
}

func (r *DurosReconciler) createOrUpdate(ctx context.Context, obj client.Object) error {
gvks, _, err := r.Scheme().ObjectKinds(obj)
if err != nil {
r.Log.Error(err, "error determining object kinds", "name", obj.GetName(), "namespace", obj.GetNamespace())
return err
}

gvk := gvks[0]
keysAndValues := []interface{}{"name", obj.GetName(), "namespace", obj.GetNamespace(), "group", gvk.Group, "version", gvk.Version, "kind", gvk.Kind}

err = r.Shoot.Create(ctx, obj)
if err == nil {
r.Log.Info("created resource", keysAndValues...)
return nil
}
if !apierrors.IsAlreadyExists(err) {
r.Log.Error(err, "error creating resource", keysAndValues...)
return err
}

part := &metav1.PartialObjectMetadata{}
part.SetGroupVersionKind(gvks[0])

err = r.Shoot.Get(ctx, client.ObjectKeyFromObject(obj), part)
if err != nil {
r.Log.Error(err, "error retrieving partial object metadata", keysAndValues)
return err
}

obj.SetResourceVersion(part.ResourceVersion)

err = r.Shoot.Update(ctx, obj)
if err == nil {
r.Log.Info("updated resource", keysAndValues...)
return nil
}

r.Log.Error(err, "error updating resource", keysAndValues...)
return err
}