-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Retry removing all finalizers (#289)
* Retry removing all finalizers * Update CHANGELOG * Break retry patch cycle * Persist changes to logger values in context * Rename var
- Loading branch information
Showing
8 changed files
with
102 additions
and
102 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,32 +23,28 @@ import ( | |
| awsclientgo "github.com/aws/aws-sdk-go/aws/client" | ||
| "github.com/aws/aws-sdk-go/service/iam/iamiface" | ||
| "github.com/giantswarm/microerror" | ||
| "github.com/go-logr/logr" | ||
| "github.com/pkg/errors" | ||
| corev1 "k8s.io/api/core/v1" | ||
| k8serrors "k8s.io/apimachinery/pkg/api/errors" | ||
| errutils "k8s.io/apimachinery/pkg/util/errors" | ||
|
|
||
| "k8s.io/apimachinery/pkg/types" | ||
| capa "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" | ||
| "sigs.k8s.io/cluster-api/util/patch" | ||
| ctrl "sigs.k8s.io/controller-runtime" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" | ||
| "sigs.k8s.io/controller-runtime/pkg/log" | ||
|
|
||
| "github.com/giantswarm/capa-iam-operator/pkg/awsclient" | ||
| "github.com/giantswarm/capa-iam-operator/pkg/iam" | ||
| "github.com/giantswarm/capa-iam-operator/pkg/key" | ||
| ) | ||
|
|
||
| const maxPatchRetries = 5 | ||
|
|
||
| // AWSMachineTemplateReconciler reconciles a AWSMachineTemplate object | ||
| type AWSMachineTemplateReconciler struct { | ||
| client.Client | ||
| EnableKiamRole bool | ||
| EnableRoute53Role bool | ||
| Log logr.Logger | ||
| AWSClient awsclient.AwsClientInterface | ||
| IAMClientFactory func(awsclientgo.ConfigProvider, string) iamiface.IAMAPI | ||
| } | ||
|
|
@@ -62,8 +58,7 @@ type AWSMachineTemplateReconciler struct { | |
| // For more details, check Reconcile and its Result here: | ||
| // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile | ||
| func (r *AWSMachineTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { | ||
| var err error | ||
| logger := r.Log.WithValues("namespace", req.Namespace, "awsMachineTemplate", req.Name) | ||
| logger := log.FromContext(ctx) | ||
|
|
||
| awsMachineTemplate := &capa.AWSMachineTemplate{} | ||
| if err := r.Get(ctx, req.NamespacedName, awsMachineTemplate); err != nil { | ||
|
|
@@ -96,6 +91,7 @@ func (r *AWSMachineTemplateReconciler) Reconcile(ctx context.Context, req ctrl.R | |
| } | ||
|
|
||
| logger = logger.WithValues("cluster", clusterName, "role", role) | ||
| ctx = log.IntoContext(ctx, logger) | ||
|
|
||
| if awsMachineTemplate.Spec.Template.Spec.IAMInstanceProfile == "" { | ||
| logger.Info("AWSMachineTemplate has empty .Spec.Template.Spec.IAMInstanceProfile, not creating IAM role") | ||
|
|
@@ -138,12 +134,14 @@ func (r *AWSMachineTemplateReconciler) Reconcile(ctx context.Context, req ctrl.R | |
| } | ||
|
|
||
| if awsMachineTemplate.DeletionTimestamp != nil { | ||
| return r.reconcileDelete(ctx, iamService, awsMachineTemplate, logger, clusterName, req.Namespace, role) | ||
| return r.reconcileDelete(ctx, iamService, awsMachineTemplate, clusterName, req.Namespace, role) | ||
| } | ||
| return r.reconcileNormal(ctx, iamService, awsMachineTemplate, logger, awsCluster, clusterName, role) | ||
| return r.reconcileNormal(ctx, iamService, awsMachineTemplate, awsCluster, clusterName, role) | ||
| } | ||
|
|
||
| func (r *AWSMachineTemplateReconciler) reconcileDelete(ctx context.Context, iamService *iam.IAMService, awsMachineTemplate *capa.AWSMachineTemplate, logger logr.Logger, clusterName, namespace, role string) (ctrl.Result, error) { | ||
| func (r *AWSMachineTemplateReconciler) reconcileDelete(ctx context.Context, iamService *iam.IAMService, awsMachineTemplate *capa.AWSMachineTemplate, clusterName, namespace, role string) (ctrl.Result, error) { | ||
| logger := log.FromContext(ctx) | ||
|
|
||
| roleUsed, err := isRoleUsedElsewhere(ctx, r.Client, awsMachineTemplate.Spec.Template.Spec.IAMInstanceProfile) | ||
| if err != nil { | ||
| return ctrl.Result{}, err | ||
|
|
@@ -169,14 +167,14 @@ func (r *AWSMachineTemplateReconciler) reconcileDelete(ctx context.Context, iamS | |
| logger.Error(err, "failed to get awsCluster") | ||
| return ctrl.Result{}, err | ||
| } | ||
| err = r.removeFinalizer(ctx, logger, awsCluster) | ||
| err = removeFinalizer(ctx, r.Client, awsCluster, iam.ControlPlaneRole) | ||
| if err != nil { | ||
| logger.Error(err, "Failed to remove finalizer from AWSCluster") | ||
| return ctrl.Result{}, err | ||
| } | ||
|
|
||
| // remove finalizer from AWSMachineTemplate | ||
| err = r.removeFinalizer(ctx, logger, awsMachineTemplate) | ||
| err = removeFinalizer(ctx, r.Client, awsMachineTemplate, iam.ControlPlaneRole) | ||
| if err != nil { | ||
| logger.Error(err, "Failed to remove finalizer from AWSMachineTemplate") | ||
| return ctrl.Result{}, err | ||
|
|
@@ -195,7 +193,7 @@ func (r *AWSMachineTemplateReconciler) reconcileDelete(ctx context.Context, iamS | |
| return ctrl.Result{}, errors.WithStack(client.IgnoreNotFound(err)) | ||
| } | ||
|
|
||
| err = r.removeFinalizer(ctx, logger, cm) | ||
| err = removeFinalizer(ctx, r.Client, cm, iam.ControlPlaneRole) | ||
| if err != nil { | ||
| logger.Error(err, "Failed to remove finalizer from ConfigMap") | ||
| return ctrl.Result{}, err | ||
|
|
@@ -204,7 +202,9 @@ func (r *AWSMachineTemplateReconciler) reconcileDelete(ctx context.Context, iamS | |
| return ctrl.Result{}, nil | ||
| } | ||
|
|
||
| func (r *AWSMachineTemplateReconciler) reconcileNormal(ctx context.Context, iamService *iam.IAMService, awsMachineTemplate *capa.AWSMachineTemplate, logger logr.Logger, awsCluster *capa.AWSCluster, clusterName, role string) (ctrl.Result, error) { | ||
| func (r *AWSMachineTemplateReconciler) reconcileNormal(ctx context.Context, iamService *iam.IAMService, awsMachineTemplate *capa.AWSMachineTemplate, awsCluster *capa.AWSCluster, clusterName, role string) (ctrl.Result, error) { | ||
| logger := log.FromContext(ctx) | ||
|
|
||
| // add finalizer to AWSMachineTemplate | ||
| if !controllerutil.ContainsFinalizer(awsMachineTemplate, key.FinalizerName(iam.ControlPlaneRole)) { | ||
| patchHelper, err := patch.NewHelper(awsMachineTemplate, r.Client) | ||
|
|
@@ -283,44 +283,3 @@ func (r *AWSMachineTemplateReconciler) SetupWithManager(mgr ctrl.Manager) error | |
| For(&capa.AWSMachineTemplate{}). | ||
| Complete(r) | ||
| } | ||
|
|
||
| func (r *AWSMachineTemplateReconciler) removeFinalizer(ctx context.Context, logger logr.Logger, object client.Object) error { | ||
| if !controllerutil.ContainsFinalizer(object, key.FinalizerName(iam.ControlPlaneRole)) { | ||
| logger.Info("finalizer already removed") | ||
| return nil | ||
| } | ||
|
|
||
| for i := 1; i <= maxPatchRetries; i++ { | ||
| patchHelper, err := patch.NewHelper(object, r.Client) | ||
| if err != nil { | ||
| logger.Error(err, "failed to create patch helper") | ||
| return errors.WithStack(err) | ||
| } | ||
| controllerutil.RemoveFinalizer(object, key.FinalizerName(iam.ControlPlaneRole)) | ||
| err = patchHelper.Patch(ctx, object) | ||
|
|
||
| // If another controller has removed its finalizer while we're | ||
| // reconciling this will fail with "Forbidden: no new finalizers can be | ||
| // added if the object is being deleted". The actual response code is | ||
| // 422 Unprocessable entity, which maps to StatusReasonInvalid in the | ||
| // k8serrors package. We have to get the cluster again with the now | ||
| // removed finalizer(s) and try again. | ||
| invalidErr := errutils.FilterOut(err, func(err error) bool { | ||
| return !k8serrors.IsInvalid(err) | ||
| }) | ||
|
|
||
| if invalidErr != nil && i < maxPatchRetries { | ||
| logger.Info(fmt.Sprintf("patching object failed, trying again: %s", err.Error())) | ||
| if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { | ||
| return microerror.Mask(err) | ||
| } | ||
| continue | ||
| } | ||
| if err != nil { | ||
| logger.Error(err, "failed to remove finalizers") | ||
| return microerror.Mask(err) | ||
| } | ||
| } | ||
| logger.Info("successfully removed finalizer") | ||
| return nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.