Skip reconciling etcdbackup CR if a newer one is pending (#486)

* skips reconciling an etcdbackup CR if there is a newer one already pending

* Update service/controller/resource/etcdbackup/backup.go

* skips reconciling an etcdbackup CR if there is a newer one already pending

* skips reconciling an etcdbackup CR if there is a newer one already pending

* skips reconciling an etcdbackup CR if there is a newer one already pending

* skips reconciling an etcdbackup CR if there is a newer one already pending

* skips reconciling an etcdbackup CR if there is a newer one already pending

* skips reconciling an etcdbackup CR if there is a newer one already pending

* skips reconciling an etcdbackup CR if there is a newer one already pending

* changelog

* add cleanup for skipped crs

* Update service/controller/resource/etcdbackup/create_empty.go

Co-authored-by: Daniel Simionato <[email protected]>

---------

Co-authored-by: Daniel Simionato <[email protected]>

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Skip reconciling etcdbackup CR's if a newer one is pending.
 - Configure `gsoci.azurecr.io` as the default container image registry.
 
 ## [4.4.6] - 2023-12-11

service/controller/resource/etcdbackup/cleanup.go

@@ -10,15 +10,22 @@ import (
 )
 
 func (r *Resource) cleanup(ctx context.Context, etcdBackup v1alpha1.ETCDBackup) error {
-	r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("Looking for completed ETCDBackup resources older than %d seconds", crKeepTimeoutSeconds))
-	diff := time.Now().UTC().Sub(etcdBackup.Status.FinishedTimestamp.Time).Seconds()
-	if diff > crKeepTimeoutSeconds {
+	return r.cleanupByTimestamp(ctx, etcdBackup.Status.FinishedTimestamp.Time, crKeepTimeoutSeconds, etcdBackup)
+}
+
+func (r *Resource) cleanupSkippedCR(ctx context.Context, etcdBackup v1alpha1.ETCDBackup) error {
+	return r.cleanupByTimestamp(ctx, etcdBackup.CreationTimestamp.Time, crSkippedKeepTimeoutSeconds, etcdBackup)
+}
+
+func (r *Resource) cleanupByTimestamp(ctx context.Context, timestamp time.Time, timeoutSeconds int64, etcdBackup v1alpha1.ETCDBackup) error {
+	r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("Looking for ETCDBackup resources older than %d seconds", timeoutSeconds))
+	diff := time.Now().UTC().Sub(timestamp).Seconds()
+	if diff > float64(timeoutSeconds) {
 		r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("Deleting (state %s, %f seconds old)", etcdBackup.Status.Status, diff))
 		err := r.k8sClient.CtrlClient().Delete(ctx, &etcdBackup)
 		if err != nil {
 			return microerror.Mask(err)
 		}
-
 	} else {
 		r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("Not due for deletion (state %s, %f seconds old)", etcdBackup.Status.Status, diff))
 	}

service/controller/resource/etcdbackup/create.go

@@ -21,6 +21,7 @@ const (
 	backupStateRunningV3BackupCompleted = "V3BackupCompleted"
 	backupStateCompleted                = "Completed"
 	backupStateFailed                   = "Failed"
+	backupStateSkipped                  = "Skipped"
 
 	// Instance States.
 	instanceBackupStatePending   = "Pending"
@@ -33,7 +34,8 @@ const (
 	maxBackupAttempts = int8(3)
 
 	// Default values.
-	crKeepTimeoutSeconds = 7 * 24 * 60 * 60
+	crKeepTimeoutSeconds        = 7 * 24 * 60 * 60
+	crSkippedKeepTimeoutSeconds = 60 * 60
 )
 
 // configureStateMachine configures and returns state machine that is driven by
@@ -48,6 +50,7 @@ func (r *Resource) configureStateMachine() {
 		backupStateRunningV3BackupCompleted: r.backupRunningV3BackupCompletedTransition,
 		backupStateCompleted:                r.backupCompletedTransition,
 		backupStateFailed:                   r.backupFailedTransition,
+		backupStateSkipped:                  r.backupSkippedTransition,
 	}
 
 	r.stateMachine = sm

service/controller/resource/etcdbackup/create_empty.go

@@ -2,13 +2,41 @@ package etcdbackup
 
 import (
 	"context"
+	"fmt"
+
+	backupv1alpha1 "github.com/giantswarm/apiextensions-backup/api/v1alpha1"
+	"github.com/giantswarm/microerror"
 
 	"github.com/giantswarm/etcd-backup-operator/v4/service/controller/resource/etcdbackup/internal/state"
 )
 
 // Sets the initial state.
 func (r *Resource) backupEmptyTransition(ctx context.Context, obj interface{}, currentState state.State) (state.State, error) {
+	var err error
 	r.logger.LogCtx(ctx, "level", "debug", "message", "no current state present")
 
+	cr, ok := obj.(*backupv1alpha1.ETCDBackup)
+	if !ok {
+		return "", microerror.Mask(fmt.Errorf("expected v1alpha1.EtcdBackup, got %T", obj))
+	}
+
+	backups := backupv1alpha1.ETCDBackupList{}
+	err = r.k8sClient.CtrlClient().List(ctx, &backups)
+	if err != nil {
+		return "", microerror.Mask(err)
+	}
+
+	var latestBackup backupv1alpha1.ETCDBackup
+	for _, backup := range backups.Items {
+		if latestBackup.CreationTimestamp.Time.Before(backup.CreationTimestamp.Time) {
+			latestBackup = backup
+		}
+	}
+
+	if cr.Name != latestBackup.Name {
+		r.logger.LogCtx(ctx, "level", "debug", "message", "backup object is not the latest one, skipping...")
+		return backupStateSkipped, nil
+	}
+
 	return backupStatePending, nil
 }

service/controller/resource/etcdbackup/create_skipped.go

@@ -0,0 +1,25 @@
+package etcdbackup
+
+import (
+	"context"
+
+	"github.com/giantswarm/microerror"
+
+	"github.com/giantswarm/etcd-backup-operator/v4/service/controller/key"
+	"github.com/giantswarm/etcd-backup-operator/v4/service/controller/resource/etcdbackup/internal/state"
+)
+
+// Deletes the ETCDBackup if it's older than the threshold.
+func (r *Resource) backupSkippedTransition(ctx context.Context, obj interface{}, currentState state.State) (state.State, error) {
+	customObject, err := key.ToCustomObject(obj)
+	if err != nil {
+		return "", microerror.Mask(err)
+	}
+
+	err = r.cleanupSkippedCR(ctx, customObject)
+	if err != nil {
+		return "", microerror.Mask(err)
+	}
+
+	return backupStateSkipped, nil
+}