Merge pull request #7 from ucloud/fixed/check_sentinel

Fixed/check sentinel
ucloud · Dec 10, 2019 · f4cf6ef · f4cf6ef
2 parents 6800845 + 9d3706a
commit f4cf6ef
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 16 deletions.
diff --git a/pkg/client/redis/client.go b/pkg/client/redis/client.go
@@ -48,7 +48,7 @@ const (
 	masterName              = "mymaster"
 
 	defaultDownAfterMilliseconds = "5000"
-	defaultFailovertimeout       = "6000"
+	defaultFailovertimeout       = "3000"
 	defaultParallelSyncs         = "2"
 )
 

diff --git a/pkg/controller/rediscluster/checker.go b/pkg/controller/rediscluster/checker.go
@@ -100,28 +100,28 @@ func (r *RedisClusterHandler) CheckAndHeal(meta *clustercache.Meta) error {
 			}
 		}
 	}
-	// TODO: check sentinel's number
-	//for _, sip := range sentinels {
-	//	if err := r.rcChecker.CheckSentinelNumberInMemory(sip, meta.Obj, meta.Auth); err != nil {
-	//		r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).Info(err.Error())
-	//		if err := r.rcHealer.RestoreSentinel(sip, meta.Auth); err != nil {
-	//			return err
-	//		}
-	//		return needRequeueErr
-	//	}
-	//}
 	for _, sip := range sentinels {
 		if err := r.rcChecker.CheckSentinelSlavesNumberInMemory(sip, meta.Obj, meta.Auth); err != nil {
-			r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).Info(err.Error())
+			r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).
+				Info("restoring sentinel ...", "sentinel", sip, "reason", err.Error())
 			if err := r.rcHealer.RestoreSentinel(sip, meta.Auth); err != nil {
 				return err
 			}
-			if err := r.waitRestoreSentinelOK(sip, meta.Obj, meta.Auth); err != nil {
+			if err := r.waitRestoreSentinelSlavesOK(sip, meta.Obj, meta.Auth); err != nil {
 				r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).Info(err.Error())
 				return err
 			}
 		}
 	}
+	for _, sip := range sentinels {
+		if err := r.rcChecker.CheckSentinelNumberInMemory(sip, meta.Obj, meta.Auth); err != nil {
+			r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).
+				Info("restoring sentinel ...", "sentinel", sip, "reason", err.Error())
+			if err := r.rcHealer.RestoreSentinel(sip, meta.Auth); err != nil {
+				return err
+			}
+		}
+	}
 
 	if err = r.setSentinelConfig(meta, sentinels); err != nil {
 		return err
@@ -161,13 +161,13 @@ func (r *RedisClusterHandler) setSentinelConfig(meta *clustercache.Meta, sentine
 	return nil
 }
 
-func (r *RedisClusterHandler) waitRestoreSentinelOK(sentinel string, rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) error {
+func (r *RedisClusterHandler) waitRestoreSentinelSlavesOK(sentinel string, rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) error {
 	timer := time.NewTimer(timeOut)
 	defer timer.Stop()
 	for {
 		select {
 		case <-timer.C:
-			return fmt.Errorf("wait for resetore sentinel timeout")
+			return fmt.Errorf("wait for resetore sentinel slave timeout")
 		default:
 			if err := r.rcChecker.CheckSentinelSlavesNumberInMemory(sentinel, rc, auth); err != nil {
 				r.logger.WithValues("namespace", rc.Namespace, "name", rc.Name).Info(err.Error())

diff --git a/pkg/controller/rediscluster/controller.go b/pkg/controller/rediscluster/controller.go
@@ -105,7 +105,8 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 			log.WithValues("namespace", e.MetaNew.GetNamespace(), "name", e.MetaNew.GetName()).V(5).Info("Call UpdateFunc")
 			// Ignore updates to CR status in which case metadata.Generation does not change
 			if e.MetaOld.GetGeneration() != e.MetaNew.GetGeneration() {
-				log.WithValues("namespace", e.MetaNew.GetNamespace(), "name", e.MetaNew.GetName()).Info("Generation change return true")
+				log.WithValues("namespace", e.MetaNew.GetNamespace(), "name", e.MetaNew.GetName()).
+					Info("Generation change return true", "old", e.ObjectOld, "new", e.ObjectNew)
 				return true
 			}
 			return false