From 181fe2a19b358906acef184e54467110fd1f6883 Mon Sep 17 00:00:00 2001
From: fishu <pole.fishu@gmail.com>
Date: Fri, 6 Dec 2019 10:22:28 +0800
Subject: [PATCH 1/2] defaultFailovertimeout=3000

---
 pkg/client/redis/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/client/redis/client.go b/pkg/client/redis/client.go
index 7de8c36..f594826 100644
--- a/pkg/client/redis/client.go
+++ b/pkg/client/redis/client.go
@@ -48,7 +48,7 @@ const (
 	masterName              = "mymaster"
 
 	defaultDownAfterMilliseconds = "5000"
-	defaultFailovertimeout       = "6000"
+	defaultFailovertimeout       = "3000"
 	defaultParallelSyncs         = "2"
 )
 

From 9d3706ac6ae0f6b56e6201d0ee1c3371ad64f0ae Mon Sep 17 00:00:00 2001
From: fishu <pole.fishu@gmail.com>
Date: Fri, 6 Dec 2019 17:52:20 +0800
Subject: [PATCH 2/2] Check Sentinel Number In Memory

---
 pkg/controller/rediscluster/checker.go    | 28 +++++++++++------------
 pkg/controller/rediscluster/controller.go |  3 ++-
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/pkg/controller/rediscluster/checker.go b/pkg/controller/rediscluster/checker.go
index a31c2c1..cd9e4f9 100644
--- a/pkg/controller/rediscluster/checker.go
+++ b/pkg/controller/rediscluster/checker.go
@@ -100,28 +100,28 @@ func (r *RedisClusterHandler) CheckAndHeal(meta *clustercache.Meta) error {
 			}
 		}
 	}
-	// TODO: check sentinel's number
-	//for _, sip := range sentinels {
-	//	if err := r.rcChecker.CheckSentinelNumberInMemory(sip, meta.Obj, meta.Auth); err != nil {
-	//		r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).Info(err.Error())
-	//		if err := r.rcHealer.RestoreSentinel(sip, meta.Auth); err != nil {
-	//			return err
-	//		}
-	//		return needRequeueErr
-	//	}
-	//}
 	for _, sip := range sentinels {
 		if err := r.rcChecker.CheckSentinelSlavesNumberInMemory(sip, meta.Obj, meta.Auth); err != nil {
-			r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).Info(err.Error())
+			r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).
+				Info("restoring sentinel ...", "sentinel", sip, "reason", err.Error())
 			if err := r.rcHealer.RestoreSentinel(sip, meta.Auth); err != nil {
 				return err
 			}
-			if err := r.waitRestoreSentinelOK(sip, meta.Obj, meta.Auth); err != nil {
+			if err := r.waitRestoreSentinelSlavesOK(sip, meta.Obj, meta.Auth); err != nil {
 				r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).Info(err.Error())
 				return err
 			}
 		}
 	}
+	for _, sip := range sentinels {
+		if err := r.rcChecker.CheckSentinelNumberInMemory(sip, meta.Obj, meta.Auth); err != nil {
+			r.logger.WithValues("namespace", meta.Obj.Namespace, "name", meta.Obj.Name).
+				Info("restoring sentinel ...", "sentinel", sip, "reason", err.Error())
+			if err := r.rcHealer.RestoreSentinel(sip, meta.Auth); err != nil {
+				return err
+			}
+		}
+	}
 
 	if err = r.setSentinelConfig(meta, sentinels); err != nil {
 		return err
@@ -161,13 +161,13 @@ func (r *RedisClusterHandler) setSentinelConfig(meta *clustercache.Meta, sentine
 	return nil
 }
 
-func (r *RedisClusterHandler) waitRestoreSentinelOK(sentinel string, rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) error {
+func (r *RedisClusterHandler) waitRestoreSentinelSlavesOK(sentinel string, rc *redisv1beta1.RedisCluster, auth *util.AuthConfig) error {
 	timer := time.NewTimer(timeOut)
 	defer timer.Stop()
 	for {
 		select {
 		case <-timer.C:
-			return fmt.Errorf("wait for resetore sentinel timeout")
+			return fmt.Errorf("wait for resetore sentinel slave timeout")
 		default:
 			if err := r.rcChecker.CheckSentinelSlavesNumberInMemory(sentinel, rc, auth); err != nil {
 				r.logger.WithValues("namespace", rc.Namespace, "name", rc.Name).Info(err.Error())
diff --git a/pkg/controller/rediscluster/controller.go b/pkg/controller/rediscluster/controller.go
index c57ee01..9ac7dfb 100644
--- a/pkg/controller/rediscluster/controller.go
+++ b/pkg/controller/rediscluster/controller.go
@@ -105,7 +105,8 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 			log.WithValues("namespace", e.MetaNew.GetNamespace(), "name", e.MetaNew.GetName()).V(5).Info("Call UpdateFunc")
 			// Ignore updates to CR status in which case metadata.Generation does not change
 			if e.MetaOld.GetGeneration() != e.MetaNew.GetGeneration() {
-				log.WithValues("namespace", e.MetaNew.GetNamespace(), "name", e.MetaNew.GetName()).Info("Generation change return true")
+				log.WithValues("namespace", e.MetaNew.GetNamespace(), "name", e.MetaNew.GetName()).
+					Info("Generation change return true", "old", e.ObjectOld, "new", e.ObjectNew)
 				return true
 			}
 			return false