Merge pull request #187 from 3scale-ops/fix-taint-too-fast

Harden requirements for taint
3scale-ops · May 22, 2023 · 3d88284 · 3d88284
2 parents 861928f + ff1aff6
commit 3d88284
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 8 deletions.
diff --git a/Makefile b/Makefile
@@ -341,6 +341,7 @@ kind-refresh-image: manifests kind docker-build ## Reloads the image into the K8
 	$(MAKE) kind-load-image
 	kubectl -n marin3r-system delete pod -l control-plane=controller-manager
 	kubectl -n marin3r-system delete pod -l control-plane=controller-webhook
+	kubectl -n default delete pod -l app.kubernetes.io/component=discovery-service
 
 kind-delete: ## Deletes the kind cluster and the registry
 kind-delete: kind

diff --git a/controllers/marin3r/envoyconfigrevision_controller.go b/controllers/marin3r/envoyconfigrevision_controller.go
@@ -157,7 +157,7 @@ func (r *EnvoyConfigRevisionReconciler) Reconcile(ctx context.Context, req ctrl.
 	}
 
 	if meta.IsStatusConditionTrue(ecr.Status.Conditions, marin3rv1alpha1.RevisionPublishedCondition) {
-		return ctrl.Result{Requeue: true, RequeueAfter: 60 * time.Second}, nil
+		return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
 	}
 
 	return ctrl.Result{}, nil

diff --git a/pkg/discoveryservice/xdss/stats/stats.go b/pkg/discoveryservice/xdss/stats/stats.go
@@ -108,7 +108,7 @@ func (s *Stats) GetPercentageFailing(nodeID, rType, version string) float64 {
 	failing := 0
 	pods := s.GetSubscribedPods(nodeID, rType)
 	for pod := range pods {
-		if v, err := s.GetCounter(nodeID, rType, version, pod, "nack_counter"); err == nil && v > 0 {
+		if v, err := s.GetCounter(nodeID, rType, version, pod, "nack_counter"); err == nil && v >= 5 {
 			failing++
 		}
 	}

diff --git a/pkg/discoveryservice/xdss/stats/stats_test.go b/pkg/discoveryservice/xdss/stats/stats_test.go
@@ -343,7 +343,7 @@ func TestStats_GetPercentageFailing(t *testing.T) {
 				"node:endpoint:*:pod-bbbb:request_counter": {Object: int64(5), Expiration: int64(defaultExpiration)},
 				"node:endpoint:*:pod-cccc:request_counter": {Object: int64(1), Expiration: int64(defaultExpiration)},
 				"node:endpoint:*:pod-dddd:request_counter": {Object: int64(1), Expiration: int64(defaultExpiration)},
-				"node:endpoint:xxxx:pod-aaaa:nack_counter": {Object: int64(1), Expiration: int64(defaultExpiration)},
+				"node:endpoint:xxxx:pod-aaaa:nack_counter": {Object: int64(5), Expiration: int64(defaultExpiration)},
 				"node:endpoint:xxxx:pod-bbbb:nack_counter": {Object: int64(10), Expiration: int64(defaultExpiration)},
 			},
 			args: args{
@@ -360,7 +360,7 @@ func TestStats_GetPercentageFailing(t *testing.T) {
 				"node:endpoint:*:pod-bbbb:request_counter": {Object: int64(5), Expiration: int64(defaultExpiration)},
 				"node:endpoint:*:pod-cccc:request_counter": {Object: int64(1), Expiration: int64(defaultExpiration)},
 				"node:endpoint:*:pod-dddd:request_counter": {Object: int64(1), Expiration: int64(defaultExpiration)},
-				"node:endpoint:xxxx:pod-aaaa:nack_counter": {Object: int64(1), Expiration: int64(defaultExpiration)},
+				"node:endpoint:xxxx:pod-aaaa:nack_counter": {Object: int64(5), Expiration: int64(defaultExpiration)},
 				"node:endpoint:xxxx:pod-bbbb:nack_counter": {Object: int64(10), Expiration: int64(defaultExpiration)},
 				"node:endpoint:xxxx:pod-cccc:nack_counter": {Object: int64(10), Expiration: int64(defaultExpiration)},
 				"node:endpoint:xxxx:pod-dddd:nack_counter": {Object: int64(10), Expiration: int64(defaultExpiration)},

diff --git a/pkg/reconcilers/marin3r/envoyconfigrevision/status_test.go b/pkg/reconcilers/marin3r/envoyconfigrevision/status_test.go
@@ -199,8 +199,8 @@ func TestIsStatusReconciled(t *testing.T) {
 				versionTrackerFactory: func() *marin3rv1alpha1.VersionTracker { return &marin3rv1alpha1.VersionTracker{Endpoints: "aaaa"} },
 				dStats: func() *stats.Stats {
 					return stats.NewWithItems(map[string]cache.Item{
-						"test:" + resource_v3.EndpointType + ":*:pod-aaaa:request_counter:stream_1": {Object: int64(2), Expiration: int64(0)},
-						"test:" + resource_v3.EndpointType + ":aaaa:pod-aaaa:nack_counter":          {Object: int64(1), Expiration: int64(0)},
+						"test:" + resource_v3.EndpointType + ":*:pod-aaaa:request_counter:stream_1": {Object: int64(7), Expiration: int64(0)},
+						"test:" + resource_v3.EndpointType + ":aaaa:pod-aaaa:nack_counter":          {Object: int64(6), Expiration: int64(0)},
 					}, time.Now())
 				},
 			},
@@ -412,7 +412,7 @@ func Test_calculateRevisionTaintedCondition(t *testing.T) {
 					"node:" + resource_v3.EndpointType + ":*:pod-cccc:request_counter:stream_3": {Object: int64(1), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":*:pod-dddd:request_counter:stream_4": {Object: int64(1), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":*:pod-aaaa:request_counter:stream_1": {Object: int64(2), Expiration: int64(0)},
-					"node:" + resource_v3.EndpointType + ":xxxx:pod-aaaa:nack_counter":          {Object: int64(1), Expiration: int64(0)},
+					"node:" + resource_v3.EndpointType + ":xxxx:pod-aaaa:nack_counter":          {Object: int64(5), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":xxxx:pod-bbbb:nack_counter":          {Object: int64(10), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":xxxx:pod-cccc:nack_counter":          {Object: int64(10), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":xxxx:pod-dddd:nack_counter":          {Object: int64(10), Expiration: int64(0)},
@@ -444,7 +444,7 @@ func Test_calculateRevisionTaintedCondition(t *testing.T) {
 					"node:" + resource_v3.EndpointType + ":*:pod-cccc:request_counter:stream_3": {Object: int64(1), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":*:pod-dddd:request_counter:stream_4": {Object: int64(1), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":*:pod-aaaa:request_counter:stream_1": {Object: int64(2), Expiration: int64(0)},
-					"node:" + resource_v3.EndpointType + ":xxxx:pod-aaaa:nack_counter":          {Object: int64(1), Expiration: int64(0)},
+					"node:" + resource_v3.EndpointType + ":xxxx:pod-aaaa:nack_counter":          {Object: int64(5), Expiration: int64(0)},
 					"node:" + resource_v3.EndpointType + ":xxxx:pod-bbbb:nack_counter":          {Object: int64(10), Expiration: int64(0)},
 				}, time.Now()),
 				thresshold: 0.5,