remove autoemergency phase and use emergency instead

mercari · Dec 6, 2024 · 0dc2749 · 0dc2749
1 parent bddb139
commit 0dc2749
Show file tree

Hide file tree

Showing 10 changed files with 98 additions and 76 deletions.
diff --git a/api/v1beta3/tortoise_types.go b/api/v1beta3/tortoise_types.go
@@ -256,8 +256,7 @@ const (
 	// TortoisePhaseBackToNormal means tortoise was in the emergency mode, and now it's coming back to the normal operation.
 	// During TortoisePhaseBackToNormal, the number of replicas of workloads are gradually reduced to the usual value.
 	//  - Emergency → BackToNormal
-	TortoisePhaseBackToNormal  TortoisePhase = "BackToNormal"
-	TortoisePhaseAutoEmergency TortoisePhase = "AutoEmergency"
+	TortoisePhaseBackToNormal TortoisePhase = "BackToNormal"
 )
 
 type TargetsStatus struct {

diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml
@@ -18,23 +18,23 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 15
   metrics:
   - containerResource:
       container: app
       name: cpu
       target:
-        averageUtilization: 55
+        averageUtilization: 30
         type: Utilization
     type: ContainerResource
   - containerResource:
       container: istio-proxy
       name: cpu
       target:
-        averageUtilization: 55
+        averageUtilization: 30
         type: Utilization
     type: ContainerResource
-  minReplicas: 20
+  minReplicas: 15
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment

diff --git a/.../controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml b/.../controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml
@@ -55,20 +55,13 @@ status:
     containerResourceRequests:
     - containerName: app
       resource:
-        cpu: "10"
+        cpu: "6"
         memory: 3Gi
     - containerName: istio-proxy
       resource:
         cpu: "4"
         memory: 3Gi
     tortoiseConditions:
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: the current number of replicas is not bigger than the preferred max
-        replica number
-      reason: ScaledUpBasedOnPreferredMaxReplicas
-      status: "False"
-      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
@@ -107,25 +100,25 @@ status:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-01-01T00:00:00Z"
-        value: 20
+        updatedAt: "2023-10-06T01:15:47Z"
+        value: 15
       minReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-01-01T00:00:00Z"
-        value: 5
+        updatedAt: "2023-10-06T01:15:47Z"
+        value: 3
       targetUtilizations:
       - containerName: app
         targetUtilization:
-          cpu: 90
+          cpu: 30
       - containerName: istio-proxy
         targetUtilization:
-          cpu: 55
+          cpu: 30
     vertical:
       containerResourceRecommendation:
       - RecommendedResource:
-          cpu: "10"
+          cpu: "6"
           memory: 3Gi
         containerName: app
       - RecommendedResource:
@@ -140,4 +133,4 @@ status:
     verticalPodAutoscalers:
     - name: tortoise-monitor-mercari
       role: Monitor
-  tortoisePhase: AutoEmergency
+  tortoisePhase: Emergency
diff --git a/...controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml b/...controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml
@@ -19,60 +19,82 @@ status:
       memory: Vertical
     containerName: istio-proxy
   conditions:
+    containerResourceRequests:
+      - containerName: "app"
+        resource:
+          cpu: "10"
+          memory: 10Gi
+      - containerName: "istio-proxy"
+        resource:
+          cpu: "4"
+          memory: 4Gi
+    tortoiseConditions:
+    - message: "HPA target utilization is updated"
+      reason: HPATargetUtilizationUpdated
+      status: "True"
+      type: HPATargetUtilizationUpdated
     containerRecommendationFromVPA:
     - containerName: app
       maxRecommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
       recommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
     - containerName: istio-proxy
       maxRecommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
       recommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
   recommendations:
     horizontal:
       maxReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:46Z"
+        updatedAt: "2023-10-06T01:15:47Z"
         value: 15
       minReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:46Z"
+        updatedAt: "2023-10-06T01:15:47Z"
         value: 3
       targetUtilizations:
       - containerName: app
         targetUtilization:
-          cpu: 50
+          cpu: 30
       - containerName: istio-proxy
         targetUtilization:
-          cpu: 50
+          cpu: 30
     vertical:
-      containerResourceRecommendation: null
+      containerResourceRecommendation:
+      - RecommendedResource:
+          cpu: "6"
+          memory: 3Gi
+        containerName: app
+      - RecommendedResource:
+          cpu: "4"
+          memory: 3Gi
+        containerName: istio-proxy
   targets:
     horizontalPodAutoscaler: tortoise-hpa-mercari
     verticalPodAutoscalers:

diff --git a/...rnal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml b/...rnal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml
@@ -18,23 +18,23 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 15
   metrics:
   - containerResource:
       container: app
       name: cpu
       target:
-        averageUtilization: 55
+        averageUtilization: 30
         type: Utilization
     type: ContainerResource
   - containerResource:
       container: istio-proxy
       name: cpu
       target:
-        averageUtilization: 55
+        averageUtilization: 30
         type: Utilization
     type: ContainerResource
-  minReplicas: 20
+  minReplicas: 15
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment

diff --git a/...controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml b/...controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml
@@ -55,20 +55,13 @@ status:
     containerResourceRequests:
     - containerName: app
       resource:
-        cpu: "10"
+        cpu: "6"
         memory: 3Gi
     - containerName: istio-proxy
       resource:
         cpu: "4"
         memory: 3Gi
     tortoiseConditions:
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: the current number of replicas is not bigger than the preferred max
-        replica number
-      reason: ScaledUpBasedOnPreferredMaxReplicas
-      status: "False"
-      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
@@ -107,25 +100,25 @@ status:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-01-01T00:00:00Z"
-        value: 20
+        updatedAt: "2023-10-06T01:15:47Z"
+        value: 15
       minReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-01-01T00:00:00Z"
-        value: 5
+        updatedAt: "2023-10-06T01:15:47Z"
+        value: 3
       targetUtilizations:
       - containerName: app
         targetUtilization:
-          cpu: 90
+          cpu: 30
       - containerName: istio-proxy
         targetUtilization:
-          cpu: 55
+          cpu: 30
     vertical:
       containerResourceRecommendation:
       - RecommendedResource:
-          cpu: "10"
+          cpu: "6"
           memory: 3Gi
         containerName: app
       - RecommendedResource:
@@ -140,4 +133,4 @@ status:
     verticalPodAutoscalers:
     - name: tortoise-monitor-mercari
       role: Monitor
-  tortoisePhase: AutoEmergency
+  tortoisePhase: Emergency
diff --git a/...nal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml b/...nal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml
@@ -58,4 +58,3 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: mercari-app
-
diff --git a/...ontroller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml b/...ontroller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml
@@ -19,60 +19,82 @@ status:
       memory: Vertical
     containerName: istio-proxy
   conditions:
+    containerResourceRequests:
+      - containerName: "app"
+        resource:
+          cpu: "10"
+          memory: 10Gi
+      - containerName: "istio-proxy"
+        resource:
+          cpu: "4"
+          memory: 4Gi
+    tortoiseConditions:
+    - message: "HPA target utilization is updated"
+      reason: HPATargetUtilizationUpdated
+      status: "True"
+      type: HPATargetUtilizationUpdated
     containerRecommendationFromVPA:
     - containerName: app
       maxRecommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
       recommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
     - containerName: istio-proxy
       maxRecommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
       recommendation:
         cpu:
-          quantity: "0"
+          quantity: "3"
           updatedAt: null
         memory:
-          quantity: "0"
+          quantity: 3Gi
           updatedAt: null
   recommendations:
     horizontal:
       maxReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:46Z"
+        updatedAt: "2023-10-06T01:15:47Z"
         value: 15
       minReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:46Z"
+        updatedAt: "2023-10-06T01:15:47Z"
         value: 3
       targetUtilizations:
       - containerName: app
         targetUtilization:
-          cpu: 50
+          cpu: 30
       - containerName: istio-proxy
         targetUtilization:
-          cpu: 50
+          cpu: 30
     vertical:
-      containerResourceRecommendation: null
+      containerResourceRecommendation:
+      - RecommendedResource:
+          cpu: "6"
+          memory: 3Gi
+        containerName: app
+      - RecommendedResource:
+          cpu: "4"
+          memory: 3Gi
+        containerName: istio-proxy
   targets:
     horizontalPodAutoscaler: tortoise-hpa-mercari
     verticalPodAutoscalers:

diff --git a/internal/controller/tortoise_controller.go b/internal/controller/tortoise_controller.go
@@ -241,10 +241,7 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 	}
 	scalingActive := r.HpaService.CheckHpaMetricStatus(ctx, hpa)
 	if scalingActive == false && tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeAuto && tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseWorking {
-		tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseAutoEmergency
-	}
-	if scalingActive == true && tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeAuto && tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseAutoEmergency {
-		tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseBackToNormal
+		tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseEmergency
 	}
 
 	tortoise = r.TortoiseService.UpdateContainerRecommendationFromVPA(tortoise, monitorvpa, now)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -58,4 +58,3 @@ spec:
		apiVersion: apps/v1
		kind: Deployment
		name: mercari-app