add controller test and fix checkHPAStatus function

api/v1beta3/tortoise_types.go

@@ -256,7 +256,8 @@ const (
 	// TortoisePhaseBackToNormal means tortoise was in the emergency mode, and now it's coming back to the normal operation.
 	// During TortoisePhaseBackToNormal, the number of replicas of workloads are gradually reduced to the usual value.
 	//  - Emergency → BackToNormal
-	TortoisePhaseBackToNormal TortoisePhase = "BackToNormal"
+	TortoisePhaseBackToNormal  TortoisePhase = "BackToNormal"
+	TortoisePhaseAutoEmergency TortoisePhase = "AutoEmergency"
 )
 
 type TargetsStatus struct {

internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml

@@ -18,23 +18,23 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 15
+  maxReplicas: 20
   metrics:
   - containerResource:
       container: app
       name: cpu
       target:
-        averageUtilization: 30
+        averageUtilization: 55
         type: Utilization
     type: ContainerResource
   - containerResource:
       container: istio-proxy
       name: cpu
       target:
-        averageUtilization: 30
+        averageUtilization: 55
         type: Utilization
     type: ContainerResource
-  minReplicas: 15
+  minReplicas: 20
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
@@ -47,5 +47,15 @@ status:
     - status: "False"
       type: ScalingActive
       message: "the HPA was unable to compute the replica count: failed to get cpu utilization"
-  currentMetrics: null
+  currentMetrics:
+    - containerResource:
+        container: app
+        name: cpu
+        current:
+          value: 0
+    - containerResource:
+        container: istio-proxy
+        name: cpu
+        current:
+          value: 0
   desiredReplicas: 0

internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml

@@ -9,7 +9,7 @@ spec:
       apiVersion: apps/v1
       kind: Deployment
       name: mercari-app
-  updateMode: Emergency
+  updateMode: Auto
 status:
   autoscalingPolicy:
   - containerName: app
@@ -55,13 +55,20 @@ status:
     containerResourceRequests:
     - containerName: app
       resource:
-        cpu: "6"
+        cpu: "10"
         memory: 3Gi
     - containerName: istio-proxy
       resource:
         cpu: "4"
         memory: 3Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
@@ -100,25 +107,25 @@ status:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:47Z"
-        value: 15
+        updatedAt: "2023-01-01T00:00:00Z"
+        value: 20
       minReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:47Z"
-        value: 3
+        updatedAt: "2023-01-01T00:00:00Z"
+        value: 5
       targetUtilizations:
       - containerName: app
         targetUtilization:
-          cpu: 30
+          cpu: 90
       - containerName: istio-proxy
         targetUtilization:
-          cpu: 30
+          cpu: 55
     vertical:
       containerResourceRecommendation:
       - RecommendedResource:
-          cpu: "6"
+          cpu: "10"
           memory: 3Gi
         containerName: app
       - RecommendedResource:
@@ -133,4 +140,4 @@ status:
     verticalPodAutoscalers:
     - name: tortoise-monitor-mercari
       role: Monitor
-  tortoisePhase: Emergency
+  tortoisePhase: AutoEmergency

internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/hpa.yaml

@@ -11,6 +11,17 @@ status:
     - status: "False"
       type: ScalingActive
       message: "the HPA was unable to compute the replica count: failed to get cpu utilization"
+  currentMetrics:
+    - containerResource:
+        container: app
+        name: cpu
+        current:
+          value: 0
+    - containerResource:
+        container: istio-proxy
+        name: cpu
+        current:
+          value: 0
 spec:
   behavior:
     scaleDown:

internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml

@@ -19,82 +19,60 @@ status:
       memory: Vertical
     containerName: istio-proxy
   conditions:
-    containerResourceRequests:
-      - containerName: "app"
-        resource:
-          cpu: "10"
-          memory: 10Gi
-      - containerName: "istio-proxy"
-        resource:
-          cpu: "4"
-          memory: 4Gi
-    tortoiseConditions:
-    - message: "HPA target utilization is updated"
-      reason: HPATargetUtilizationUpdated
-      status: "True"
-      type: HPATargetUtilizationUpdated
     containerRecommendationFromVPA:
     - containerName: app
       maxRecommendation:
         cpu:
-          quantity: "3"
+          quantity: "0"
           updatedAt: null
         memory:
-          quantity: 3Gi
+          quantity: "0"
           updatedAt: null
       recommendation:
         cpu:
-          quantity: "3"
+          quantity: "0"
           updatedAt: null
         memory:
-          quantity: 3Gi
+          quantity: "0"
           updatedAt: null
     - containerName: istio-proxy
       maxRecommendation:
         cpu:
-          quantity: "3"
+          quantity: "0"
           updatedAt: null
         memory:
-          quantity: 3Gi
+          quantity: "0"
           updatedAt: null
       recommendation:
         cpu:
-          quantity: "3"
+          quantity: "0"
           updatedAt: null
         memory:
-          quantity: 3Gi
+          quantity: "0"
           updatedAt: null
   recommendations:
     horizontal:
       maxReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:47Z"
+        updatedAt: "2023-10-06T01:15:46Z"
         value: 15
       minReplicas:
       - from: 0
         timezone: Local
         to: 24
-        updatedAt: "2023-10-06T01:15:47Z"
+        updatedAt: "2023-10-06T01:15:46Z"
         value: 3
       targetUtilizations:
       - containerName: app
         targetUtilization:
-          cpu: 30
+          cpu: 50
       - containerName: istio-proxy
         targetUtilization:
-          cpu: 30
+          cpu: 50
     vertical:
-      containerResourceRecommendation:
-      - RecommendedResource:
-          cpu: "6"
-          memory: 3Gi
-        containerName: app
-      - RecommendedResource:
-          cpu: "4"
-          memory: 3Gi
-        containerName: istio-proxy
+      containerResourceRecommendation: null
   targets:
     horizontalPodAutoscaler: tortoise-hpa-mercari
     verticalPodAutoscalers:

internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/deployment.yaml

@@ -0,0 +1,30 @@
+metadata:
+  name: mercari-app
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: mercari
+  strategy: {}
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z"
+      creationTimestamp: null
+      labels:
+        app: mercari
+    spec:
+      containers:
+      - image: awesome-mercari-app-image
+        name: app
+        resources:
+          requests:
+            cpu: "10"
+            memory: 10Gi
+      - image: awesome-istio-proxy-image
+        name: istio-proxy
+        resources:
+          requests:
+            cpu: "4"
+            memory: 4Gi
+status: {}

internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml

@@ -0,0 +1,61 @@
+metadata:
+  annotations:
+    tortoise.autoscaling.mercari.com/managed-by-tortoise: "true"
+  name: tortoise-hpa-mercari
+  namespace: default
+spec:
+  behavior:
+    scaleDown:
+      policies:
+      - periodSeconds: 90
+        type: Percent
+        value: 2
+      selectPolicy: Max
+    scaleUp:
+      policies:
+      - periodSeconds: 60
+        type: Percent
+        value: 100
+      selectPolicy: Max
+      stabilizationWindowSeconds: 0
+  maxReplicas: 20
+  metrics:
+  - containerResource:
+      container: app
+      name: cpu
+      target:
+        averageUtilization: 55
+        type: Utilization
+    type: ContainerResource
+  - containerResource:
+      container: istio-proxy
+      name: cpu
+      target:
+        averageUtilization: 55
+        type: Utilization
+    type: ContainerResource
+  minReplicas: 20
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: mercari-app
+status:
+  conditions:
+    - status: "True"
+      type: AbleToScale
+      message: "recommended size matches current size"
+    - status: "True"
+      type: ScalingActive
+      message: "the HPA was able to compute the replica count"
+  currentMetrics:
+    - containerResource:
+        container: app
+        name: cpu
+        current:
+          value: 0
+    - containerResource:
+        container: istio-proxy
+        name: cpu
+        current:
+          value: 0
+  desiredReplicas: 0