implement MinimumMemoryBytesPerContainer/MinimumCPUCoresPerContainer (#…

…312)
mercari · Feb 5, 2024 · 83e3194 · 83e3194
1 parent 39756b7
commit 83e3194
Show file tree

Hide file tree

Showing 19 changed files with 324 additions and 57 deletions.
diff --git a/...suggested-too-small/after/deployment.yaml → ...suggested-too-small/after/deployment.yaml b/...suggested-too-small/after/deployment.yaml → ...suggested-too-small/after/deployment.yaml
@@ -12,6 +12,7 @@ spec:
       labels:
         app: mercari
       annotations:
+        sidecar.istio.io/inject: "true"
         kubectl.kubernetes.io/restartedAt: "exist"
     spec:
       containers:

diff --git a/...er-pod-suggested-too-small/after/hpa.yaml → ...rs-pod-suggested-too-small/after/hpa.yaml b/...er-pod-suggested-too-small/after/hpa.yaml → ...rs-pod-suggested-too-small/after/hpa.yaml
@@ -21,6 +21,13 @@ spec:
       stabilizationWindowSeconds: 0
   maxReplicas: 20
   metrics:
+  - containerResource:
+      container: istio-proxy
+      name: cpu
+      target:
+        averageUtilization: 75
+        type: Utilization
+    type: ContainerResource
   - containerResource:
       container: app
       name: cpu

diff --git a/...d-suggested-too-small/after/tortoise.yaml → ...d-suggested-too-small/after/tortoise.yaml b/...d-suggested-too-small/after/tortoise.yaml → ...d-suggested-too-small/after/tortoise.yaml
@@ -13,12 +13,20 @@ status:
       cpu: Horizontal
       memory: Vertical
     containerName: app
+  - policy:
+      cpu: Horizontal
+      memory: Vertical
+    containerName: istio-proxy
   conditions:
     containerResourceRequests:
       - containerName: "app"
         resource:
           cpu: "4"
           memory: 4Gi
+      - containerName: "istio-proxy"
+        resource:
+          cpu: "100m"
+          memory: "100Mi"
     tortoiseConditions:
     - message: "HPA target utilization is updated"
       reason: HPATargetUtilizationUpdated
@@ -40,6 +48,21 @@ status:
         memory:
           quantity: "1Mi"
           updatedAt: null
+    - containerName: istio-proxy
+      maxRecommendation:
+        cpu:
+          quantity: "75m"
+          updatedAt: null
+        memory:
+          quantity: "1Mi"
+          updatedAt: null
+      recommendation:
+        cpu:
+          quantity: "75m"
+          updatedAt: null
+        memory:
+          quantity: "1Mi"
+          updatedAt: null
   recommendations:
     horizontal:
       maxReplicas:
@@ -58,12 +81,19 @@ status:
       - containerName: app
         targetUtilization:
           cpu: 75
+      - containerName: istio-proxy
+        targetUtilization:
+          cpu: 75
     vertical:
       containerResourceRecommendation:
       - RecommendedResource:
           cpu: "4"
           memory: "10Mi" # global min
         containerName: app
+      - RecommendedResource:
+          cpu: "100m"
+          memory: "11Mi" # istio-proxy min
+        containerName: istio-proxy
   targets:
     horizontalPodAutoscaler: tortoise-hpa-mercari
     verticalPodAutoscalers:
@@ -74,6 +104,12 @@ status:
   tortoisePhase: Working
   containerResourcePhases:
     - containerName: "app"
+      resourcePhases:
+        cpu: 
+          phase: Working 
+        memory:
+          phase: Working 
+    - containerName: "istio-proxy"
       resourcePhases:
         cpu: 
           phase: Working 

diff --git a/...ggested-too-small/before/vpa-Monitor.yaml → ...uggested-too-small/after/vpa-Monitor.yaml b/...ggested-too-small/before/vpa-Monitor.yaml → ...uggested-too-small/after/vpa-Monitor.yaml
@@ -31,3 +31,13 @@ status:
       upperBound:
         cpu: "5"
         memory: "1Mi"
+    - containerName: istio-proxy
+      lowerBound:
+        cpu: "75m"
+        memory: "1Mi"
+      target:
+        cpu: "75m"
+        memory: "1Mi"
+      upperBound:
+        cpu: "75m"
+        memory: "1Mi"
diff --git a/...uggested-too-small/after/vpa-Updater.yaml → ...uggested-too-small/after/vpa-Updater.yaml b/...uggested-too-small/after/vpa-Updater.yaml → ...uggested-too-small/after/vpa-Updater.yaml
@@ -36,3 +36,16 @@ status:
       upperBound:
         cpu: "4"
         memory: "10Mi"
+    - containerName: istio-proxy
+      lowerBound:
+        cpu: "100m"
+        memory: "11Mi"
+      target:
+        cpu: "100m"
+        memory: "11Mi"
+      uncappedTarget:
+        cpu: "100m"
+        memory: "11Mi"
+      upperBound:
+        cpu: "100m"
+        memory: "11Mi"
diff --git a/...uggested-too-small/before/deployment.yaml → ...uggested-too-small/before/deployment.yaml b/...uggested-too-small/before/deployment.yaml → ...uggested-too-small/before/deployment.yaml
@@ -11,6 +11,8 @@ spec:
       creationTimestamp: null
       labels:
         app: mercari
+      annotations:
+        sidecar.istio.io/inject: "true"
     spec:
       containers:
       - image: awesome-mercari-app-image

diff --git a/...r-pod-suggested-too-small/before/hpa.yaml → ...s-pod-suggested-too-small/before/hpa.yaml b/...r-pod-suggested-too-small/before/hpa.yaml → ...s-pod-suggested-too-small/before/hpa.yaml
@@ -21,6 +21,13 @@ spec:
       stabilizationWindowSeconds: 0
   maxReplicas: 100
   metrics:
+  - containerResource:
+      container: istio-proxy
+      name: cpu
+      target:
+        averageUtilization: 50
+        type: Utilization
+    type: ContainerResource
   - containerResource:
       container: app
       name: cpu

diff --git a/...-suggested-too-small/before/tortoise.yaml → ...-suggested-too-small/before/tortoise.yaml b/...-suggested-too-small/before/tortoise.yaml → ...-suggested-too-small/before/tortoise.yaml
@@ -13,6 +13,10 @@ status:
       cpu: Horizontal
       memory: Vertical
     containerName: app
+  - policy:
+      cpu: Horizontal
+      memory: Vertical
+    containerName: istio-proxy
   conditions:
     containerRecommendationFromVPA:
     - containerName: app
@@ -30,6 +34,21 @@ status:
         memory:
           quantity: "0"
           updatedAt: null
+    - containerName: istio-proxy
+      maxRecommendation:
+        cpu:
+          quantity: "0"
+          updatedAt: null
+        memory:
+          quantity: "0"
+          updatedAt: null
+      recommendation:
+        cpu:
+          quantity: "0"
+          updatedAt: null
+        memory:
+          quantity: "0"
+          updatedAt: null
   recommendations:
     horizontal:
       maxReplicas:
@@ -60,6 +79,12 @@ status:
   tortoisePhase: Working
   containerResourcePhases:
     - containerName: "app"
+      resourcePhases:
+        cpu: 
+          phase: Working 
+        memory:
+          phase: Working 
+    - containerName: "istio-proxy"
       resourcePhases:
         cpu: 
           phase: Working 

diff --git a/...uggested-too-small/after/vpa-Monitor.yaml → ...ggested-too-small/before/vpa-Monitor.yaml b/...uggested-too-small/after/vpa-Monitor.yaml → ...ggested-too-small/before/vpa-Monitor.yaml
@@ -31,3 +31,13 @@ status:
       upperBound:
         cpu: "5"
         memory: "1Mi"
+    - containerName: istio-proxy
+      lowerBound:
+        cpu: "75m"
+        memory: "1Mi"
+      target:
+        cpu: "75m"
+        memory: "1Mi"
+      upperBound:
+        cpu: "75m"
+        memory: "1Mi"
diff --git a/...ggested-too-small/before/vpa-Updater.yaml → ...ggested-too-small/before/vpa-Updater.yaml b/...ggested-too-small/before/vpa-Updater.yaml → ...ggested-too-small/before/vpa-Updater.yaml
diff --git a/controllers/tortoise_controller_test.go b/controllers/tortoise_controller_test.go
@@ -172,7 +172,7 @@ func startController(ctx context.Context) func() {
 		VpaService:         cli,
 		DeploymentService:  deployment.New(mgr.GetClient(), "100m", "100Mi", recorder),
 		TortoiseService:    tortoiseService,
-		RecommenderService: recommender.New(2.0, 0.5, 90, 40, 3, 30, "10m", "10Mi", "10", "10Gi", 10000, recorder),
+		RecommenderService: recommender.New(2.0, 0.5, 90, 40, 3, 30, "10m", "10Mi", map[string]string{"istio-proxy": "11m"}, map[string]string{"istio-proxy": "11Mi"}, "10", "10Gi", 10000, recorder),
 	}
 	err = reconciler.SetupWithManager(mgr)
 	Expect(err).ShouldNot(HaveOccurred())
@@ -345,9 +345,6 @@ var _ = Describe("Test TortoiseController", func() {
 		It("TortoisePhaseWorking (GatheringData is just finished)", func() {
 			runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-gathering-data-finished"))
 		})
-		It("TortoisePhaseWorking (VPA suggestion too small)", func() {
-			runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-suggested-too-small"))
-		})
 		It("TortoisePhaseWorking (dryrun)", func() {
 			runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-dryrun"))
 		})
@@ -374,6 +371,9 @@ var _ = Describe("Test TortoiseController", func() {
 		It("TortoisePhaseWorking (All AutoscalingTypeOff)", func() {
 			runTest(filepath.Join("testdata", "reconcile-for-the-multiple-containers-pod-all-off"))
 		})
+		It("TortoisePhaseWorking (VPA suggestion too small)", func() {
+			runTest(filepath.Join("testdata", "reconcile-for-the-multiple-containers-pod-suggested-too-small"))
+		})
 		It("user just enabled TortoisePhaseEmergency", func() {
 			runTest(filepath.Join("testdata", "reconcile-for-the-multiple-containers-pod-emergency-started"))
 		})

diff --git a/main.go b/main.go
@@ -154,14 +154,29 @@ func main() {
 	}
 
 	if err = (&controllers.TortoiseReconciler{
-		Scheme:             mgr.GetScheme(),
-		HpaService:         hpaService,
-		VpaService:         vpaClient,
-		DeploymentService:  deployment.New(mgr.GetClient(), config.IstioSidecarProxyDefaultCPU, config.IstioSidecarProxyDefaultMemory, eventRecorder),
-		RecommenderService: recommender.New(config.MaxReplicasFactor, config.MinReplicasFactor, config.MaximumTargetResourceUtilization, config.MinimumTargetResourceUtilization, config.MinimumMinReplicas, config.PreferredMaxReplicas, config.MinimumCPUCores, config.MinimumMemoryBytes, config.MaximumCPUCores, config.MaximumMemoryBytes, config.MaximumMaxReplicas, eventRecorder),
-		TortoiseService:    tortoiseService,
-		Interval:           config.TortoiseUpdateInterval,
-		EventRecorder:      eventRecorder,
+		Scheme:            mgr.GetScheme(),
+		HpaService:        hpaService,
+		VpaService:        vpaClient,
+		DeploymentService: deployment.New(mgr.GetClient(), config.IstioSidecarProxyDefaultCPU, config.IstioSidecarProxyDefaultMemory, eventRecorder),
+		RecommenderService: recommender.New(
+			config.MaxReplicasFactor,
+			config.MinReplicasFactor,
+			config.MaximumTargetResourceUtilization,
+			config.MinimumTargetResourceUtilization,
+			config.MinimumMinReplicas,
+			config.PreferredMaxReplicas,
+			config.MinimumCPUCores,
+			config.MinimumMemoryBytes,
+			config.MinimumCPUCoresPerContainer,
+			config.MinimumMemoryBytesPerContainer,
+			config.MaximumCPUCores,
+			config.MaximumMemoryBytes,
+			config.MaximumMaxReplicas,
+			eventRecorder,
+		),
+		TortoiseService: tortoiseService,
+		Interval:        config.TortoiseUpdateInterval,
+		EventRecorder:   eventRecorder,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Tortoise")
 		os.Exit(1)

diff --git a/pkg/annotation/external_annotation.go b/pkg/annotation/external_annotation.go
@@ -6,4 +6,6 @@ const (
 
 	IstioSidecarProxyCPUAnnotation    = "sidecar.istio.io/proxyCPU"
 	IstioSidecarProxyMemoryAnnotation = "sidecar.istio.io/proxyMemory"
+
+	UpdatedAtAnnotation = "kubectl.kubernetes.io/restartedAt"
 )
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -44,8 +44,29 @@ type Config struct {
 	MaximumMemoryBytes string `yaml:"MaximumMemoryBytes"`
 	// MinimumCPUCores is the minimum CPU cores that the tortoise can give to the container (default: 50m)
 	MinimumCPUCores string `yaml:"MinimumCPUCores"`
+	// MinimumCPUCoresPerContainer is the minimum CPU cores per container that the tortoise can give to the container (default: nil)
+	// It has a higher priority than MinimumCPUCores.
+	// If you specify both, the tortoise uses MinimumCPUCoresPerContainer basically, but if the container name is not found in this map, the tortoise uses MinimumCPUCores.
+	//
+	// You can specify like this:
+	// ```
+	// MinimumCPUCoresPerContainer:
+	//  istio-proxy: 100m
+	//  hoge-agent: 120m
+	// ```
+	MinimumCPUCoresPerContainer map[string]string `yaml:"MinimumCPUCoresPerContainer"`
 	// MinimumMemoryBytes is the minimum memory bytes that the tortoise can give to the container (default: 50Mi)
 	MinimumMemoryBytes string `yaml:"MinimumMemoryBytes"`
+	// MinimumMemoryBytesPerContainer is the minimum memory bytes per container that the tortoise can give to the container (default: nil)
+	// If you specify both, the tortoise uses MinimumMemoryBytesPerContainer basically, but if the container name is not found in this map, the tortoise uses MinimumMemoryBytes.
+	//
+	// You can specify like this:
+	// ```
+	// MinimumMemoryBytesPerContainer:
+	//  istio-proxy: 100m
+	//  hoge-agent: 120m
+	// ```
+	MinimumMemoryBytesPerContainer map[string]string `yaml:"MinimumMemoryBytesPerContainer"`
 	// TimeZone is the timezone used to record time in tortoise objects (default: Asia/Tokyo)
 	TimeZone string `yaml:"TimeZone"`
 	// TortoiseUpdateInterval is the interval of updating each tortoise (default: 15s)
@@ -81,8 +102,10 @@ func defaultConfig() *Config {
 		PreferredMaxReplicas:                     30,
 		MaximumCPUCores:                          "10",
 		MinimumCPUCores:                          "50m",
+		MinimumCPUCoresPerContainer:              map[string]string{},
 		MaximumMemoryBytes:                       "10Gi",
 		MinimumMemoryBytes:                       "50Mi",
+		MinimumMemoryBytesPerContainer:           map[string]string{},
 		TimeZone:                                 "Asia/Tokyo",
 		TortoiseUpdateInterval:                   15 * time.Second,
 		HPATargetUtilizationMaxIncrease:          5,

diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -43,6 +43,14 @@ func TestParseConfig(t *testing.T) {
 				HPATargetUtilizationUpdateInterval:       3 * time.Hour,
 				IstioSidecarProxyDefaultCPU:              "100m",
 				IstioSidecarProxyDefaultMemory:           "200Mi",
+				MinimumCPUCoresPerContainer: map[string]string{
+					"istio-proxy": "100m",
+					"hoge-agent":  "120m",
+				},
+				MinimumMemoryBytesPerContainer: map[string]string{
+					"istio-proxy": "1Mi",
+					"hoge-agent":  "2Mi",
+				},
 			},
 		},
 		{
@@ -72,6 +80,8 @@ func TestParseConfig(t *testing.T) {
 				HPATargetUtilizationUpdateInterval:       24 * time.Hour,
 				IstioSidecarProxyDefaultCPU:              "100m",
 				IstioSidecarProxyDefaultMemory:           "200Mi",
+				MinimumCPUCoresPerContainer:              map[string]string{},
+				MinimumMemoryBytesPerContainer:           map[string]string{},
 			},
 		},
 		{
@@ -108,6 +118,8 @@ func TestParseConfig(t *testing.T) {
 				HPATargetUtilizationUpdateInterval:       24 * time.Hour,
 				IstioSidecarProxyDefaultCPU:              "100m",
 				IstioSidecarProxyDefaultMemory:           "200Mi",
+				MinimumCPUCoresPerContainer:              map[string]string{},
+				MinimumMemoryBytesPerContainer:           map[string]string{},
 			},
 		},
 	}

diff --git a/pkg/config/testdata/config.yaml b/pkg/config/testdata/config.yaml
@@ -11,4 +11,10 @@ MaximumMemoryBytes:                      "10Gi"
 TimeZone:                                 "Asia/Tokyo"
 TortoiseUpdateInterval:                   "1h"
 HPATargetUtilizationMaxIncrease:   10
-HPATargetUtilizationUpdateInterval:                   "3h"
+HPATargetUtilizationUpdateInterval:                   "3h"
+MinimumMemoryBytesPerContainer:
+  istio-proxy: 1Mi
+  hoge-agent: 2Mi
+MinimumCPUCoresPerContainer:
+  istio-proxy: 100m
+  hoge-agent: 120m
diff --git a/pkg/deployment/service.go b/pkg/deployment/service.go
@@ -41,13 +41,11 @@ func (c *Service) GetDeploymentOnTortoise(ctx context.Context, tortoise *autosca
 	return d, nil
 }
 
-const updatedAtAnnotation = "kubectl.kubernetes.io/restartedAt"
-
 func (c *Service) RolloutRestart(ctx context.Context, dm *v1.Deployment, tortoise *autoscalingv1beta3.Tortoise) error {
 	if dm.Spec.Template.ObjectMeta.Annotations == nil {
 		dm.Spec.Template.ObjectMeta.Annotations = make(map[string]string)
 	}
-	dm.Spec.Template.ObjectMeta.Annotations[updatedAtAnnotation] = time.Now().Format(time.RFC3339)
+	dm.Spec.Template.ObjectMeta.Annotations[annotation.UpdatedAtAnnotation] = time.Now().Format(time.RFC3339)
 
 	if err := c.c.Update(ctx, dm); err != nil {
 		return fmt.Errorf("failed to update deployment: %w", err)