diff --git a/Makefile b/Makefile index 388951c3..36e8c61e 100644 --- a/Makefile +++ b/Makefile @@ -165,7 +165,7 @@ build: manifests generate fmt vet ## Build manager binary. CGO_ENABLED=0 GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -o bin/mondoo-operator -ldflags $(LDFLAGS) cmd/mondoo-operator/main.go run: manifests generate fmt vet ## Run a controller from your host. - MONDOO_OPERATOR_NAMESPACE=mondoo-operator go run ./cmd/mondoo-operator/main.go operator + MONDOO_NAMESPACE_OVERRIDE=mondoo-operator go run ./cmd/mondoo-operator/main.go operator docker-build: TARGET_OS=linux docker-build: build ## Build docker image with the manager. diff --git a/controllers/nodes/resources.go b/controllers/nodes/resources.go index f48feb77..05a7afc2 100644 --- a/controllers/nodes/resources.go +++ b/controllers/nodes/resources.go @@ -24,6 +24,7 @@ import ( "go.mondoo.com/mondoo-operator/controllers/scanapi" "go.mondoo.com/mondoo-operator/pkg/constants" "go.mondoo.com/mondoo-operator/pkg/feature_flags" + "go.mondoo.com/mondoo-operator/pkg/utils/gomemlimit" "go.mondoo.com/mondoo-operator/pkg/utils/k8s" ) @@ -75,12 +76,15 @@ func UpdateCronJob(cj *batchv1.CronJob, image string, node corev1.Node, m *v1alp // The node scanning does not use the Kubernetes API at all, therefore the service account token // should not be mounted at all. cj.Spec.JobTemplate.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) + containerResources := k8s.ResourcesRequirementsWithDefaults(m.Spec.Nodes.Resources, k8s.DefaultNodeScanningResources) + gcLimit := gomemlimit.CalculateGoMemLimit(containerResources) + cj.Spec.JobTemplate.Spec.Template.Spec.Containers = []corev1.Container{ { Image: image, Name: "cnspec", Command: cmd, - Resources: k8s.ResourcesRequirementsWithDefaults(m.Spec.Nodes.Resources, k8s.DefaultNodeScanningResources), + Resources: containerResources, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(isOpenshift), ReadOnlyRootFilesystem: ptr.To(true), @@ -128,6 +132,10 @@ func UpdateCronJob(cj *batchv1.CronJob, image string, node corev1.Node, m *v1alp Name: "NODE_NAME", Value: node.Name, }, + { + Name: "GOMEMLIMIT", + Value: gcLimit, + }, }, m.Spec.Nodes.Env), TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile, @@ -213,12 +221,15 @@ func UpdateDaemonSet( // The node scanning does not use the Kubernetes API at all, therefore the service account token // should not be mounted at all. ds.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(false) + containerResources := k8s.ResourcesRequirementsWithDefaults(m.Spec.Nodes.Resources, k8s.DefaultNodeScanningResources) + gcLimit := gomemlimit.CalculateGoMemLimit(containerResources) + ds.Spec.Template.Spec.Containers = []corev1.Container{ { Image: image, Name: "cnspec", Command: cmd, - Resources: k8s.ResourcesRequirementsWithDefaults(m.Spec.Nodes.Resources, k8s.DefaultNodeScanningResources), + Resources: containerResources, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(isOpenshift), ReadOnlyRootFilesystem: ptr.To(true), @@ -265,6 +276,10 @@ func UpdateDaemonSet( Name: "MONDOO_AUTO_UPDATE", Value: "false", }, + { + Name: "GOMEMLIMIT", + Value: gcLimit, + }, { Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{ diff --git a/controllers/nodes/resources_test.go b/controllers/nodes/resources_test.go index e9ec4e8b..04edcf32 100644 --- a/controllers/nodes/resources_test.go +++ b/controllers/nodes/resources_test.go @@ -13,6 +13,7 @@ import ( "go.mondoo.com/mondoo-operator/pkg/constants" "go.mondoo.com/mondoo-operator/pkg/utils/k8s" "go.mondoo.com/mondoo-operator/tests/framework/utils" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -122,6 +123,85 @@ func TestResources(t *testing.T) { } } +func TestResources_GOMEMLIMIT(t *testing.T) { + tests := []struct { + name string + mondooauditconfig func() *v1alpha2.MondooAuditConfig + expectedGoMemLimit string + }{ + { + name: "resources should match default", + mondooauditconfig: func() *v1alpha2.MondooAuditConfig { + return testMondooAuditConfig() + }, + expectedGoMemLimit: "225000000", + }, + { + name: "resources should match spec", + mondooauditconfig: func() *v1alpha2.MondooAuditConfig { + mac := testMondooAuditConfig() + mac.Spec.Nodes.Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + return mac + }, + expectedGoMemLimit: "94371840", + }, + { + name: "resources should match off", + mondooauditconfig: func() *v1alpha2.MondooAuditConfig { + mac := testMondooAuditConfig() + mac.Spec.Nodes.Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + } + return mac + }, + expectedGoMemLimit: "off", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node-name", + }, + } + mac := *test.mondooauditconfig() + cj := &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Name: "name", Namespace: mac.Namespace}} + UpdateCronJob(cj, "test123", *testNode, &mac, false, v1alpha2.MondooOperatorConfig{}) + goMemLimitEnv := corev1.EnvVar{} + for _, env := range cj.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Env { + if env.Name == "GOMEMLIMIT" { + goMemLimitEnv = env + break + } + } + assert.Equal(t, test.expectedGoMemLimit, goMemLimitEnv.Value) + }) + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mac := *test.mondooauditconfig() + ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "name", Namespace: mac.Namespace}} + UpdateDaemonSet(ds, mac, false, "test123", v1alpha2.MondooOperatorConfig{}) + goMemLimitEnv := corev1.EnvVar{} + for _, env := range ds.Spec.Template.Spec.Containers[0].Env { + if env.Name == "GOMEMLIMIT" { + goMemLimitEnv = env + break + } + } + assert.Equal(t, test.expectedGoMemLimit, goMemLimitEnv.Value) + }) + } +} + func TestCronJob_PrivilegedOpenshift(t *testing.T) { testNode := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controllers/scanapi/resources.go b/controllers/scanapi/resources.go index 9ee2903e..fb79424e 100644 --- a/controllers/scanapi/resources.go +++ b/controllers/scanapi/resources.go @@ -15,6 +15,7 @@ import ( "k8s.io/utils/ptr" "go.mondoo.com/mondoo-operator/api/v1alpha2" + "go.mondoo.com/mondoo-operator/pkg/utils/gomemlimit" "go.mondoo.com/mondoo-operator/pkg/utils/k8s" ) @@ -56,6 +57,8 @@ func ScanApiDeployment(ns, image string, m v1alpha2.MondooAuditConfig, cfg v1alp } healthcheckEndpoint := "/Scan/HealthCheck" + containerResources := k8s.ResourcesRequirementsWithDefaults(m.Spec.Scanner.Resources, k8s.DefaultCnspecResources) + gcLimit := gomemlimit.CalculateGoMemLimit(containerResources) scanApiDeployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -78,7 +81,7 @@ func ScanApiDeployment(ns, image string, m v1alpha2.MondooAuditConfig, cfg v1alp ImagePullPolicy: corev1.PullAlways, Name: name, Command: cmd, - Resources: k8s.ResourcesRequirementsWithDefaults(m.Spec.Scanner.Resources, k8s.DefaultCnspecResources), + Resources: containerResources, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -153,6 +156,7 @@ func ScanApiDeployment(ns, image string, m v1alpha2.MondooAuditConfig, cfg v1alp // Required so the scan API knows it is running as a Kubernetes integration {Name: "KUBERNETES_ADMISSION_CONTROLLER", Value: "true"}, + {Name: "GOMEMLIMIT", Value: gcLimit}, }, }}, ServiceAccountName: m.Spec.Scanner.ServiceAccountName, diff --git a/controllers/scanapi/resources_test.go b/controllers/scanapi/resources_test.go new file mode 100644 index 00000000..263d0887 --- /dev/null +++ b/controllers/scanapi/resources_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package scanapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.mondoo.com/mondoo-operator/api/v1alpha2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + testMondooAuditConfigName = "mondoo-config" + testNamespace = "mondoo-operator" +) + +func TestResources_GOMEMLIMIT(t *testing.T) { + tests := []struct { + name string + mondooauditconfig func() *v1alpha2.MondooAuditConfig + expectedGoMemLimit string + }{ + { + name: "resources should match default", + mondooauditconfig: func() *v1alpha2.MondooAuditConfig { + return testMondooAuditConfig() + }, + expectedGoMemLimit: "405000000", + }, + { + name: "resources should match spec", + mondooauditconfig: func() *v1alpha2.MondooAuditConfig { + mac := testMondooAuditConfig() + mac.Spec.Scanner.Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + return mac + }, + expectedGoMemLimit: "94371840", + }, + { + name: "resources should match off", + mondooauditconfig: func() *v1alpha2.MondooAuditConfig { + mac := testMondooAuditConfig() + mac.Spec.Scanner.Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + } + return mac + }, + expectedGoMemLimit: "off", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mac := *test.mondooauditconfig() + dep := ScanApiDeployment(testNamespace, "test123", mac, v1alpha2.MondooOperatorConfig{}, "", false) + goMemLimitEnv := corev1.EnvVar{} + for _, env := range dep.Spec.Template.Spec.Containers[0].Env { + if env.Name == "GOMEMLIMIT" { + goMemLimitEnv = env + break + } + } + assert.Equal(t, test.expectedGoMemLimit, goMemLimitEnv.Value) + }) + } +} + +func testMondooAuditConfig() *v1alpha2.MondooAuditConfig { + return &v1alpha2.MondooAuditConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: testMondooAuditConfigName, + Namespace: testNamespace, + }, + } +} diff --git a/pkg/utils/gomemlimit/gomemlimit.go b/pkg/utils/gomemlimit/gomemlimit.go new file mode 100644 index 00000000..303ee0ee --- /dev/null +++ b/pkg/utils/gomemlimit/gomemlimit.go @@ -0,0 +1,30 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package gomemlimit + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" +) + +func CalculateGoMemLimit(containerResources v1.ResourceRequirements) string { + // https://cs.opensource.google/go/go/+/master:src/runtime/mgcpacer.go;l=96?q=GOMEMLIMIT&ss=go%2Fgo + // Initialized from GOMEMLIMIT. GOMEMLIMIT=off is equivalent to MaxInt64 + // which means no soft memory limit in practice. + gcLimit := "off" + memoryLimit := containerResources.Limits.Memory() + + if memoryLimit != nil { + // https://go.dev/doc/gc-guide#Suggested_uses + // deployment ... into containers with a fixed amount of available memory. + // In this case, a good rule of thumb is to leave an additional 5-10% of headroom to account for memory sources the Go runtime is unaware of. + gcLimit = fmt.Sprintf("%.0f", (float64(memoryLimit.Value()) * 0.9)) + if gcLimit == "0" { + gcLimit = "off" + } + } + + return gcLimit +} diff --git a/pkg/utils/gomemlimit/gomemlimit_test.go b/pkg/utils/gomemlimit/gomemlimit_test.go new file mode 100644 index 00000000..54e5bbe3 --- /dev/null +++ b/pkg/utils/gomemlimit/gomemlimit_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package gomemlimit + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestCalculateGoMemLimit(t *testing.T) { + tests := []struct { + name string + resources v1.ResourceRequirements + expectedGoMemLimit string + }{ + { + name: "resources should match default", + resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("250M"), + }, + }, + expectedGoMemLimit: "225000000", + }, + { + name: "resources should match spec", + resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + expectedGoMemLimit: "94371840", + }, + { + name: "resources should match off", + resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + }, + }, + expectedGoMemLimit: "off", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expectedGoMemLimit, CalculateGoMemLimit(test.resources)) + }) + } +}