From 8b7d445ff927ad7beac6d2c7dda3af0f552edfa4 Mon Sep 17 00:00:00 2001 From: Vincent Shen Date: Tue, 12 Sep 2023 08:10:14 -0700 Subject: [PATCH] Deprecating old kubeletconfig logic This commit remove old kubeletconfig check logic and fix e2e with it --- cmd/manager/scap.go | 110 +----- cmd/manager/scap_test.go | 326 ------------------ .../complianceremediation_controller.go | 9 +- .../compliancesuite_controller.go | 22 +- .../compliancesuite_controller_test.go | 79 +---- pkg/utils/compare_json.go | 167 --------- pkg/utils/nodeutils.go | 100 ------ pkg/utils/nodeutils_test.go | 298 ---------------- tests/e2e/framework/common.go | 38 +- tests/e2e/parallel/main_test.go | 190 ---------- 10 files changed, 31 insertions(+), 1308 deletions(-) delete mode 100644 pkg/utils/compare_json.go diff --git a/cmd/manager/scap.go b/cmd/manager/scap.go index eb27bf9e9..38e83f53e 100644 --- a/cmd/manager/scap.go +++ b/cmd/manager/scap.go @@ -32,9 +32,7 @@ import ( mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" mcfgcommon "github.com/openshift/machine-config-operator/pkg/controller/common" - "github.com/wI2L/jsondiff" "gopkg.in/yaml.v3" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" runtimejson "k8s.io/apimachinery/pkg/runtime/serializer/json" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -48,10 +46,8 @@ import ( ) const ( - contentFileTimeout = 3600 - valuePrefix = "xccdf_org.ssgproject.content_value_" - kubeletConfigPathPrefix = "/kubeletconfig/" - kubeletConfigRolePathPrefix = "/kubeletconfig/role/" + contentFileTimeout = 3600 + valuePrefix = "xccdf_org.ssgproject.content_value_" ) var ( @@ -192,16 +188,6 @@ func (c *scapContentDataStream) FigureResources(profile string) error { }, } - roleNodesList, err := fetchNodesWithRole(context.Background(), c.resourceFetcherClients.client) - if err != nil { - LOG("Failed to fetch role list with nodes, error: %v", err) - return err - } - - if len(roleNodesList) > 0 { - found = append(found, getKubeletConfigResourcePath(roleNodesList)...) - } - effectiveProfile := profile var valuesList map[string]string @@ -245,56 +231,6 @@ func getPathFromWarningXML(in *xmlquery.Node, valueList map[string]string) []uti return path } -// Fetch all nodes from the cluster and find all roles for each node. -func fetchNodesWithRole(ctx context.Context, c runtimeclient.Client) (map[string][]string, error) { - nodeList := v1.NodeList{} - if err := c.List(ctx, &nodeList); err != nil { - return nil, fmt.Errorf("failed to list nodes: %w", err) - } - - roleNodesList := make(map[string][]string) - for _, node := range nodeList.Items { - nodeName := node.Name - nodeRoles := utils.GetNodeRoles(node.ObjectMeta.Labels) - for _, role := range nodeRoles { - roleNodesList[role] = append(roleNodesList[role], nodeName) - } - } - - return roleNodesList, nil - -} - -// Get resourcePath for KubeletConfig -func getKubeletConfigResourcePath(roleNodesList map[string][]string) []utils.ResourcePath { - resourcePath := []utils.ResourcePath{} - for role, nodeList := range roleNodesList { - for _, node := range nodeList { - resourcePath = append(resourcePath, utils.ResourcePath{ - ObjPath: "/api/v1/nodes/" + node + "/proxy/configz", - DumpPath: kubeletConfigPathPrefix + role + "/" + node, - Filter: `.kubeletconfig|.kind="KubeletConfiguration"|.apiVersion="kubelet.config.k8s.io/v1beta1"`, - }) - } - } - return resourcePath -} - -// Get role name and node name from DumpPath -func getRoleNodeNameFromDumpPath(dumpPath string) (roleName string, nodeName string) { - if strings.HasPrefix(dumpPath, kubeletConfigPathPrefix) { - dumpPathSplit := strings.Split(dumpPath, "/") - if len(dumpPathSplit) != 4 { - return "", "" - } - // example of DumpPath: /kubeletconfig/master/node-1 - roleName = dumpPathSplit[2] - nodeName = dumpPathSplit[3] - return roleName, nodeName - } - return "", "" -} - // Collect the resource paths for objects that this scan needs to obtain. // The profile will have a series of "selected" checks that we grab all of the path info from. func getResourcePaths(profileDefs *xmlquery.Node, ruleDefs *xmlquery.Node, profile string, overrideValueList map[string]string) ([]utils.ResourcePath, map[string]string) { @@ -597,47 +533,7 @@ func fetch(ctx context.Context, streamDispatcher streamerDispatcherFn, rfClients return nil, warnings, err } } - results, warnings, err := saveConsistentKubeletResult(results, warnings) - return results, warnings, err -} - -// Only save consistent KubeletConfigs per node role. -func saveConsistentKubeletResult(result map[string][]byte, warning []string) (map[string][]byte, []string, error) { - if len(result) == 0 { - return result, warning, nil - } - kubeletConfigsRole := make(map[string][]byte) - for dumpPath, content := range result { - role, node := getRoleNodeNameFromDumpPath(dumpPath) - if role == "" { - continue - } - if existingKC, ok := kubeletConfigsRole[role]; ok { - diff, err := jsondiff.CompareJSON(existingKC, content) - if err != nil { - return nil, nil, fmt.Errorf("couldn't compare kubelet configs: %w for %s", err, node) - } - if diff != nil { - why := fmt.Sprintf("Kubelet configs for %s are not consistent with role %s, Diff: %s of KubeletConfigs for %s role will not be saved.", node, role, diff, role) - LOG(why) - warning = append(warning, why) - intersectionKC, err := utils.JSONIntersection(existingKC, content) - if err != nil { - return nil, nil, fmt.Errorf("couldn't get intersection of kubelet configs: %w for %s", err, node) - } - kubeletConfigsRole[role] = intersectionKC - } - } else { - kubeletConfigsRole[role] = content - } - } - for role, content := range kubeletConfigsRole { - if role == "" { - continue - } - result[kubeletConfigRolePathPrefix+role] = content - } - return result, warning, nil + return results, warnings, nil } func filter(ctx context.Context, rawobj []byte, filter string) ([]byte, error) { diff --git a/cmd/manager/scap_test.go b/cmd/manager/scap_test.go index b8b99ed01..c2b70e3a2 100644 --- a/cmd/manager/scap_test.go +++ b/cmd/manager/scap_test.go @@ -13,8 +13,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" - "github.com/wI2L/jsondiff" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -488,328 +486,4 @@ var _ = Describe("Testing fetching", func() { }) }) - Context("Test fetching KubeletConfig", func() { - var fetchedResult map[string][]byte - var fetchedInconsistentResult map[string][]byte - var warnings []string - var err error - var roleNodesList map[string][]string - var expectedNodeList map[string][]string - var expectedFiguredResources []utils.ResourcePath - var figuredResources []utils.ResourcePath - var expectedAggregatedResult map[string][]byte - var expectedInconsistentResult map[string][]byte - JustBeforeEach(func() { - - // Fake KubeletConfig - kubeletConfig := []byte(`{ - "enableServer": false, - "staticPodPath": "/etc/kubernetes/manifests", - "syncFrequency": "1m0s", - "fileCheckFrequency": "20s", - "httpCheckFrequency": "20s", - "address": "0.0.0.0", - "port": 10250, - "tlsCipherSuites": [ - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" - ], - "tlsMinVersion": "VersionTLS12", - "rotateCertificates": true, - "serverTLSBootstrap": true, - "authentication": { - "x509": { - "clientCAFile": "/etc/kubernetes/kubelet-ca.crt" - }, - "webhook": { - "enabled": true, - "cacheTTL": "2m0s" - }, - "anonymous": { - "enabled": false - } - }, - "authorization": { - "mode": "Webhook", - "webhook": { - "cacheAuthorizedTTL": "5m0s", - "cacheUnauthorizedTTL": "30s" - } - }, - "registryPullQPS": 5, - "registryBurst": 10, - "eventRecordQPS": 5, - "eventBurst": 10, - "enableDebuggingHandlers": true, - "healthzPort": 10248, - "healthzBindAddress": "127.0.0.1", - "oomScoreAdj": -999, - "clusterDomain": "cluster.local", - "clusterDNS": [ - "172.30.0.10" - ], - "streamingConnectionIdleTimeout": "4h0m0s", - "nodeStatusUpdateFrequency": "10s", - "nodeStatusReportFrequency": "5m0s", - "nodeLeaseDurationSeconds": 40, - "imageMinimumGCAge": "2m0s", - "imageGCHighThresholdPercent": 85, - "imageGCLowThresholdPercent": 80, - "volumeStatsAggPeriod": "1m0s", - "systemCgroups": "/system.slice", - "cgroupRoot": "/", - "cgroupsPerQOS": true, - "cgroupDriver": "systemd", - "cpuManagerPolicy": "none", - "serializeImagePulls": false, - "evictionHard": { - "imagefs.available": "15%", - "memory.available": "100Mi", - "nodefs.available": "10%", - "nodefs.inodesFree": "5%" - }, - "evictionPressureTransitionPeriod": "5m0s", - "enableControllerAttachDetach": true, - "makeIPTablesUtilChains": true, - "iptablesMasqueradeBit": 14, - "iptablesDropBit": 15, - "featureGates": { - "APIPriorityAndFairness": true, - "CSIMigrationAWS": false, - "CSIMigrationAzureFile": false, - "CSIMigrationGCE": false, - "CSIMigrationvSphere": false, - "DownwardAPIHugePages": true, - "PodSecurity": true, - "RotateKubeletServerCertificate": true - }, - "enableSystemLogHandler": true, - "shutdownGracePeriod": "0s", - "shutdownGracePeriodCriticalPods": "0s", - "enableProfilingHandler": true, - "enableDebugFlagsHandler": true, - "seccompDefault": false, - "memoryThrottlingFactor": 0.8, - "registerWithTaints": [ - { - "key": "node-role.kubernetes.io/master", - "effect": "NoSchedule" - } - ], - "registerNode": true, - "kind": "KubeletConfiguration" - } - `) - - kubeletConfigInconsistent := []byte(`{ - "enableSystemLogHandler": true, - "shutdownGracePeriod": "1s", - "shutdownGracePeriodCriticalPods": "3s", - "enableProfilingHandler": true, - "enableDebugFlagsHandler": true, - "seccompDefault": false, - "memoryThrottlingFactor": 0.8, - "registerWithTaints": [ - { - "key": "node-role.kubernetes.io/master", - "effect": "NoSchedule" - } - ], - "registerNode": true, - "kind": "KubeletConfiguration" - } - `) - - kubeletConfigIntersection := []byte(`{ - "enableSystemLogHandler": true, - "enableProfilingHandler": true, - "enableDebugFlagsHandler": true, - "seccompDefault": false, - "memoryThrottlingFactor": 0.8, - "registerWithTaints": [ - { - "key": "node-role.kubernetes.io/master", - "effect": "NoSchedule" - } - ], - "registerNode": true, - "kind": "KubeletConfiguration" - } - `) - - // create fake node list - fakeNodeList := corev1.NodeList{Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-master-0", - Labels: map[string]string{ - "node-role.kubernetes.io/master": "", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-master-1", - Labels: map[string]string{ - "node-role.kubernetes.io/master": "", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-worker-0", - Labels: map[string]string{ - "node-role.kubernetes.io/worker": "", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-worker-1", - Labels: map[string]string{ - "node-role.kubernetes.io/worker": "", - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-worker-2", - Labels: map[string]string{ - "node-role.kubernetes.io/worker": "", - }, - }, - }, - }} - - // create fake KubeletConfig for each node - - scheme := scheme.Scheme - scheme.AddKnownTypes(corev1.SchemeGroupVersion, &fakeNodeList, &fakeNodeList.Items[0]) - - client := fake.NewFakeClientWithScheme(scheme, &fakeNodeList) - fakeClients = resourceFetcherClients{client: client} - - expectedFiguredResources = []utils.ResourcePath{ - { - ObjPath: "/api/v1/nodes/test-node-master-0/proxy/configz", - DumpPath: "/kubeletconfig/master/test-node-master-0", - Filter: `.kubeletconfig|.kind="KubeletConfiguration"|.apiVersion="kubelet.config.k8s.io/v1beta1"`, - }, - { - ObjPath: "/api/v1/nodes/test-node-master-1/proxy/configz", - DumpPath: "/kubeletconfig/master/test-node-master-1", - Filter: `.kubeletconfig|.kind="KubeletConfiguration"|.apiVersion="kubelet.config.k8s.io/v1beta1"`, - }, - { - ObjPath: "/api/v1/nodes/test-node-worker-0/proxy/configz", - DumpPath: "/kubeletconfig/worker/test-node-worker-0", - Filter: `.kubeletconfig|.kind="KubeletConfiguration"|.apiVersion="kubelet.config.k8s.io/v1beta1"`, - }, - { - ObjPath: "/api/v1/nodes/test-node-worker-1/proxy/configz", - DumpPath: "/kubeletconfig/worker/test-node-worker-1", - Filter: `.kubeletconfig|.kind="KubeletConfiguration"|.apiVersion="kubelet.config.k8s.io/v1beta1"`, - }, - { - ObjPath: "/api/v1/nodes/test-node-worker-2/proxy/configz", - DumpPath: "/kubeletconfig/worker/test-node-worker-2", - Filter: `.kubeletconfig|.kind="KubeletConfiguration"|.apiVersion="kubelet.config.k8s.io/v1beta1"`, - }, - } - fetchedResult = make(map[string][]byte) - fetchedInconsistentResult = make(map[string][]byte) - expectedAggregatedResult = make(map[string][]byte) - expectedInconsistentResult = make(map[string][]byte) - for _, resource := range expectedFiguredResources { - fetchedResult[resource.DumpPath] = kubeletConfig - if resource.DumpPath == "/kubeletconfig/master/test-node-master-1" { - fetchedInconsistentResult[resource.DumpPath] = kubeletConfigInconsistent - expectedInconsistentResult[resource.DumpPath] = kubeletConfigInconsistent - } else { - fetchedInconsistentResult[resource.DumpPath] = kubeletConfig - expectedInconsistentResult[resource.DumpPath] = kubeletConfig - } - expectedAggregatedResult[resource.DumpPath] = kubeletConfig - } - expectedAggregatedResult["/kubeletconfig/role/worker"] = kubeletConfig - expectedAggregatedResult["/kubeletconfig/role/master"] = kubeletConfig - - expectedInconsistentResult["/kubeletconfig/role/master"] = kubeletConfigIntersection - expectedInconsistentResult["/kubeletconfig/role/worker"] = kubeletConfig - - expectedNodeList = map[string][]string{ - "master": {"test-node-master-0", "test-node-master-1"}, - "worker": {"test-node-worker-0", "test-node-worker-1", "test-node-worker-2"}, - } - - }) - When("Fetching NodeList", func() { - It("Get Expected Node List", func() { - roleNodesList, err = fetchNodesWithRole(context.Background(), fakeClients.client) - Expect(err).To(BeNil()) - Expect(roleNodesList["master"]).To(ConsistOf(expectedNodeList["master"])) - Expect(roleNodesList["worker"]).To(ConsistOf(expectedNodeList["worker"])) - }) - - It("Get expcted KubeletConfig resource path", func() { - figuredResources = getKubeletConfigResourcePath(roleNodesList) - Expect(compareResourcePaths(figuredResources, expectedFiguredResources)).To(Equal(true)) - }) - }) - When("Test for consistency after fetching api resource", func() { - It("Resource is consistent", func() { - aggregatedResult, warning, err := saveConsistentKubeletResult(fetchedResult, warnings) - Expect(err).To(BeNil()) - Expect(warning).To(BeNil()) - Expect(compareFetchedResults(aggregatedResult, expectedAggregatedResult)).To(Equal(true)) - }) - It("Resource is not consistent", func() { - aggregatedResult, warning, err := saveConsistentKubeletResult(fetchedInconsistentResult, warnings) - Expect(err).To(BeNil()) - Expect(warning[0]).To(ContainSubstring("not consistent")) - Expect(compareFetchedResults(aggregatedResult, expectedInconsistentResult)).To(Equal(true)) - }) - }) - - }) }) - -// compare resourcePath arrays -func compareResourcePaths(a, b []utils.ResourcePath) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if !compareResourcePathsHelper(a[i], b) { - return false - } - } - return true -} - -func compareResourcePathsHelper(a utils.ResourcePath, b []utils.ResourcePath) bool { - for _, v := range b { - if a.ObjPath == v.ObjPath && a.DumpPath == v.DumpPath && a.Filter == v.Filter { - return true - } - } - return false -} - -// compare parseResults -func compareFetchedResults(a, b map[string][]byte) bool { - if len(a) != len(b) { - return false - } - for k, v := range a { - diff, err := jsondiff.CompareJSON(b[k], v) - if err != nil || diff != nil { - return false - } - } - return true -} diff --git a/pkg/controller/complianceremediation/complianceremediation_controller.go b/pkg/controller/complianceremediation/complianceremediation_controller.go index c5ab3aa76..e13269b7c 100644 --- a/pkg/controller/complianceremediation/complianceremediation_controller.go +++ b/pkg/controller/complianceremediation/complianceremediation_controller.go @@ -683,18 +683,11 @@ func (r *ReconcileComplianceRemediation) verifyAndCompleteKC(obj *unstructured.U if err := r.Client.List(context.TODO(), mcfgpools); err != nil { return fmt.Errorf("couldn't list the pools for the remediation: %w", err) } - nodeSelector := map[string]string{} - // If the scan is a platform scan, we need to check if KubeletConfig has the role label set. - if scan.Spec.ScanType == compv1alpha1.ScanTypePlatform { - nodeSelector = utils.GetNodeRoleSelectorFromRemediation(rem) - } else { - nodeSelector = scan.Spec.NodeSelector - } // The scans contain a nodeSelector that ultimately must match a machineConfigPool. The only way we can // ensure it does is by checking if it matches any MachineConfigPool's labels. // See also: https://github.com/openshift/machine-config-operator/blob/master/docs/custom-pools.md - ok, pool := utils.AnyMcfgPoolLabelMatches(nodeSelector, mcfgpools) + ok, pool := utils.AnyMcfgPoolLabelMatches(scan.Spec.NodeSelector, mcfgpools) if !ok { return common.NewNonRetriableCtrlError("not applying remediation that doesn't have a matching MachineconfigPool. Scan: %s", scan.Name) } diff --git a/pkg/controller/compliancesuite/compliancesuite_controller.go b/pkg/controller/compliancesuite/compliancesuite_controller.go index 1ebec009a..609568997 100644 --- a/pkg/controller/compliancesuite/compliancesuite_controller.go +++ b/pkg/controller/compliancesuite/compliancesuite_controller.go @@ -506,16 +506,6 @@ func (r *ReconcileComplianceSuite) reconcileRemediations(suite *compv1alpha1.Com // Only un-pause MachineConfigPools once the remediations have been applied for idx := range affectedMcfgPools { pool := affectedMcfgPools[idx] - // only un-pause if the kubeletconfig is fully rendered for the pool - isRendered, err, diffString := utils.AreKubeletConfigsRendered(pool, r.Client) - if err != nil { - return reconcile.Result{}, err - } - if !isRendered { - logger.Info("Waiting until all kubeletconfigs are rendered before un-pausing", "MachineConfigPool.Name", pool.Name) - logger.Info("KubeletConfig render diff:", "MachineConfigPool.Name", pool.Name, "Diff", diffString) - return reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil - } poolKey := types.NamespacedName{Name: pool.GetName()} // refresh pool reference directly from the API Server if getErr := r.Reader.Get(context.TODO(), poolKey, pool); getErr != nil { @@ -555,7 +545,7 @@ func (r *ReconcileComplianceSuite) applyRemediation(rem compv1alpha1.ComplianceR logger logr.Logger) error { if utils.IsMachineConfig(rem.Spec.Current.Object) || utils.IsKubeletConfig(rem.Spec.Current.Object) { // get affected pool - pool := r.getAffectedMcfgPool(scan, &rem, mcfgpools) + pool := r.getAffectedMcfgPool(scan, mcfgpools) // we only need to operate on pools that are affected if pool != nil { foundPool, poolIsTracked := affectedMcfgPools[pool.Name] @@ -622,16 +612,10 @@ func (r *ReconcileComplianceSuite) applyMcfgRemediationAndPausePool(rem compv1al return nil } -func (r *ReconcileComplianceSuite) getAffectedMcfgPool(scan *compv1alpha1.ComplianceScan, rem *compv1alpha1.ComplianceRemediation, mcfgpools *mcfgv1.MachineConfigPoolList) *mcfgv1.MachineConfigPool { +func (r *ReconcileComplianceSuite) getAffectedMcfgPool(scan *compv1alpha1.ComplianceScan, mcfgpools *mcfgv1.MachineConfigPoolList) *mcfgv1.MachineConfigPool { for i := range mcfgpools.Items { pool := &mcfgpools.Items[i] - nodeSelector := map[string]string{} - if scan.Spec.ScanType == compv1alpha1.ScanTypePlatform { - nodeSelector = utils.GetNodeRoleSelectorFromRemediation(rem) - } else { - nodeSelector = scan.Spec.NodeSelector - } - if utils.McfgPoolLabelMatches(nodeSelector, pool) { + if utils.McfgPoolLabelMatches(scan.Spec.NodeSelector, pool) { return pool } } diff --git a/pkg/controller/compliancesuite/compliancesuite_controller_test.go b/pkg/controller/compliancesuite/compliancesuite_controller_test.go index 601e8f66a..8eec3b8e1 100644 --- a/pkg/controller/compliancesuite/compliancesuite_controller_test.go +++ b/pkg/controller/compliancesuite/compliancesuite_controller_test.go @@ -3,6 +3,7 @@ package compliancesuite import ( "context" "encoding/json" + "github.com/ComplianceAsCode/compliance-operator/pkg/controller/metrics" "github.com/ComplianceAsCode/compliance-operator/pkg/controller/metrics/metricsfakes" @@ -451,25 +452,6 @@ var _ = Describe("ComplianceSuiteController", func() { "something": "0s" } ` - remediationKCMCPayload := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [ - { - "contents": { - "source": "data:text/plain,%7B%0A%20%20%22kind%22%3A%20%22KubeletConfiguration%22%2C%0A%20%20%22apiVersion%22%3A%20%22kubelet.config.k8s.io%2Fv1beta1%22%2C%0A%20%20%22staticPodPath%22%3A%20%22%2Fetc%2Fkubernetes%2Fmanifests%22%2C%0A%20%20%22syncFrequency%22%3A%20%220s%22%2C%0A%20%20%22fileCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22httpCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22tlsCipherSuites%22%3A%20%5B%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256%22%0A%20%20%5D%2C%0A%20%20%22tlsMinVersion%22%3A%20%22VersionTLS12%22%2C%0A%20%20%22rotateCertificates%22%3A%20true%2C%0A%20%20%22serverTLSBootstrap%22%3A%20true%2C%0A%20%20%22authentication%22%3A%20%7B%0A%20%20%20%20%22x509%22%3A%20%7B%0A%20%20%20%20%20%20%22clientCAFile%22%3A%20%22%2Fetc%2Fkubernetes%2Fkubelet-ca.crt%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22anonymous%22%3A%20%7B%0A%20%20%20%20%20%20%22enabled%22%3A%20false%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22authorization%22%3A%20%7B%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheAuthorizedTTL%22%3A%20%220s%22%2C%0A%20%20%20%20%20%20%22cacheUnauthorizedTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22clusterDomain%22%3A%20%22cluster.local%22%2C%0A%20%20%22clusterDNS%22%3A%20%5B%0A%20%20%20%20%22172.30.0.10%22%0A%20%20%5D%2C%0A%20%20%22streamingConnectionIdleTimeout%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusUpdateFrequency%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusReportFrequency%22%3A%20%220s%22%2C%0A%20%20%22imageMinimumGCAge%22%3A%20%220s%22%2C%0A%20%20%22volumeStatsAggPeriod%22%3A%20%220s%22%2C%0A%20%20%22systemCgroups%22%3A%20%22%2Fsystem.slice%22%2C%0A%20%20%22cgroupRoot%22%3A%20%22%2F%22%2C%0A%20%20%22cgroupDriver%22%3A%20%22systemd%22%2C%0A%20%20%22cpuManagerReconcilePeriod%22%3A%20%220s%22%2C%0A%20%20%22runtimeRequestTimeout%22%3A%20%220s%22%2C%0A%20%20%22maxPods%22%3A%20250%2C%0A%20%20%22something%22%3A%20%220s%22%2C%0A%20%20%22kubeAPIBurst%22%3A%20100%2C%0A%20%20%22serializeImagePulls%22%3A%20false%2C%0A%20%20%22evictionPressureTransitionPeriod%22%3A%20%220s%22%2C%0A%20%20%22featureGates%22%3A%20%7B%0A%20%20%20%20%22APIPriorityAndFairness%22%3A%20true%2C%0A%20%20%20%20%22CSIMigrationAWS%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureDisk%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureFile%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationGCE%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationOpenStack%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationvSphere%22%3A%20false%2C%0A%20%20%20%20%22DownwardAPIHugePages%22%3A%20true%2C%0A%20%20%20%20%22LegacyNodeRoleBehavior%22%3A%20false%2C%0A%20%20%20%20%22NodeDisruptionExclusion%22%3A%20true%2C%0A%20%20%20%20%22PodSecurity%22%3A%20true%2C%0A%20%20%20%20%22RotateKubeletServerCertificate%22%3A%20true%2C%0A%20%20%20%20%22ServiceNodeExclusion%22%3A%20true%2C%0A%20%20%20%20%22SupportPodPidsLimit%22%3A%20true%0A%20%20%7D%2C%0A%20%20%22memorySwap%22%3A%20%7B%7D%2C%0A%20%20%22containerLogMaxSize%22%3A%20%2250Mi%22%2C%0A%20%20%22systemReserved%22%3A%20%7B%0A%20%20%20%20%22ephemeral-storage%22%3A%20%221Gi%22%0A%20%20%7D%2C%0A%20%20%22logging%22%3A%20%7B%0A%20%20%20%20%22flushFrequency%22%3A%200%2C%0A%20%20%20%20%22verbosity%22%3A%200%2C%0A%20%20%20%20%22options%22%3A%20%7B%0A%20%20%20%20%20%20%22json%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22infoBufferSize%22%3A%20%220%22%0A%20%20%20%20%20%20%7D%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22shutdownGracePeriod%22%3A%20%220s%22%2C%0A%20%20%22shutdownGracePeriodCriticalPods%22%3A%20%220s%22%0A%7D%0A" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/kubernetes/kubelet.conf" - } - ] - } - }` - BeforeEach(func() { mcp := &mcfgv1.MachineConfigPool{ TypeMeta: metav1.TypeMeta{ @@ -619,25 +601,7 @@ var _ = Describe("ComplianceSuiteController", func() { _, err = reconciler.reconcileRemediations(suite, logger) Expect(err).To(BeNil()) - By("the pool should not be un-paused because the KubeletConfig is not rendered into Machine Config") - err = reconciler.Client.Get(ctx, poolkey, p) - Expect(err).To(BeNil()) - Expect(p.Spec.Paused).To(BeTrue()) - - By("Render KubeLetconfig into Machine Config") - mcCurrent := &mcfgv1.MachineConfig{} - mckey := types.NamespacedName{Name: "99-master-generated-kubelet"} - err = reconciler.Client.Get(ctx, mckey, mcCurrent) - Expect(err).To(BeNil()) - mcCurrent.Spec.Config.Raw = []byte(remediationKCMCPayload) - err = reconciler.Client.Update(ctx, mcCurrent) - Expect(err).To(BeNil()) - - By("Running a second reconcile loop") - _, err = reconciler.reconcileRemediations(suite, logger) - Expect(err).To(BeNil()) - - By("the pool should be un-paused because machine config has been updated with the new kubelet config content") + By("the pool should be un-paused") err = reconciler.Client.Get(ctx, poolkey, p) Expect(err).To(BeNil()) Expect(p.Spec.Paused).To(BeFalse()) @@ -730,24 +694,6 @@ var _ = Describe("ComplianceSuiteController", func() { "something": "0s" } ` - remediationKCMCPayload := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [ - { - "contents": { - "source": "data:text/plain;charset=utf-8;base64,ewogICJraW5kIjogIkt1YmVsZXRDb25maWd1cmF0aW9uIiwKICAiYXBpVmVyc2lvbiI6ICJrdWJlbGV0LmNvbmZpZy5rOHMuaW8vdjFiZXRhMSIsCiAgInN0YXRpY1BvZFBhdGgiOiAiL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cyIsCiAgInN5bmNGcmVxdWVuY3kiOiAiMHMiLAogICJmaWxlQ2hlY2tGcmVxdWVuY3kiOiAiMHMiLAogICJodHRwQ2hlY2tGcmVxdWVuY3kiOiAiMHMiLAogICJ0bHNDaXBoZXJTdWl0ZXMiOiBbCiAgICAiVExTX0VDREhFX0VDRFNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2IiwKICAgICJUTFNfRUNESEVfUlNBX1dJVEhfQUVTXzEyOF9HQ01fU0hBMjU2IiwKICAgICJUTFNfRUNESEVfRUNEU0FfV0lUSF9BRVNfMjU2X0dDTV9TSEEzODQiLAogICAgIlRMU19FQ0RIRV9SU0FfV0lUSF9BRVNfMjU2X0dDTV9TSEEzODQiLAogICAgIlRMU19FQ0RIRV9FQ0RTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1X1NIQTI1NiIsCiAgICAiVExTX0VDREhFX1JTQV9XSVRIX0NIQUNIQTIwX1BPTFkxMzA1X1NIQTI1NiIKICBdLAogICJ0bHNNaW5WZXJzaW9uIjogIlZlcnNpb25UTFMxMiIsCiAgInJvdGF0ZUNlcnRpZmljYXRlcyI6IHRydWUsCiAgInNlcnZlclRMU0Jvb3RzdHJhcCI6IHRydWUsCiAgImF1dGhlbnRpY2F0aW9uIjogewogICAgIng1MDkiOiB7CiAgICAgICJjbGllbnRDQUZpbGUiOiAiL2V0Yy9rdWJlcm5ldGVzL2t1YmVsZXQtY2EuY3J0IgogICAgfSwKICAgICJ3ZWJob29rIjogewogICAgICAiY2FjaGVUVEwiOiAiMHMiCiAgICB9LAogICAgImFub255bW91cyI6IHsKICAgICAgImVuYWJsZWQiOiBmYWxzZQogICAgfQogIH0sCiAgImF1dGhvcml6YXRpb24iOiB7CiAgICAid2ViaG9vayI6IHsKICAgICAgImNhY2hlQXV0aG9yaXplZFRUTCI6ICIwcyIsCiAgICAgICJjYWNoZVVuYXV0aG9yaXplZFRUTCI6ICIwcyIKICAgIH0KICB9LAogICJjbHVzdGVyRG9tYWluIjogImNsdXN0ZXIubG9jYWwiLAogICJjbHVzdGVyRE5TIjogWwogICAgIjE3Mi4zMC4wLjEwIgogIF0sCiAgInN0cmVhbWluZ0Nvbm5lY3Rpb25JZGxlVGltZW91dCI6ICIwcyIsCiAgIm5vZGVTdGF0dXNVcGRhdGVGcmVxdWVuY3kiOiAiMHMiLAogICJub2RlU3RhdHVzUmVwb3J0RnJlcXVlbmN5IjogIjBzIiwKICAiaW1hZ2VNaW5pbXVtR0NBZ2UiOiAiMHMiLAogICJ2b2x1bWVTdGF0c0FnZ1BlcmlvZCI6ICIwcyIsCiAgInN5c3RlbUNncm91cHMiOiAiL3N5c3RlbS5zbGljZSIsCiAgImNncm91cFJvb3QiOiAiLyIsCiAgImNncm91cERyaXZlciI6ICJzeXN0ZW1kIiwKICAiY3B1TWFuYWdlclJlY29uY2lsZVBlcmlvZCI6ICIwcyIsCiAgInJ1bnRpbWVSZXF1ZXN0VGltZW91dCI6ICIwcyIsCiAgIm1heFBvZHMiOiAyNTAsCiAgInNvbWV0aGluZyI6ICIwcyIsCiAgImt1YmVBUElCdXJzdCI6IDEwMCwKICAic2VyaWFsaXplSW1hZ2VQdWxscyI6IGZhbHNlLAogICJldmljdGlvblByZXNzdXJlVHJhbnNpdGlvblBlcmlvZCI6ICIwcyIsCiAgImZlYXR1cmVHYXRlcyI6IHsKICAgICJBUElQcmlvcml0eUFuZEZhaXJuZXNzIjogdHJ1ZSwKICAgICJDU0lNaWdyYXRpb25BV1MiOiBmYWxzZSwKICAgICJDU0lNaWdyYXRpb25BenVyZURpc2siOiBmYWxzZSwKICAgICJDU0lNaWdyYXRpb25BenVyZUZpbGUiOiBmYWxzZSwKICAgICJDU0lNaWdyYXRpb25HQ0UiOiBmYWxzZSwKICAgICJDU0lNaWdyYXRpb25PcGVuU3RhY2siOiBmYWxzZSwKICAgICJDU0lNaWdyYXRpb252U3BoZXJlIjogZmFsc2UsCiAgICAiRG93bndhcmRBUElIdWdlUGFnZXMiOiB0cnVlLAogICAgIkxlZ2FjeU5vZGVSb2xlQmVoYXZpb3IiOiBmYWxzZSwKICAgICJOb2RlRGlzcnVwdGlvbkV4Y2x1c2lvbiI6IHRydWUsCiAgICAiUG9kU2VjdXJpdHkiOiB0cnVlLAogICAgIlJvdGF0ZUt1YmVsZXRTZXJ2ZXJDZXJ0aWZpY2F0ZSI6IHRydWUsCiAgICAiU2VydmljZU5vZGVFeGNsdXNpb24iOiB0cnVlLAogICAgIlN1cHBvcnRQb2RQaWRzTGltaXQiOiB0cnVlCiAgfSwKICAibWVtb3J5U3dhcCI6IHt9LAogICJjb250YWluZXJMb2dNYXhTaXplIjogIjUwTWkiLAogICJzeXN0ZW1SZXNlcnZlZCI6IHsKICAgICJlcGhlbWVyYWwtc3RvcmFnZSI6ICIxR2kiCiAgfSwKICAibG9nZ2luZyI6IHsKICAgICJmbHVzaEZyZXF1ZW5jeSI6IDAsCiAgICAidmVyYm9zaXR5IjogMCwKICAgICJvcHRpb25zIjogewogICAgICAianNvbiI6IHsKICAgICAgICAiaW5mb0J1ZmZlclNpemUiOiAiMCIKICAgICAgfQogICAgfQogIH0sCiAgInNodXRkb3duR3JhY2VQZXJpb2QiOiAiMHMiLAogICJzaHV0ZG93bkdyYWNlUGVyaW9kQ3JpdGljYWxQb2RzIjogIjBzIgp9Cg==" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/kubernetes/kubelet.conf" - } - ] - } - }` BeforeEach(func() { mcp := &mcfgv1.MachineConfigPool{ @@ -898,29 +844,10 @@ var _ = Describe("ComplianceSuiteController", func() { _, err = reconciler.reconcileRemediations(suite, logger) Expect(err).To(BeNil()) - By("the pool should not be un-paused because the KubeletConfig is not rendered into Machine Config") - err = reconciler.Client.Get(ctx, poolkey, p) - Expect(err).To(BeNil()) - Expect(p.Spec.Paused).To(BeTrue()) - - By("Render KubeLetconfig into Machine Config") - mcCurrent := &mcfgv1.MachineConfig{} - mckey := types.NamespacedName{Name: "99-master-generated-kubelet"} - err = reconciler.Client.Get(ctx, mckey, mcCurrent) - Expect(err).To(BeNil()) - mcCurrent.Spec.Config.Raw = []byte(remediationKCMCPayload) - err = reconciler.Client.Update(ctx, mcCurrent) - Expect(err).To(BeNil()) - - By("Running a second reconcile loop") - _, err = reconciler.reconcileRemediations(suite, logger) - Expect(err).To(BeNil()) - - By("the pool should be un-paused because machine config has been updated with the new kubelet config content") + By("the pool should be un-paused") err = reconciler.Client.Get(ctx, poolkey, p) Expect(err).To(BeNil()) Expect(p.Spec.Paused).To(BeFalse()) - s := &compv1alpha1.ComplianceRemediation{} key := types.NamespacedName{Name: remediationName, Namespace: namespace} reconciler.Client.Get(ctx, key, s) diff --git a/pkg/utils/compare_json.go b/pkg/utils/compare_json.go deleted file mode 100644 index b8122f6dd..000000000 --- a/pkg/utils/compare_json.go +++ /dev/null @@ -1,167 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - "reflect" -) - -// JSONDiff represents the whole diff -type JSONDiff struct { - Rows []JSONDiffRow -} - -// JSONDiffRow represents a single non-existent subset item -type JSONDiffRow struct { - Key string - Expected interface{} - Got interface{} -} - -// JSONIsSubset checks if a is a subset json of b -func JSONIsSubset(a, b []byte) (bool, *JSONDiff, error) { - if a == nil || b == nil { - return false, nil, fmt.Errorf("nil json") - } - return jsonIsSubsetR(a, b, nil, nil) -} - -// find intersection of two json -func JSONIntersection(a, b []byte) ([]byte, error) { - var ai map[string]interface{} - if err := json.Unmarshal([]byte(a), &ai); err != nil { - return nil, err - } - var bi map[string]interface{} - if err := json.Unmarshal([]byte(b), &bi); err != nil { - return nil, err - } - dst := procMap(ai, bi) - return json.Marshal(dst) -} - -func procMap(a, b map[string]interface{}) (dst map[string]interface{}) { - dst = map[string]interface{}{} - for k, v := range a { - if innerMap, ok := v.(map[string]interface{}); ok { - if b[k] != nil { - dst[k] = procMap(innerMap, b[k].(map[string]interface{})) - } - } else { - if b[k] != nil { - if reflect.DeepEqual(v, b[k]) { - dst[k] = b[k] - } - } - } - } - return dst -} - -func jsonIsSubsetR(a, b []byte, diff *JSONDiff, prefix interface{}) (bool, *JSONDiff, error) { - // Initialize - if diff == nil { - diff = &JSONDiff{} - } - if diff.Rows == nil { - diff.Rows = make([]JSONDiffRow, 0) - } - - // Prefix for keeping around more info (path of the diffs) - sprefix := "" - if prefix != nil { - sprefix = prefix.(string) - } - - // Unmarshal both interfaces. If something fails here, we have nothing to do - // jai: JSON A Interface - // jbi: JSON B Interface - var jai, jbi interface{} - if err := json.Unmarshal(a, &jai); err != nil { - return false, nil, err - } - if err := json.Unmarshal(b, &jbi); err != nil { - return false, nil, err - } - - // Switch JSON (map) or array of JSON (array of interface) - // ja: JSON A (map or []interface) - // jb: JSON B (map or []interface) - switch ja := jai.(type) { - case map[string]interface{}: - // Cast B to same type as A - // TODO: Add a check to see if this fails - jb := jbi.(map[string]interface{}) - - // Iterate all keys of ja and check if each is present - // and equal to the same key in jb - for k, vu := range ja { - switch vu.(type) { - // A primitive value such as string or number will be compared natively - default: - // Check if we have the key at all - if val, ok := jb[k]; ok { - // Check if the key matches if we have it - if vu != val { - diff.Rows = append(diff.Rows, JSONDiffRow{ - Key: fmt.Sprintf("%s/%s", sprefix, k), Expected: vu, Got: jb[k]}) - } - } else { - // We didn't find a key we wanted - diff.Rows = append(diff.Rows, JSONDiffRow{ - Key: fmt.Sprintf("%s/%s", sprefix, k), Expected: vu, Got: "NOT FOUND"}) - } - - // Compare nested json by calling this function recursively - case map[string]interface{}, []interface{}: - sja, err := json.Marshal(vu) - if err != nil { - return false, nil, err - } - sjb, err := json.Marshal(jb[k]) - if err != nil { - return false, nil, err - } - _, _, err = jsonIsSubsetR(sja, sjb, diff, fmt.Sprintf("%s/%s", sprefix, k)) - if err != nil { - return false, nil, err - } - } - } - - // Compare arrays - case []interface{}: - // Case jbi to an array as well - // TODO: Add a check to see if this fails - jb := jbi.([]interface{}) - - // Check if length is equal first - if len(jb) != len(ja) { - // Length not equal so that is not good - diff.Rows = append(diff.Rows, JSONDiffRow{ - Key: fmt.Sprintf("%s", sprefix), Expected: fmt.Sprintf("LEN=%d", len(ja)), Got: fmt.Sprintf("LEN=%d", len(jb))}) - } else { - // Recurse for each object inside - for i, x := range ja { - sja, err := json.Marshal(x) - if err != nil { - return false, nil, err - } - sjb, err := json.Marshal(jb[i]) - if err != nil { - return false, nil, err - } - _, _, err = jsonIsSubsetR(sja, sjb, diff, fmt.Sprintf("%s[%d]", sprefix, i)) - if err != nil { - return false, nil, err - } - } - } - // Compare primitive types directly - default: - return jai == jbi, diff, nil - } - - // No diff means all keys in A were found and equal in B - return diff == nil || len(diff.Rows) == 0, diff, nil -} diff --git a/pkg/utils/nodeutils.go b/pkg/utils/nodeutils.go index de03d1e54..eeb861749 100644 --- a/pkg/utils/nodeutils.go +++ b/pkg/utils/nodeutils.go @@ -17,16 +17,13 @@ package utils import ( "context" - "encoding/base64" "encoding/json" "fmt" - "net/url" "reflect" "strconv" "strings" compliancev1alpha1 "github.com/ComplianceAsCode/compliance-operator/pkg/apis/compliance/v1alpha1" - "github.com/PaesslerAG/jsonpath" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "k8s.io/apimachinery/pkg/types" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -161,103 +158,6 @@ func GetScanType(annotations map[string]string) compliancev1alpha1.ComplianceSca return compliancev1alpha1.ScanTypePlatform } -func AreKubeletConfigsRendered(pool *mcfgv1.MachineConfigPool, client runtimeclient.Client) (bool, error, string) { - // find out if pool is using a custom kubelet config - isUsingKC, currentKCMCName, err := IsMcfgPoolUsingKC(pool) - if err != nil { - return false, fmt.Errorf("failed to check if pool %s is using a custom kubelet config: %w", pool.Name, err), "" - } - if !isUsingKC || currentKCMCName == "" { - return true, nil, "" - } - // if the pool is using a custom kubelet config, check if the kubelet config is rendered - kcmcfg := &mcfgv1.MachineConfig{} - err = client.Get(context.TODO(), types.NamespacedName{Name: currentKCMCName}, kcmcfg) - if err != nil { - return false, fmt.Errorf("failed to get machine config %s: %w", currentKCMCName, err), "" - } - - kc, err := GetKCFromMC(kcmcfg, client) - if err != nil { - return false, fmt.Errorf("failed to get kubelet config using machine config name %s: %w", currentKCMCName, err), "" - } - - return IsKCSubsetOfMC(kc, kcmcfg) -} - -// IsKCSubsetOfMC determines if the KubeletConfig is a subset of the MachineConfig -func IsKCSubsetOfMC(kc *mcfgv1.KubeletConfig, mc *mcfgv1.MachineConfig) (bool, error, string) { - if kc == nil { - return false, fmt.Errorf("kubelet config is nil"), "" - } - if mc == nil { - return false, fmt.Errorf("machine config is nil"), "" - } - if kc.Spec.KubeletConfig == nil { - return false, fmt.Errorf("kubelet config spec is nil, please check if KubeletConfig object has been used correctly"), "" - } - - var obj interface{} - if err := json.Unmarshal(mc.Spec.Config.Raw, &obj); err != nil { - return false, fmt.Errorf("failed to unmarshal machine config %s: %w", mc.Name, err), "" - } - - // Filter to kubelet.conf file as other files (e.g. /etc/node-sizing-enabled.env) can exist. - encodedKC, err := jsonpath.Get(`$.storage.files[?(@.path=="/etc/kubernetes/kubelet.conf")].contents.source`, obj) - if err != nil { - return false, fmt.Errorf("failed to get encoded kubelet config from machine config %s: %w", mc.Name, err), "" - } - encodedKCSlice := encodedKC.([]interface{}) - if len(encodedKCSlice) == 0 { - return false, fmt.Errorf("encoded kubeletconfig %s is missing", mc.Name), "" - } - encodedKCStr := encodedKCSlice[0].(string) - if encodedKCStr == "" { - return false, fmt.Errorf("encoded kubeletconfig %s is empty", mc.Name), "" - } - - // Decode kubelet.conf file content - var encodedKCStrTrimmed string - var decodedKC []byte - if strings.HasPrefix(encodedKCStr, mcBase64PayloadPrefix) { - encodedKCStrTrimmed = strings.TrimPrefix(encodedKCStr, mcBase64PayloadPrefix) - decodedKC, err = base64.StdEncoding.DecodeString(encodedKCStrTrimmed) - if err != nil { - return false, fmt.Errorf("failed to decode base64 encoded kubeletconfig %s: %w", mc.Name, err), "" - } - } else if strings.HasPrefix(encodedKCStr, mcPayloadPrefix) { - encodedKCStrTrimmed = strings.TrimPrefix(encodedKCStr, mcPayloadPrefix) - decodedStr, err := url.PathUnescape(encodedKCStrTrimmed) - decodedKC = []byte(decodedStr) - if err != nil { - return false, fmt.Errorf("failed to decode urlencoded kubeletconfig %s: %w", mc.Name, err), "" - } - } else { - return false, fmt.Errorf("encoded kubeletconfig %s does not contain encoding prefix", mc.Name), "" - } - - // remove node sizing related fields from kubelet config - filteredKC, err := removeNodeSizingEnvParams(kc.Spec.KubeletConfig.Raw) - if err != nil { - return false, fmt.Errorf("failed to remove node sizing related fields from decoded kubelet config: %w", err), "" - } - - // Check if KubeletConfig is a subset of the MachineConfig - isSubset, diff, err := JSONIsSubset(filteredKC, decodedKC) - if err != nil { - return false, fmt.Errorf("failed to check if kubeletconfig %s is subset of rendered MC %s: %w", kc.Name, mc.Name, err), "" - } - - if isSubset { - return true, nil, "" - } - diffData := make([][]string, 0) - for _, r := range diff.Rows { - diffData = append(diffData, []string{fmt.Sprintf("Path: %s", r.Key), fmt.Sprintf("Expected: %s", r.Expected), fmt.Sprintf("Got: %s", r.Got)}) - } - return false, nil, fmt.Sprintf("kubeletconfig %s is not subset of rendered MC %s, diff: %v", kc.Name, mc.Name, diffData) -} - func GetKCFromMC(mc *mcfgv1.MachineConfig, client runtimeclient.Client) (*mcfgv1.KubeletConfig, error) { if mc == nil { return nil, fmt.Errorf("machine config is nil") diff --git a/pkg/utils/nodeutils_test.go b/pkg/utils/nodeutils_test.go index 2d34349cd..4345bcbc8 100644 --- a/pkg/utils/nodeutils_test.go +++ b/pkg/utils/nodeutils_test.go @@ -7,7 +7,6 @@ import ( mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "github.com/ComplianceAsCode/compliance-operator/pkg/utils" ) @@ -209,303 +208,6 @@ var _ = Describe("Nodeutils", func() { }) }) - When("Testing IsKCSubsetOfMC", func() { - defaultKCPayload := ` - { - "streamingConnectionIdleTimeout": "0s", - "something": "0s", - "systemReserved": { - "cpu": "0s", - "memory": "0s" - }, - "autoSizingReserved": true - } - ` - testKubeletConfigNil := func() *mcfgv1.KubeletConfig { - return &mcfgv1.KubeletConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "KubeletConfig", - APIVersion: "machineconfiguration.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "kubelet-config-compliance-operator", - }, - Spec: mcfgv1.KubeletConfigSpec{}, - } - } - testKubeletConfig := func(kcPayload string) *mcfgv1.KubeletConfig { - return &mcfgv1.KubeletConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "KubeletConfig", - APIVersion: "machineconfiguration.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "kubelet-config-compliance-operator", - }, - Spec: mcfgv1.KubeletConfigSpec{ - KubeletConfig: &runtime.RawExtension{ - Raw: []byte(kcPayload), - }, - }, - } - } - - testMachineConfig := func(renderdKC string) *mcfgv1.MachineConfig { - return &mcfgv1.MachineConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "MachineConfig", - APIVersion: "machineconfiguration.openshift.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "99-master-generated-kubelet", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "machineconfiguration.openshift.io/v1", - Kind: "KubeletConfig", - Name: "kubelet-config-compliance-operator", - UID: "12345", - }, - }, - }, - Spec: mcfgv1.MachineConfigSpec{ - Config: runtime.RawExtension{ - Raw: []byte(renderdKC), - }, - }, - } - } - - Context("KubeletConfig is a subset of MachineConfig", func() { - renderdKC := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [ - { - "contents": { - "source": "data:text/plain,%7B%0A%20%20%22kind%22%3A%20%22KubeletConfiguration%22%2C%0A%20%20%22apiVersion%22%3A%20%22kubelet.config.k8s.io%2Fv1beta1%22%2C%0A%20%20%22staticPodPath%22%3A%20%22%2Fetc%2Fkubernetes%2Fmanifests%22%2C%0A%20%20%22syncFrequency%22%3A%20%220s%22%2C%0A%20%20%22fileCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22httpCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22tlsCipherSuites%22%3A%20%5B%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256%22%0A%20%20%5D%2C%0A%20%20%22tlsMinVersion%22%3A%20%22VersionTLS12%22%2C%0A%20%20%22rotateCertificates%22%3A%20true%2C%0A%20%20%22serverTLSBootstrap%22%3A%20true%2C%0A%20%20%22authentication%22%3A%20%7B%0A%20%20%20%20%22x509%22%3A%20%7B%0A%20%20%20%20%20%20%22clientCAFile%22%3A%20%22%2Fetc%2Fkubernetes%2Fkubelet-ca.crt%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22anonymous%22%3A%20%7B%0A%20%20%20%20%20%20%22enabled%22%3A%20false%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22authorization%22%3A%20%7B%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheAuthorizedTTL%22%3A%20%220s%22%2C%0A%20%20%20%20%20%20%22cacheUnauthorizedTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22clusterDomain%22%3A%20%22cluster.local%22%2C%0A%20%20%22clusterDNS%22%3A%20%5B%0A%20%20%20%20%22172.30.0.10%22%0A%20%20%5D%2C%0A%20%20%22streamingConnectionIdleTimeout%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusUpdateFrequency%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusReportFrequency%22%3A%20%220s%22%2C%0A%20%20%22imageMinimumGCAge%22%3A%20%220s%22%2C%0A%20%20%22volumeStatsAggPeriod%22%3A%20%220s%22%2C%0A%20%20%22systemCgroups%22%3A%20%22%2Fsystem.slice%22%2C%0A%20%20%22cgroupRoot%22%3A%20%22%2F%22%2C%0A%20%20%22cgroupDriver%22%3A%20%22systemd%22%2C%0A%20%20%22cpuManagerReconcilePeriod%22%3A%20%220s%22%2C%0A%20%20%22runtimeRequestTimeout%22%3A%20%220s%22%2C%0A%20%20%22maxPods%22%3A%20250%2C%0A%20%20%22something%22%3A%20%220s%22%2C%0A%20%20%22kubeAPIBurst%22%3A%20100%2C%0A%20%20%22serializeImagePulls%22%3A%20false%2C%0A%20%20%22evictionPressureTransitionPeriod%22%3A%20%220s%22%2C%0A%20%20%22featureGates%22%3A%20%7B%0A%20%20%20%20%22APIPriorityAndFairness%22%3A%20true%2C%0A%20%20%20%20%22CSIMigrationAWS%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureDisk%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureFile%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationGCE%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationOpenStack%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationvSphere%22%3A%20false%2C%0A%20%20%20%20%22DownwardAPIHugePages%22%3A%20true%2C%0A%20%20%20%20%22LegacyNodeRoleBehavior%22%3A%20false%2C%0A%20%20%20%20%22NodeDisruptionExclusion%22%3A%20true%2C%0A%20%20%20%20%22PodSecurity%22%3A%20true%2C%0A%20%20%20%20%22RotateKubeletServerCertificate%22%3A%20true%2C%0A%20%20%20%20%22ServiceNodeExclusion%22%3A%20true%2C%0A%20%20%20%20%22SupportPodPidsLimit%22%3A%20true%0A%20%20%7D%2C%0A%20%20%22memorySwap%22%3A%20%7B%7D%2C%0A%20%20%22containerLogMaxSize%22%3A%20%2250Mi%22%2C%0A%20%20%22systemReserved%22%3A%20%7B%0A%20%20%20%20%22ephemeral-storage%22%3A%20%221Gi%22%0A%20%20%7D%2C%0A%20%20%22logging%22%3A%20%7B%0A%20%20%20%20%22flushFrequency%22%3A%200%2C%0A%20%20%20%20%22verbosity%22%3A%200%2C%0A%20%20%20%20%22options%22%3A%20%7B%0A%20%20%20%20%20%20%22json%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22infoBufferSize%22%3A%20%220%22%0A%20%20%20%20%20%20%7D%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22shutdownGracePeriod%22%3A%20%220s%22%2C%0A%20%20%22shutdownGracePeriodCriticalPods%22%3A%20%220s%22%0A%7D%0A" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/kubernetes/kubelet.conf" - } - ] - } - }` - kc := testKubeletConfig(defaultKCPayload) - mc := testMachineConfig(renderdKC) - - It("It should evaluate as true", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(BeNil()) - Expect(isSubset).To(BeTrue()) - Expect(diffString).To(BeEmpty()) - }) - }) - - Context("KubeletConfig is a subset of MachineConfig with multiple files", func() { - renderdKC := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [ - { - "contents": { - "source": "data:text/plain,NODE_SIZING_ENABLED%3Dtrue%0ASYSTEM_RESERVED_MEMORY%3D1Gi%0ASYSTEM_RESERVED_CPU%3D500m%0A" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/node-sizing-enabled.env" - }, - { - "contents": { - "source": "data:text/plain,%7B%0A%20%20%22kind%22%3A%20%22KubeletConfiguration%22%2C%0A%20%20%22apiVersion%22%3A%20%22kubelet.config.k8s.io%2Fv1beta1%22%2C%0A%20%20%22staticPodPath%22%3A%20%22%2Fetc%2Fkubernetes%2Fmanifests%22%2C%0A%20%20%22syncFrequency%22%3A%20%220s%22%2C%0A%20%20%22fileCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22httpCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22tlsCipherSuites%22%3A%20%5B%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256%22%0A%20%20%5D%2C%0A%20%20%22tlsMinVersion%22%3A%20%22VersionTLS12%22%2C%0A%20%20%22rotateCertificates%22%3A%20true%2C%0A%20%20%22serverTLSBootstrap%22%3A%20true%2C%0A%20%20%22authentication%22%3A%20%7B%0A%20%20%20%20%22x509%22%3A%20%7B%0A%20%20%20%20%20%20%22clientCAFile%22%3A%20%22%2Fetc%2Fkubernetes%2Fkubelet-ca.crt%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22anonymous%22%3A%20%7B%0A%20%20%20%20%20%20%22enabled%22%3A%20false%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22authorization%22%3A%20%7B%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheAuthorizedTTL%22%3A%20%220s%22%2C%0A%20%20%20%20%20%20%22cacheUnauthorizedTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22clusterDomain%22%3A%20%22cluster.local%22%2C%0A%20%20%22clusterDNS%22%3A%20%5B%0A%20%20%20%20%22172.30.0.10%22%0A%20%20%5D%2C%0A%20%20%22streamingConnectionIdleTimeout%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusUpdateFrequency%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusReportFrequency%22%3A%20%220s%22%2C%0A%20%20%22imageMinimumGCAge%22%3A%20%220s%22%2C%0A%20%20%22volumeStatsAggPeriod%22%3A%20%220s%22%2C%0A%20%20%22systemCgroups%22%3A%20%22%2Fsystem.slice%22%2C%0A%20%20%22cgroupRoot%22%3A%20%22%2F%22%2C%0A%20%20%22cgroupDriver%22%3A%20%22systemd%22%2C%0A%20%20%22cpuManagerReconcilePeriod%22%3A%20%220s%22%2C%0A%20%20%22runtimeRequestTimeout%22%3A%20%220s%22%2C%0A%20%20%22maxPods%22%3A%20250%2C%0A%20%20%22something%22%3A%20%220s%22%2C%0A%20%20%22kubeAPIBurst%22%3A%20100%2C%0A%20%20%22serializeImagePulls%22%3A%20false%2C%0A%20%20%22evictionPressureTransitionPeriod%22%3A%20%220s%22%2C%0A%20%20%22featureGates%22%3A%20%7B%0A%20%20%20%20%22APIPriorityAndFairness%22%3A%20true%2C%0A%20%20%20%20%22CSIMigrationAWS%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureDisk%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureFile%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationGCE%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationOpenStack%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationvSphere%22%3A%20false%2C%0A%20%20%20%20%22DownwardAPIHugePages%22%3A%20true%2C%0A%20%20%20%20%22LegacyNodeRoleBehavior%22%3A%20false%2C%0A%20%20%20%20%22NodeDisruptionExclusion%22%3A%20true%2C%0A%20%20%20%20%22PodSecurity%22%3A%20true%2C%0A%20%20%20%20%22RotateKubeletServerCertificate%22%3A%20true%2C%0A%20%20%20%20%22ServiceNodeExclusion%22%3A%20true%2C%0A%20%20%20%20%22SupportPodPidsLimit%22%3A%20true%0A%20%20%7D%2C%0A%20%20%22memorySwap%22%3A%20%7B%7D%2C%0A%20%20%22containerLogMaxSize%22%3A%20%2250Mi%22%2C%0A%20%20%22systemReserved%22%3A%20%7B%0A%20%20%20%20%22ephemeral-storage%22%3A%20%221Gi%22%0A%20%20%7D%2C%0A%20%20%22logging%22%3A%20%7B%0A%20%20%20%20%22flushFrequency%22%3A%200%2C%0A%20%20%20%20%22verbosity%22%3A%200%2C%0A%20%20%20%20%22options%22%3A%20%7B%0A%20%20%20%20%20%20%22json%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22infoBufferSize%22%3A%20%220%22%0A%20%20%20%20%20%20%7D%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22shutdownGracePeriod%22%3A%20%220s%22%2C%0A%20%20%22shutdownGracePeriodCriticalPods%22%3A%20%220s%22%0A%7D%0A" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/kubernetes/kubelet.conf" - } - ] - } - }` - kc := testKubeletConfig(defaultKCPayload) - mc := testMachineConfig(renderdKC) - - It("It should evaluate as true", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(BeNil()) - Expect(isSubset).To(BeTrue()) - Expect(diffString).To(BeEmpty()) - }) - }) - - Context("KubeletConfig is not a subset of MachineConfig", func() { - renderdKC := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [ - { - "contents": { - "source": "data:text/plain,%7B%0A%20%20%22kind%22%3A%20%22KubeletConfiguration%22%2C%0A%20%20%22apiVersion%22%3A%20%22kubelet.config.k8s.io%2Fv1beta1%22%2C%0A%20%20%22staticPodPath%22%3A%20%22%2Fetc%2Fkubernetes%2Fmanifests%22%2C%0A%20%20%22syncFrequency%22%3A%20%220s%22%2C%0A%20%20%22fileCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22httpCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22tlsCipherSuites%22%3A%20%5B%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256%22%0A%20%20%5D%2C%0A%20%20%22tlsMinVersion%22%3A%20%22VersionTLS12%22%2C%0A%20%20%22rotateCertificates%22%3A%20true%2C%0A%20%20%22serverTLSBootstrap%22%3A%20true%2C%0A%20%20%22authentication%22%3A%20%7B%0A%20%20%20%20%22x509%22%3A%20%7B%0A%20%20%20%20%20%20%22clientCAFile%22%3A%20%22%2Fetc%2Fkubernetes%2Fkubelet-ca.crt%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22anonymous%22%3A%20%7B%0A%20%20%20%20%20%20%22enabled%22%3A%20false%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22authorization%22%3A%20%7B%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheAuthorizedTTL%22%3A%20%220s%22%2C%0A%20%20%20%20%20%20%22cacheUnauthorizedTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22clusterDomain%22%3A%20%22cluster.local%22%2C%0A%20%20%22clusterDNS%22%3A%20%5B%0A%20%20%20%20%22172.30.0.10%22%0A%20%20%5D%2C%0A%20%20%22streamingConnectionIdleTimeout%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusUpdateFrequency%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusReportFrequency%22%3A%20%220s%22%2C%0A%20%20%22imageMinimumGCAge%22%3A%20%220s%22%2C%0A%20%20%22volumeStatsAggPeriod%22%3A%20%220s%22%2C%0A%20%20%22systemCgroups%22%3A%20%22%2Fsystem.slice%22%2C%0A%20%20%22cgroupRoot%22%3A%20%22%2F%22%2C%0A%20%20%22cgroupDriver%22%3A%20%22systemd%22%2C%0A%20%20%22cpuManagerReconcilePeriod%22%3A%20%220s%22%2C%0A%20%20%22runtimeRequestTimeout%22%3A%20%220s%22%2C%0A%20%20%22maxPods%22%3A%20250%2C%0A%20%20%22kubeAPIQPS%22%3A%2050%2C%0A%20%20%22kubeAPIBurst%22%3A%20100%2C%0A%20%20%22serializeImagePulls%22%3A%20false%2C%0A%20%20%22evictionPressureTransitionPeriod%22%3A%20%220s%22%2C%0A%20%20%22featureGates%22%3A%20%7B%0A%20%20%20%20%22APIPriorityAndFairness%22%3A%20true%2C%0A%20%20%20%20%22CSIMigrationAWS%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureDisk%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureFile%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationGCE%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationOpenStack%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationvSphere%22%3A%20false%2C%0A%20%20%20%20%22DownwardAPIHugePages%22%3A%20true%2C%0A%20%20%20%20%22LegacyNodeRoleBehavior%22%3A%20false%2C%0A%20%20%20%20%22NodeDisruptionExclusion%22%3A%20true%2C%0A%20%20%20%20%22PodSecurity%22%3A%20true%2C%0A%20%20%20%20%22RotateKubeletServerCertificate%22%3A%20true%2C%0A%20%20%20%20%22ServiceNodeExclusion%22%3A%20true%2C%0A%20%20%20%20%22SupportPodPidsLimit%22%3A%20true%0A%20%20%7D%2C%0A%20%20%22memorySwap%22%3A%20%7B%7D%2C%0A%20%20%22containerLogMaxSize%22%3A%20%2250Mi%22%2C%0A%20%20%22systemReserved%22%3A%20%7B%0A%20%20%20%20%22ephemeral-storage%22%3A%20%221Gi%22%0A%20%20%7D%2C%0A%20%20%22logging%22%3A%20%7B%0A%20%20%20%20%22flushFrequency%22%3A%200%2C%0A%20%20%20%20%22verbosity%22%3A%200%2C%0A%20%20%20%20%22options%22%3A%20%7B%0A%20%20%20%20%20%20%22json%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22infoBufferSize%22%3A%20%220%22%0A%20%20%20%20%20%20%7D%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22shutdownGracePeriod%22%3A%20%220s%22%2C%0A%20%20%22shutdownGracePeriodCriticalPods%22%3A%20%220s%22%0A%7D%0A" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/kubernetes/kubelet.conf" - } - ] - } - }` - kc := testKubeletConfig(defaultKCPayload) - mc := testMachineConfig(renderdKC) - expectedDiffString := "kubeletconfig kubelet-config-compliance-operator is not subset of rendered MC 99-master-generated-kubelet, diff: [[Path: /something Expected: 0s Got: NOT FOUND]]" - - It("It should evaluate as false", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(BeNil()) - Expect(isSubset).To(BeFalse()) - Expect(diffString).To(Equal(expectedDiffString)) - }) - }) - - Context("KubeletConfig is nil", func() { - renderdKC := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [ - { - "contents": { - "source": "data:text/plain,NODE_SIZING_ENABLED%3Dtrue%0ASYSTEM_RESERVED_MEMORY%3D1Gi%0ASYSTEM_RESERVED_CPU%3D500m%0A" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/node-sizing-enabled.env" - }, - { - "contents": { - "source": "data:text/plain,%7B%0A%20%20%22kind%22%3A%20%22KubeletConfiguration%22%2C%0A%20%20%22apiVersion%22%3A%20%22kubelet.config.k8s.io%2Fv1beta1%22%2C%0A%20%20%22staticPodPath%22%3A%20%22%2Fetc%2Fkubernetes%2Fmanifests%22%2C%0A%20%20%22syncFrequency%22%3A%20%220s%22%2C%0A%20%20%22fileCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22httpCheckFrequency%22%3A%20%220s%22%2C%0A%20%20%22tlsCipherSuites%22%3A%20%5B%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%22%2C%0A%20%20%20%20%22TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256%22%2C%0A%20%20%20%20%22TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256%22%0A%20%20%5D%2C%0A%20%20%22tlsMinVersion%22%3A%20%22VersionTLS12%22%2C%0A%20%20%22rotateCertificates%22%3A%20true%2C%0A%20%20%22serverTLSBootstrap%22%3A%20true%2C%0A%20%20%22authentication%22%3A%20%7B%0A%20%20%20%20%22x509%22%3A%20%7B%0A%20%20%20%20%20%20%22clientCAFile%22%3A%20%22%2Fetc%2Fkubernetes%2Fkubelet-ca.crt%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22anonymous%22%3A%20%7B%0A%20%20%20%20%20%20%22enabled%22%3A%20false%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22authorization%22%3A%20%7B%0A%20%20%20%20%22webhook%22%3A%20%7B%0A%20%20%20%20%20%20%22cacheAuthorizedTTL%22%3A%20%220s%22%2C%0A%20%20%20%20%20%20%22cacheUnauthorizedTTL%22%3A%20%220s%22%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22clusterDomain%22%3A%20%22cluster.local%22%2C%0A%20%20%22clusterDNS%22%3A%20%5B%0A%20%20%20%20%22172.30.0.10%22%0A%20%20%5D%2C%0A%20%20%22streamingConnectionIdleTimeout%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusUpdateFrequency%22%3A%20%220s%22%2C%0A%20%20%22nodeStatusReportFrequency%22%3A%20%220s%22%2C%0A%20%20%22imageMinimumGCAge%22%3A%20%220s%22%2C%0A%20%20%22volumeStatsAggPeriod%22%3A%20%220s%22%2C%0A%20%20%22systemCgroups%22%3A%20%22%2Fsystem.slice%22%2C%0A%20%20%22cgroupRoot%22%3A%20%22%2F%22%2C%0A%20%20%22cgroupDriver%22%3A%20%22systemd%22%2C%0A%20%20%22cpuManagerReconcilePeriod%22%3A%20%220s%22%2C%0A%20%20%22runtimeRequestTimeout%22%3A%20%220s%22%2C%0A%20%20%22maxPods%22%3A%20250%2C%0A%20%20%22something%22%3A%20%220s%22%2C%0A%20%20%22kubeAPIBurst%22%3A%20100%2C%0A%20%20%22serializeImagePulls%22%3A%20false%2C%0A%20%20%22evictionPressureTransitionPeriod%22%3A%20%220s%22%2C%0A%20%20%22featureGates%22%3A%20%7B%0A%20%20%20%20%22APIPriorityAndFairness%22%3A%20true%2C%0A%20%20%20%20%22CSIMigrationAWS%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureDisk%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationAzureFile%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationGCE%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationOpenStack%22%3A%20false%2C%0A%20%20%20%20%22CSIMigrationvSphere%22%3A%20false%2C%0A%20%20%20%20%22DownwardAPIHugePages%22%3A%20true%2C%0A%20%20%20%20%22LegacyNodeRoleBehavior%22%3A%20false%2C%0A%20%20%20%20%22NodeDisruptionExclusion%22%3A%20true%2C%0A%20%20%20%20%22PodSecurity%22%3A%20true%2C%0A%20%20%20%20%22RotateKubeletServerCertificate%22%3A%20true%2C%0A%20%20%20%20%22ServiceNodeExclusion%22%3A%20true%2C%0A%20%20%20%20%22SupportPodPidsLimit%22%3A%20true%0A%20%20%7D%2C%0A%20%20%22memorySwap%22%3A%20%7B%7D%2C%0A%20%20%22containerLogMaxSize%22%3A%20%2250Mi%22%2C%0A%20%20%22systemReserved%22%3A%20%7B%0A%20%20%20%20%22ephemeral-storage%22%3A%20%221Gi%22%0A%20%20%7D%2C%0A%20%20%22logging%22%3A%20%7B%0A%20%20%20%20%22flushFrequency%22%3A%200%2C%0A%20%20%20%20%22verbosity%22%3A%200%2C%0A%20%20%20%20%22options%22%3A%20%7B%0A%20%20%20%20%20%20%22json%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22infoBufferSize%22%3A%20%220%22%0A%20%20%20%20%20%20%7D%0A%20%20%20%20%7D%0A%20%20%7D%2C%0A%20%20%22shutdownGracePeriod%22%3A%20%220s%22%2C%0A%20%20%22shutdownGracePeriodCriticalPods%22%3A%20%220s%22%0A%7D%0A" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/kubernetes/kubelet.conf" - } - ] - } - }` - kc := testKubeletConfigNil() - mc := testMachineConfig(renderdKC) - It("It should evaluate as false", func() { - errorMsg := "kubelet config spec is nil, please check if KubeletConfig object has been used correctly" - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(MatchError(errorMsg)) - Expect(isSubset).To(BeFalse()) - Expect(diffString).To(BeEmpty()) - }) - }) - - Context("MachineConfig is missing kubeconfig file", func() { - renderdKC := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [] - } - }` - kc := testKubeletConfig(defaultKCPayload) - mc := testMachineConfig(renderdKC) - expectedError := "encoded kubeletconfig 99-master-generated-kubelet is missing" - - It("It should evaluate as false", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(MatchError(expectedError)) - Expect(isSubset).To(BeFalse()) - Expect(diffString).To(BeEmpty()) - }) - }) - - Context("MachineConfig kubeconfig is empty", func() { - renderdKC := ` - { - "ignition": { - "version": "3.2.0" - }, - "storage": { - "files": [ - { - "contents": { - "source": "" - }, - "mode": 420, - "overwrite": true, - "path": "/etc/kubernetes/kubelet.conf" - } - ] - } - }` - kc := testKubeletConfig(defaultKCPayload) - mc := testMachineConfig(renderdKC) - expectedError := "encoded kubeletconfig 99-master-generated-kubelet is empty" - - It("It should evaluate as false", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(MatchError(expectedError)) - Expect(isSubset).To(BeFalse()) - Expect(diffString).To(BeEmpty()) - }) - }) - - Context("KubeletConfig is nil", func() { - var kc *mcfgv1.KubeletConfig - mc := &mcfgv1.MachineConfig{} - expectedError := "kubelet config is nil" - - It("It should evaluate as false", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(MatchError(expectedError)) - Expect(isSubset).To(BeFalse()) - Expect(diffString).To(BeEmpty()) - }) - }) - - Context("MachineConfig is nil", func() { - kc := &mcfgv1.KubeletConfig{} - var mc *mcfgv1.MachineConfig - expectedError := "machine config is nil" - - It("It should evaluate as false", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(MatchError(expectedError)) - Expect(isSubset).To(BeFalse()) - Expect(diffString).To(BeEmpty()) - }) - }) - - Context("KubeletConfig and MachineConfig are empty", func() { - kc := &mcfgv1.KubeletConfig{} - mc := &mcfgv1.MachineConfig{} - expectedError := "kubelet config spec is nil, please check if KubeletConfig object has been used correctly" - - It("It should evaluate as false", func() { - isSubset, err, diffString := utils.IsKCSubsetOfMC(kc, mc) - Expect(err).To(MatchError(expectedError)) - Expect(isSubset).To(BeFalse()) - Expect(diffString).To(BeEmpty()) - }) - }) - - }) - When("Testing getting labels from roles", func() { DescribeTable("Gets expected output", func(role string, expectation map[string]string) { diff --git a/tests/e2e/framework/common.go b/tests/e2e/framework/common.go index 1514d26f7..879447c60 100644 --- a/tests/e2e/framework/common.go +++ b/tests/e2e/framework/common.go @@ -795,25 +795,29 @@ func (f *Framework) WaitForScanStatus(namespace, name string, targetStatus compv func (f *Framework) EnableRuleExistInTailoredProfile(namespace, name, ruleName string) (bool, error) { tp := &compv1alpha1.TailoredProfile{} defer f.logContainerOutput(namespace, name) - if tp.Status.State == compv1alpha1.TailoredProfileStateReady || tp.Status.State == compv1alpha1.TailoredProfileStateError { - // check if rule exist in tailoredprofile enableRules - hasRule := false - for _, r := range tp.Spec.EnableRules { - if r.Name == ruleName { - hasRule = true - break - } - } - if hasRule { - log.Printf("rule %s exist in tailoredprofile enableRules\n", ruleName) - return true, nil - } else { - log.Printf("rule %s does not exist in tailoredprofile enableRules\n", ruleName) - return false, nil - } + // check if rule exist in tailoredprofile enableRules + err := f.Client.Get(context.TODO(), types.NamespacedName{ + Name: name, + Namespace: namespace, + }, tp) + if err != nil { + return false, err } - return false, fmt.Errorf("tailoredprofile %s is not ready", name) + hasRule := false + for _, r := range tp.Spec.EnableRules { + if r.Name == ruleName { + hasRule = true + break + } + } + if hasRule { + log.Printf("rule %s exist in tailoredprofile enableRules\n", ruleName) + return true, nil + } else { + log.Printf("rule %s does not exist in tailoredprofile enableRules\n", ruleName) + return false, nil + } } func (f *Framework) WaitForTailoredProfileStatus(namespace, name string, targetStatus compv1alpha1.TailoredProfileState) error { diff --git a/tests/e2e/parallel/main_test.go b/tests/e2e/parallel/main_test.go index 42b879f74..2ac362498 100644 --- a/tests/e2e/parallel/main_test.go +++ b/tests/e2e/parallel/main_test.go @@ -158,78 +158,6 @@ func TestProfileModification(t *testing.T) { } } -func TestRuleCheckType(t *testing.T) { - t.Parallel() - f := framework.Global - const ( - changeTypeRule = "kubelet-anonymous-auth" - moderateProfileName = "moderate" - ) - var ( - baselineImage = fmt.Sprintf("%s:%s", brokenContentImagePath, "kubelet_default") - modifiedImage = fmt.Sprintf("%s:%s", brokenContentImagePath, "new_kubeletconfig") - ) - - prefixName := func(profName, ruleBaseName string) string { return profName + "-" + ruleBaseName } - - pbName := framework.GetObjNameFromTest(t) - origPb := &compv1alpha1.ProfileBundle{ - ObjectMeta: metav1.ObjectMeta{ - Name: pbName, - Namespace: f.OperatorNamespace, - }, - Spec: compv1alpha1.ProfileBundleSpec{ - ContentImage: baselineImage, - ContentFile: framework.OcpContentFile, - }, - } - // Pass nil in as the cleanupOptions since so we don't invoke all the - // cleanup function code in Create. Use defer to cleanup the - // ProfileBundle at the end of the test, instead of at the end of the - // suite. - if err := f.Client.Create(context.TODO(), origPb, nil); err != nil { - t.Fatalf("failed to create ProfileBundle: %s", err) - } - // This should get cleaned up at the end of the test - defer f.Client.Delete(context.TODO(), origPb) - - if err := f.WaitForProfileBundleStatus(pbName, compv1alpha1.DataStreamValid); err != nil { - t.Fatalf("failed waiting for the ProfileBundle to become available: %s", err) - } - - // Check that the rule exists in the original profile - changeTypeRuleName := prefixName(pbName, changeTypeRule) - err, found := f.DoesRuleExist(origPb.Namespace, changeTypeRuleName) - if err != nil { - t.Fatal(err) - } else if found != true { - t.Fatalf("expected rule %s to exist in namespace %s", changeTypeRuleName, origPb.Namespace) - } - - // update the image with a new hash - modPb := origPb.DeepCopy() - if err := f.Client.Get(context.TODO(), types.NamespacedName{Namespace: modPb.Namespace, Name: modPb.Name}, modPb); err != nil { - t.Fatalf("failed to get ProfileBundle %s", modPb.Name) - } - - modPb.Spec.ContentImage = modifiedImage - if err := f.Client.Update(context.TODO(), modPb); err != nil { - t.Fatalf("failed to update ProfileBundle %s: %s", modPb.Name, err) - } - - // Wait for the update to happen, the PB will flip first to pending, then to valid - if err := f.WaitForProfileBundleStatus(pbName, compv1alpha1.DataStreamValid); err != nil { - t.Fatalf("failed to parse ProfileBundle %s: %s", pbName, err) - } - - if err := f.AssertProfileBundleMustHaveParsedRules(pbName); err != nil { - t.Fatal(err) - } - if err := f.AssertRuleCheckTypeChangedAnnotationKey(f.OperatorNamespace, changeTypeRuleName, "Platform"); err != nil { - t.Fatal(err) - } -} - func TestProfileISTagUpdate(t *testing.T) { t.Parallel() f := framework.Global @@ -2724,124 +2652,6 @@ func TestManualRulesTailoredProfile(t *testing.T) { } } -func TestCheckDefaultKubeletConfig(t *testing.T) { - t.Parallel() - f := framework.Global - var baselineImage = fmt.Sprintf("%s:%s", brokenContentImagePath, "kubelet_default") - const requiredRule = "kubelet-test-cipher" - pbName := framework.GetObjNameFromTest(t) - prefixName := func(profName, ruleBaseName string) string { return profName + "-" + ruleBaseName } - - ocpPb := &compv1alpha1.ProfileBundle{ - ObjectMeta: metav1.ObjectMeta{ - Name: pbName, - Namespace: f.OperatorNamespace, - }, - Spec: compv1alpha1.ProfileBundleSpec{ - ContentImage: baselineImage, - ContentFile: framework.OcpContentFile, - }, - } - if err := f.Client.Create(context.TODO(), ocpPb, nil); err != nil { - t.Fatal(err) - } - defer f.Client.Delete(context.TODO(), ocpPb) - err := f.WaitForProfileBundleStatus(pbName, compv1alpha1.DataStreamValid) - if err != nil { - t.Fatal(err) - } - - // Check that if the rule we are going to test is there - requiredRuleName := prefixName(pbName, requiredRule) - err, found := f.DoesRuleExist(ocpPb.Namespace, requiredRuleName) - if err != nil { - t.Fatal(err) - } else if !found { - t.Fatalf("Expected rule %s not found", requiredRuleName) - } - - suiteName := "kubelet-default-test-suite" - - tp := &compv1alpha1.TailoredProfile{ - ObjectMeta: metav1.ObjectMeta{ - Name: suiteName, - Namespace: f.OperatorNamespace, - Labels: map[string]string{ - compv1alpha1.OutdatedReferenceValidationDisable: "true", - }, - }, - Spec: compv1alpha1.TailoredProfileSpec{ - Title: "kubelet-default-test", - Description: "A test tailored profile to test default kubelet", - EnableRules: []compv1alpha1.RuleReferenceSpec{ - { - Name: prefixName(pbName, requiredRule), - Rationale: "To be tested", - }, - }, - SetValues: []compv1alpha1.VariableValueSpec{ - { - Name: prefixName(pbName, "var-kubelet-tls-cipher-suites-regex"), - Rationale: "Value to be set", - Value: "^(TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384|TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384|TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256|TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256)$", - }, - }, - }, - } - - createTPErr := f.Client.Create(context.TODO(), tp, nil) - if createTPErr != nil { - t.Fatal(createTPErr) - } - defer f.Client.Delete(context.TODO(), tp) - - ssb := &compv1alpha1.ScanSettingBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: suiteName, - Namespace: f.OperatorNamespace, - }, - Profiles: []compv1alpha1.NamedObjectReference{ - { - APIGroup: "compliance.openshift.io/v1alpha1", - Kind: "TailoredProfile", - Name: suiteName, - }, - }, - SettingsRef: &compv1alpha1.NamedObjectReference{ - APIGroup: "compliance.openshift.io/v1alpha1", - Kind: "ScanSetting", - Name: "default", - }, - } - - err = f.Client.Create(context.TODO(), ssb, nil) - if err != nil { - t.Fatal(err) - } - defer f.Client.Delete(context.TODO(), ssb) - - // Ensure that all the scans in the suite have finished and are marked as Done - err = f.WaitForSuiteScansStatus(f.OperatorNamespace, suiteName, compv1alpha1.PhaseDone, compv1alpha1.ResultCompliant) - if err != nil { - t.Fatal(err) - } - - // the check should be shown as manual - checkResult := compv1alpha1.ComplianceCheckResult{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-kubelet-test-cipher", suiteName), - Namespace: f.OperatorNamespace, - }, - ID: "xccdf_org.ssgproject.content_rule_kubelet_test_cipher", - Status: compv1alpha1.CheckResultPass, - Severity: compv1alpha1.CheckResultSeverityMedium, - } - err = f.AssertHasCheck(suiteName, suiteName, checkResult) - if err != nil { - t.Fatal(err) - } -} - func TestHideRule(t *testing.T) { t.Parallel() f := framework.Global