diff --git a/bundle/manifests/keda.clusterserviceversion.yaml b/bundle/manifests/keda.clusterserviceversion.yaml index 96c0bc073..eaaee99d6 100644 --- a/bundle/manifests/keda.clusterserviceversion.yaml +++ b/bundle/manifests/keda.clusterserviceversion.yaml @@ -625,7 +625,16 @@ spec: requests: cpu: 100m memory: 100Mi + volumeMounts: + - mountPath: /certs + name: certificates + readOnly: true serviceAccountName: keda-olm-operator + volumes: + - name: certificates + secret: + optional: true + secretName: kedaorg-certs strategy: deployment installModes: - supported: true diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 3dceb310e..b4eaf1de6 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -49,3 +49,12 @@ spec: env: - name: WATCH_NAMESPACE value: keda + volumeMounts: + - mountPath: /certs + name: certificates + readOnly: true + volumes: + - name: certificates + secret: + optional: true + secretName: kedaorg-certs diff --git a/config/manifests/bases/keda.clusterserviceversion.yaml b/config/manifests/bases/keda.clusterserviceversion.yaml index 48590ec93..9fe8d54b6 100644 --- a/config/manifests/bases/keda.clusterserviceversion.yaml +++ b/config/manifests/bases/keda.clusterserviceversion.yaml @@ -625,9 +625,18 @@ spec: capabilities: drop: - ALL + volumeMounts: + - mountPath: /certs + name: certificates + readOnly: true securityContext: runAsNonRoot: true serviceAccountName: keda-olm-operator + volumes: + - name: certificates + secret: + optional: true + secretName: kedaorg-certs strategy: deployment installModes: - supported: true diff --git a/controllers/keda/kedacontroller_controller.go b/controllers/keda/kedacontroller_controller.go index cd10a484f..c1507f27c 100644 --- a/controllers/keda/kedacontroller_controller.go +++ b/controllers/keda/kedacontroller_controller.go @@ -18,6 +18,7 @@ package keda import ( "context" + "crypto/x509" goerrors "errors" "fmt" "os" @@ -29,6 +30,7 @@ import ( "github.com/go-logr/logr" mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" + "github.com/open-policy-agent/cert-controller/pkg/rotator" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -57,6 +59,7 @@ const ( // Allowed Name of KedaController resource kedaControllerResourceName = "keda" + grpcClientCertsSecretName = "kedaorg-certs" caBundleConfigMapName = "keda-ocp-cabundle" injectCABundleAnnotation = "service.beta.openshift.io/inject-cabundle" injectCABundleAnnotationValue = "true" @@ -74,6 +77,10 @@ type KedaControllerReconciler struct { client.Client Log logr.Logger Scheme *runtime.Scheme + CertDir string + LeaderElection bool + rotatorStarted bool + mgr ctrl.Manager resourcesGeneral mf.Manifest resourcesController mf.Manifest resourcesMetrics mf.Manifest @@ -85,6 +92,7 @@ type KedaControllerReconciler struct { func (r *KedaControllerReconciler) SetupWithManager(mgr ctrl.Manager, kedaControllerResourceNamespace string, logger logr.Logger) error { r.resourceNamespace = kedaControllerResourceNamespace + r.mgr = mgr resourcesManifest, err := resources.GetResourcesManifest() if err != nil { return err @@ -253,6 +261,8 @@ func parseManifestsFromFile(manifest mf.Manifest, c client.Client) (manifestGene } else { metricsResources = append(metricsResources, r) } + case "Secret": + controllerResources = append(controllerResources, r) case "Namespace", "ServiceAccount": generalResources = append(generalResources, r) case "PodMonitor", "ServiceMonitor": @@ -359,9 +369,25 @@ func (r *KedaControllerReconciler) installController(ctx context.Context, logger transform.ReplaceWatchNamespace(instance.Spec.WatchNamespace, "keda-operator", r.Scheme, logger), } - if util.RunningOnOpenshift(ctx, logger, r.Client) { + runningOnOpenshift := util.RunningOnOpenshift(ctx, logger, r.Client) + + if runningOnOpenshift { + // certificates rotation works only on Openshift due to openshift/service-ca-operator + serviceName := "keda-operator" + certsSecretName := serviceName + "-certs" transforms = append(transforms, + transform.EnsureCertInjectionForService(serviceName, servingCertsAnnotation, certsSecretName), + transform.KedaOperatorEnsureCertificatesVolume(certsSecretName, grpcClientCertsSecretName, r.Scheme), transform.EnsureOpenshiftCABundleForOperatorDeployment(caBundleConfigMapName, r.Scheme), + transform.SetOperatorCertRotation(false, r.Scheme, logger), // don't use KEDA operator's built-in cert rotation when on OpenShift + ) + // on OpenShift 4.10 (kube 1.23) and earlier, the RuntimeDefault SeccompProfile won't validate against any SCC + if util.RunningOnClusterWithoutSeccompProfileDefault(logger, r.discoveryClient) { + transforms = append(transforms, transform.RemoveSeccompProfileFromKedaOperator(r.Scheme, logger)) + } + } else { + transforms = append(transforms, + transform.SetOperatorCertRotation(true, r.Scheme, logger), // use KEDA operator's built-in cert rotation when not on OpenShift ) } @@ -370,11 +396,6 @@ func (r *KedaControllerReconciler) installController(ctx context.Context, logger transforms = append(transforms, transform.ReplaceKedaOperatorImage(controllerImage, r.Scheme)) } - // on OpenShift 4.10 (kube 1.23) and earlier, the RuntimeDefault SeccompProfile won't validate against any SCC - if util.RunningOnOpenshift(ctx, logger, r.Client) && util.RunningOnClusterWithoutSeccompProfileDefault(logger, r.discoveryClient) { - transforms = append(transforms, transform.RemoveSeccompProfileFromKedaOperator(r.Scheme, logger)) - } - if len(instance.Spec.Operator.LogLevel) > 0 { transforms = append(transforms, transform.ReplaceKedaOperatorLogLevel(instance.Spec.Operator.LogLevel, r.Scheme, logger)) } @@ -439,6 +460,28 @@ func (r *KedaControllerReconciler) installController(ctx context.Context, logger return err } + if runningOnOpenshift && !r.rotatorStarted { + err = rotator.AddRotator(r.mgr, &rotator.CertRotator{ + SecretKey: types.NamespacedName{ + Namespace: r.resourceNamespace, + Name: grpcClientCertsSecretName, + }, + // The 3 values for SecretKey.Name above and CAName, CAOrganization below match the names the KEDA operator uses to generate its certificate + CAName: "KEDA", + CAOrganization: "KEDAORG", + CertDir: r.CertDir, + IsReady: make(chan struct{}), + RequireLeaderElection: r.LeaderElection, + ExtKeyUsages: &[]x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + }, + }) + if err != nil { + return err + } + r.rotatorStarted = true + } + return nil } @@ -502,7 +545,7 @@ func (r *KedaControllerReconciler) installMetricsServer(ctx context.Context, log } argsPrefixes := []transform.Prefix{transform.ClientCAFile, transform.TLSCertFile, transform.TLSPrivateKeyFile} - newArgs := []string{"/certs/ocp-ca.crt", "/certs/ocp-tls.crt", "/certs/ocp-tls.key"} + newArgs := []string{"/certs/ca.crt", "/certs/ocp-tls.crt", "/certs/ocp-tls.key"} serviceName := "keda-metrics-apiserver" certsSecretName := serviceName + "-certs" diff --git a/controllers/keda/transform/transform.go b/controllers/keda/transform/transform.go index 19831e1e5..0fe9768ee 100644 --- a/controllers/keda/transform/transform.go +++ b/controllers/keda/transform/transform.go @@ -33,6 +33,7 @@ const ( ClientCAFile Prefix = "--client-ca-file=" TLSCertFile Prefix = "--tls-cert-file=" TLSPrivateKeyFile Prefix = "--tls-private-key-file=" + CertRotation Prefix = "--enable-cert-rotation=" ) func (p Prefix) String() string { @@ -227,6 +228,91 @@ func MetricsServerEnsureCertificatesVolume(configMapName, secretName string, sch return ensureCertificatesVolumeForDeployment(containerNameMetricsServer, configMapName, secretName, scheme) } +func KedaOperatorEnsureCertificatesVolume(serviceSecretName string, grpcClientCertsSecretName string, scheme *runtime.Scheme) mf.Transformer { + return func(u *unstructured.Unstructured) error { + if u.GetKind() == "Deployment" { + deploy := &appsv1.Deployment{} + if err := scheme.Convert(u, deploy, nil); err != nil { + return err + } + + certificatesVolume := corev1.Volume{ + Name: "certificates", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: serviceSecretName, + }, + Items: []corev1.KeyToPath{ + {Key: "tls.crt", Path: "tls.crt"}, // use OpenShift-generated service cert for gRPC service + {Key: "tls.key", Path: "tls.key"}, + }, + }, + }, + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: grpcClientCertsSecretName, + }, + Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}, // trust clients using kedaorg-certs cert + }, + }, + }, + }, + }, + } + + volumes := deploy.Spec.Template.Spec.Volumes + certificatesVolumeFound := false + for i := range volumes { + if volumes[i].Name == "certificates" { + volumes[i] = certificatesVolume + certificatesVolumeFound = true + } + } + + if !certificatesVolumeFound { + deploy.Spec.Template.Spec.Volumes = append(deploy.Spec.Template.Spec.Volumes, certificatesVolume) + } + + containers := deploy.Spec.Template.Spec.Containers + for i := range containers { + if containers[i].Name == containerNameAdmissionWebhooks { + // mount Volume referencing certs in Secrets + certificatesVolumeMount := corev1.VolumeMount{ + Name: "certificates", + MountPath: "/certs", + ReadOnly: true, + } + + volumeMounts := containers[i].VolumeMounts + certificatesVolumeMountFound := false + for j := range volumeMounts { + if volumeMounts[j].Name == "certificates" { + volumeMounts[j] = certificatesVolumeMount + certificatesVolumeMountFound = true + } + } + + if !certificatesVolumeMountFound { + containers[i].VolumeMounts = append(containers[i].VolumeMounts, certificatesVolumeMount) + } + + break + } + } + + if err := scheme.Convert(deploy, u, nil); err != nil { + return err + } + } + return nil + } +} + func AdmissionWebhooksEnsureCertificatesVolume(configMapName, secretName string, scheme *runtime.Scheme) mf.Transformer { return func(u *unstructured.Unstructured) error { if u.GetKind() == "Deployment" { @@ -341,6 +427,10 @@ func ensureCertificatesVolumeForDeployment(containerName, configMapName, secretN LocalObjectReference: corev1.LocalObjectReference{ Name: "kedaorg-certs", }, + Items: []corev1.KeyToPath{ + {Key: "tls.crt", Path: "tls.crt"}, // use the generated gRPC client cert + {Key: "tls.key", Path: "tls.key"}, + }, }, }, { @@ -348,7 +438,10 @@ func ensureCertificatesVolumeForDeployment(containerName, configMapName, secretN LocalObjectReference: corev1.LocalObjectReference{ Name: secretName, }, - Items: []corev1.KeyToPath{{Key: "tls.crt", Path: "ocp-tls.crt"}, {Key: "tls.key", Path: "ocp-tls.key"}}, + Items: []corev1.KeyToPath{ + {Key: "tls.crt", Path: "ocp-tls.crt"}, + {Key: "tls.key", Path: "ocp-tls.key"}, + }, }, }, { @@ -356,7 +449,7 @@ func ensureCertificatesVolumeForDeployment(containerName, configMapName, secretN LocalObjectReference: corev1.LocalObjectReference{ Name: configMapName, }, - Items: []corev1.KeyToPath{{Key: "service-ca.crt", Path: "ocp-ca.crt"}}, + Items: []corev1.KeyToPath{{Key: "service-ca.crt", Path: "ca.crt"}}, // trust openshift-generated service certs }, }, }, @@ -760,6 +853,14 @@ func ReplaceArbitraryArg(argument string, resource string, scheme *runtime.Schem } } +func SetOperatorCertRotation(enable bool, scheme *runtime.Scheme, logger logr.Logger) mf.Transformer { + arg := "false" + if enable { + arg = "true" + } + return replaceContainerArg(arg, CertRotation, containerNameKedaOperator, scheme, logger) +} + func ReplaceAuditConfig(argument string, selector string, scheme *runtime.Scheme, logger logr.Logger) mf.Transformer { var prefix string switch selector { diff --git a/go.mod b/go.mod index ae8e6d802..ec151b4c2 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/manifestival/manifestival v0.7.3-0.20230801201407-f20c69532c27 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.27.10 + github.com/open-policy-agent/cert-controller v0.10.0 github.com/openshift/api v0.0.0-20230920152731-7d89b46689d4 k8s.io/api v0.28.2 k8s.io/apimachinery v0.28.2 @@ -67,6 +68,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.starlark.net v0.0.0-20230921161717-a9587466d7a5 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect diff --git a/go.sum b/go.sum index 7ab804d2e..9272416a5 100644 --- a/go.sum +++ b/go.sum @@ -428,6 +428,9 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/open-policy-agent/cert-controller v0.10.0 h1:9hBJsnpHsBqKR7VVtOHW19mk/a1vQvje6+QSJeRHuDg= +github.com/open-policy-agent/cert-controller v0.10.0/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= github.com/openshift/api v0.0.0-20230920152731-7d89b46689d4 h1:1BdCmGkO+aitiGzGYm6rqPtwY6+2etUWMi7429swku0= github.com/openshift/api v0.0.0-20230920152731-7d89b46689d4/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -546,6 +549,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= diff --git a/main.go b/main.go index a198297d9..97e7ee428 100644 --- a/main.go +++ b/main.go @@ -68,8 +68,10 @@ func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string + var certDir string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&certDir, "cert-dir", "/certs", "Directory where gRPC client certs secret is mounted.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -97,8 +99,10 @@ func main() { } if err = (&kedacontrollers.KedaControllerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + CertDir: certDir, + LeaderElection: enableLeaderElection, }).SetupWithManager(mgr, installNamespace, setupLog); err != nil { setupLog.Error(err, "unable to create controller", "controller", "KedaController") os.Exit(1) diff --git a/resources/keda-olm-operator.yaml b/resources/keda-olm-operator.yaml index 48c924d5e..2a6b5ce8c 100644 --- a/resources/keda-olm-operator.yaml +++ b/resources/keda-olm-operator.yaml @@ -1,4 +1,15 @@ --- +apiVersion: v1 +kind: Secret +metadata: + labels: + app: keda-operator + app.kubernetes.io/component: keda-operator + app.kubernetes.io/name: keda-operator + app.kubernetes.io/part-of: keda + name: kedaorg-certs + namespace: keda +--- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: