Add gRPC client cert generation/rotation when running on OpenShift

When not running on OpenShift, use the certificate generation/rotation
built in to the KEDA operator, and use its single certificate and the
CA certificate which signed it for all of the following:
  * KEDA operator's gRPC service
  * Metrics Server (adapter) API service endpoint
  * Validating admission webhook service endpoint
  * Client certificate used by the adapter to authenticate against the gRPC service

When running on OpenShift, use OpenShift-generated certificates (and the
cluster's service CA for validation) for each of the following services:
  * KEDA operator's gRPC service
  * Metrics Server (adapter) API service endpoint
  * Validating admission webhook service endpoint
The OLM operator generates CA and a gRPC client certificate for:
  * The adapter to authenticate itself to the KEDA operator (key/cert)
  * The KEDA operator's gRPC service to verify clients (the adapter) (CA
    cert)

Signed-off-by: Joel Smith <[email protected]>

bundle/manifests/keda.clusterserviceversion.yaml

@@ -625,7 +625,16 @@ spec:
                   requests:
                     cpu: 100m
                     memory: 100Mi
+                volumeMounts:
+                - mountPath: /certs
+                  name: certificates
+                  readOnly: true
               serviceAccountName: keda-olm-operator
+              volumes:
+              - name: certificates
+                secret:
+                  optional: true
+                  secretName: kedaorg-certs
     strategy: deployment
   installModes:
   - supported: true

config/manager/manager.yaml

@@ -49,3 +49,12 @@ spec:
           env:
             - name: WATCH_NAMESPACE
               value: keda
+          volumeMounts:
+          - mountPath: /certs
+            name: certificates
+            readOnly: true
+      volumes:
+      - name: certificates
+        secret:
+          optional: true
+          secretName: kedaorg-certs

config/manifests/bases/keda.clusterserviceversion.yaml

@@ -625,9 +625,18 @@ spec:
                   capabilities:
                     drop:
                     - ALL
+                volumeMounts:
+                - mountPath: /certs
+                  name: certificates
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: keda-olm-operator
+              volumes:
+              - name: certificates
+                secret:
+                  optional: true
+                  secretName: kedaorg-certs
     strategy: deployment
   installModes:
   - supported: true

controllers/keda/kedacontroller_controller.go

@@ -18,6 +18,7 @@ package keda
 
 import (
 	"context"
+	"crypto/x509"
 	goerrors "errors"
 	"fmt"
 	"os"
@@ -29,6 +30,7 @@ import (
 	"github.com/go-logr/logr"
 	mfc "github.com/manifestival/controller-runtime-client"
 	mf "github.com/manifestival/manifestival"
+	"github.com/open-policy-agent/cert-controller/pkg/rotator"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -57,6 +59,7 @@ const (
 	// Allowed Name of KedaController resource
 	kedaControllerResourceName = "keda"
 
+	grpcClientCertsSecretName     = "kedaorg-certs"
 	caBundleConfigMapName         = "keda-ocp-cabundle"
 	injectCABundleAnnotation      = "service.beta.openshift.io/inject-cabundle"
 	injectCABundleAnnotationValue = "true"
@@ -74,6 +77,10 @@ type KedaControllerReconciler struct {
 	client.Client
 	Log                 logr.Logger
 	Scheme              *runtime.Scheme
+	CertDir             string
+	LeaderElection      bool
+	rotatorStarted      bool
+	mgr                 ctrl.Manager
 	resourcesGeneral    mf.Manifest
 	resourcesController mf.Manifest
 	resourcesMetrics    mf.Manifest
@@ -85,6 +92,7 @@ type KedaControllerReconciler struct {
 
 func (r *KedaControllerReconciler) SetupWithManager(mgr ctrl.Manager, kedaControllerResourceNamespace string, logger logr.Logger) error {
 	r.resourceNamespace = kedaControllerResourceNamespace
+	r.mgr = mgr
 	resourcesManifest, err := resources.GetResourcesManifest()
 	if err != nil {
 		return err
@@ -253,6 +261,8 @@ func parseManifestsFromFile(manifest mf.Manifest, c client.Client) (manifestGene
 			} else {
 				metricsResources = append(metricsResources, r)
 			}
+		case "Secret":
+			controllerResources = append(controllerResources, r)
 		case "Namespace", "ServiceAccount":
 			generalResources = append(generalResources, r)
 		case "PodMonitor", "ServiceMonitor":
@@ -359,9 +369,25 @@ func (r *KedaControllerReconciler) installController(ctx context.Context, logger
 		transform.ReplaceWatchNamespace(instance.Spec.WatchNamespace, "keda-operator", r.Scheme, logger),
 	}
 
-	if util.RunningOnOpenshift(ctx, logger, r.Client) {
+	runningOnOpenshift := util.RunningOnOpenshift(ctx, logger, r.Client)
+
+	if runningOnOpenshift {
+		// certificates rotation works only on Openshift due to openshift/service-ca-operator
+		serviceName := "keda-operator"
+		certsSecretName := serviceName + "-certs"
 		transforms = append(transforms,
+			transform.EnsureCertInjectionForService(serviceName, servingCertsAnnotation, certsSecretName),
+			transform.KedaOperatorEnsureCertificatesVolume(certsSecretName, grpcClientCertsSecretName, r.Scheme),
 			transform.EnsureOpenshiftCABundleForOperatorDeployment(caBundleConfigMapName, r.Scheme),
+			transform.SetOperatorCertRotation(false, r.Scheme, logger), // don't use KEDA operator's built-in cert rotation when on OpenShift
+		)
+		// on OpenShift 4.10 (kube 1.23) and earlier, the RuntimeDefault SeccompProfile won't validate against any SCC
+		if util.RunningOnClusterWithoutSeccompProfileDefault(logger, r.discoveryClient) {
+			transforms = append(transforms, transform.RemoveSeccompProfileFromKedaOperator(r.Scheme, logger))
+		}
+	} else {
+		transforms = append(transforms,
+			transform.SetOperatorCertRotation(true, r.Scheme, logger), // use KEDA operator's built-in cert rotation when not on OpenShift
 		)
 	}
 
@@ -370,11 +396,6 @@ func (r *KedaControllerReconciler) installController(ctx context.Context, logger
 		transforms = append(transforms, transform.ReplaceKedaOperatorImage(controllerImage, r.Scheme))
 	}
 
-	// on OpenShift 4.10 (kube 1.23) and earlier, the RuntimeDefault SeccompProfile won't validate against any SCC
-	if util.RunningOnOpenshift(ctx, logger, r.Client) && util.RunningOnClusterWithoutSeccompProfileDefault(logger, r.discoveryClient) {
-		transforms = append(transforms, transform.RemoveSeccompProfileFromKedaOperator(r.Scheme, logger))
-	}
-
 	if len(instance.Spec.Operator.LogLevel) > 0 {
 		transforms = append(transforms, transform.ReplaceKedaOperatorLogLevel(instance.Spec.Operator.LogLevel, r.Scheme, logger))
 	}
@@ -439,6 +460,28 @@ func (r *KedaControllerReconciler) installController(ctx context.Context, logger
 		return err
 	}
 
+	if runningOnOpenshift && !r.rotatorStarted {
+		err = rotator.AddRotator(r.mgr, &rotator.CertRotator{
+			SecretKey: types.NamespacedName{
+				Namespace: r.resourceNamespace,
+				Name:      grpcClientCertsSecretName,
+			},
+			// The 3 values for SecretKey.Name above and CAName, CAOrganization below match the names the KEDA operator uses to generate its certificate
+			CAName:                "KEDA",
+			CAOrganization:        "KEDAORG",
+			CertDir:               r.CertDir,
+			IsReady:               make(chan struct{}),
+			RequireLeaderElection: r.LeaderElection,
+			ExtKeyUsages: &[]x509.ExtKeyUsage{
+				x509.ExtKeyUsageClientAuth,
+			},
+		})
+		if err != nil {
+			return err
+		}
+		r.rotatorStarted = true
+	}
+
 	return nil
 }
 
@@ -502,7 +545,7 @@ func (r *KedaControllerReconciler) installMetricsServer(ctx context.Context, log
 		}
 
 		argsPrefixes := []transform.Prefix{transform.ClientCAFile, transform.TLSCertFile, transform.TLSPrivateKeyFile}
-		newArgs := []string{"/certs/ocp-ca.crt", "/certs/ocp-tls.crt", "/certs/ocp-tls.key"}
+		newArgs := []string{"/certs/ca.crt", "/certs/ocp-tls.crt", "/certs/ocp-tls.key"}
 
 		serviceName := "keda-metrics-apiserver"
 		certsSecretName := serviceName + "-certs"

controllers/keda/transform/transform.go

@@ -33,6 +33,7 @@ const (
 	ClientCAFile          Prefix = "--client-ca-file="
 	TLSCertFile           Prefix = "--tls-cert-file="
 	TLSPrivateKeyFile     Prefix = "--tls-private-key-file="
+	CertRotation          Prefix = "--enable-cert-rotation="
 )
 
 func (p Prefix) String() string {
@@ -227,6 +228,91 @@ func MetricsServerEnsureCertificatesVolume(configMapName, secretName string, sch
 	return ensureCertificatesVolumeForDeployment(containerNameMetricsServer, configMapName, secretName, scheme)
 }
 
+func KedaOperatorEnsureCertificatesVolume(serviceSecretName string, grpcClientCertsSecretName string, scheme *runtime.Scheme) mf.Transformer {
+	return func(u *unstructured.Unstructured) error {
+		if u.GetKind() == "Deployment" {
+			deploy := &appsv1.Deployment{}
+			if err := scheme.Convert(u, deploy, nil); err != nil {
+				return err
+			}
+
+			certificatesVolume := corev1.Volume{
+				Name: "certificates",
+				VolumeSource: corev1.VolumeSource{
+					Projected: &corev1.ProjectedVolumeSource{
+						Sources: []corev1.VolumeProjection{
+							{
+								Secret: &corev1.SecretProjection{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: serviceSecretName,
+									},
+									Items: []corev1.KeyToPath{
+										{Key: "tls.crt", Path: "tls.crt"}, // use OpenShift-generated service cert for gRPC service
+										{Key: "tls.key", Path: "tls.key"},
+									},
+								},
+							},
+							{
+								Secret: &corev1.SecretProjection{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: grpcClientCertsSecretName,
+									},
+									Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}, // trust clients using kedaorg-certs cert
+								},
+							},
+						},
+					},
+				},
+			}
+
+			volumes := deploy.Spec.Template.Spec.Volumes
+			certificatesVolumeFound := false
+			for i := range volumes {
+				if volumes[i].Name == "certificates" {
+					volumes[i] = certificatesVolume
+					certificatesVolumeFound = true
+				}
+			}
+
+			if !certificatesVolumeFound {
+				deploy.Spec.Template.Spec.Volumes = append(deploy.Spec.Template.Spec.Volumes, certificatesVolume)
+			}
+
+			containers := deploy.Spec.Template.Spec.Containers
+			for i := range containers {
+				if containers[i].Name == containerNameAdmissionWebhooks {
+					// mount Volume referencing certs in Secrets
+					certificatesVolumeMount := corev1.VolumeMount{
+						Name:      "certificates",
+						MountPath: "/certs",
+						ReadOnly:  true,
+					}
+
+					volumeMounts := containers[i].VolumeMounts
+					certificatesVolumeMountFound := false
+					for j := range volumeMounts {
+						if volumeMounts[j].Name == "certificates" {
+							volumeMounts[j] = certificatesVolumeMount
+							certificatesVolumeMountFound = true
+						}
+					}
+
+					if !certificatesVolumeMountFound {
+						containers[i].VolumeMounts = append(containers[i].VolumeMounts, certificatesVolumeMount)
+					}
+
+					break
+				}
+			}
+
+			if err := scheme.Convert(deploy, u, nil); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
 func AdmissionWebhooksEnsureCertificatesVolume(configMapName, secretName string, scheme *runtime.Scheme) mf.Transformer {
 	return func(u *unstructured.Unstructured) error {
 		if u.GetKind() == "Deployment" {
@@ -341,22 +427,29 @@ func ensureCertificatesVolumeForDeployment(containerName, configMapName, secretN
 									LocalObjectReference: corev1.LocalObjectReference{
 										Name: "kedaorg-certs",
 									},
+									Items: []corev1.KeyToPath{
+										{Key: "tls.crt", Path: "tls.crt"}, // use the generated gRPC client cert
+										{Key: "tls.key", Path: "tls.key"},
+									},
 								},
 							},
 							{
 								Secret: &corev1.SecretProjection{
 									LocalObjectReference: corev1.LocalObjectReference{
 										Name: secretName,
 									},
-									Items: []corev1.KeyToPath{{Key: "tls.crt", Path: "ocp-tls.crt"}, {Key: "tls.key", Path: "ocp-tls.key"}},
+									Items: []corev1.KeyToPath{
+										{Key: "tls.crt", Path: "ocp-tls.crt"},
+										{Key: "tls.key", Path: "ocp-tls.key"},
+									},
 								},
 							},
 							{
 								ConfigMap: &corev1.ConfigMapProjection{
 									LocalObjectReference: corev1.LocalObjectReference{
 										Name: configMapName,
 									},
-									Items: []corev1.KeyToPath{{Key: "service-ca.crt", Path: "ocp-ca.crt"}},
+									Items: []corev1.KeyToPath{{Key: "service-ca.crt", Path: "ca.crt"}}, // trust openshift-generated service certs
 								},
 							},
 						},
@@ -760,6 +853,14 @@ func ReplaceArbitraryArg(argument string, resource string, scheme *runtime.Schem
 	}
 }
 
+func SetOperatorCertRotation(enable bool, scheme *runtime.Scheme, logger logr.Logger) mf.Transformer {
+	arg := "false"
+	if enable {
+		arg = "true"
+	}
+	return replaceContainerArg(arg, CertRotation, containerNameKedaOperator, scheme, logger)
+}
+
 func ReplaceAuditConfig(argument string, selector string, scheme *runtime.Scheme, logger logr.Logger) mf.Transformer {
 	var prefix string
 	switch selector {

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/manifestival/manifestival v0.7.3-0.20230801201407-f20c69532c27
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.27.10
+	github.com/open-policy-agent/cert-controller v0.10.0
 	github.com/openshift/api v0.0.0-20230920152731-7d89b46689d4
 	k8s.io/api v0.28.2
 	k8s.io/apimachinery v0.28.2
@@ -67,6 +68,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	go.starlark.net v0.0.0-20230921161717-a9587466d7a5 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect

go.sum

@@ -428,6 +428,9 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
+github.com/open-policy-agent/cert-controller v0.10.0 h1:9hBJsnpHsBqKR7VVtOHW19mk/a1vQvje6+QSJeRHuDg=
+github.com/open-policy-agent/cert-controller v0.10.0/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ=
 github.com/openshift/api v0.0.0-20230920152731-7d89b46689d4 h1:1BdCmGkO+aitiGzGYm6rqPtwY6+2etUWMi7429swku0=
 github.com/openshift/api v0.0.0-20230920152731-7d89b46689d4/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -546,6 +549,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=

main.go

@@ -68,8 +68,10 @@ func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
+	var certDir string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	flag.StringVar(&certDir, "cert-dir", "/certs", "Directory where gRPC client certs secret is mounted.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -97,8 +99,10 @@ func main() {
 	}
 
 	if err = (&kedacontrollers.KedaControllerReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:         mgr.GetClient(),
+		Scheme:         mgr.GetScheme(),
+		CertDir:        certDir,
+		LeaderElection: enableLeaderElection,
 	}).SetupWithManager(mgr, installNamespace, setupLog); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "KedaController")
 		os.Exit(1)

resources/keda-olm-operator.yaml

@@ -1,4 +1,15 @@
 ---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app: keda-operator
+    app.kubernetes.io/component: keda-operator
+    app.kubernetes.io/name: keda-operator
+    app.kubernetes.io/part-of: keda
+  name: kedaorg-certs
+  namespace: keda
+---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata: