apiext: add crd ca bundle check to ready probe

Although the apiext server has CA bundle it might not
have been injected into the CRD. Unfortunatelly, there is
no good Condition/Readiness check on the CRD to ensure
it has been patched correctly.

This causes a race condition when using something like Helm
because the apiext pod will say it is ready because it has a CA
cert but the CA bundle might not have been picked up by the
k8s api-extension server.

This adds an additional check to the Ready Probe to validate
both that we have a CA Cert and in fact it matches the CA bundle
in the CRD's. Since we are using the controller-runtime Manager
client which caches this List this will be a low latency way to ensure
the CRD's are patched and ready as well.

Signed-off-by: Lance Austin <[email protected]>

pkg/apiext/server.go

@@ -1,6 +1,7 @@
 package apiext
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"errors"
@@ -48,6 +49,7 @@ type WebhookRunner interface {
 type WebhookServer struct {
 	logger               *zap.Logger
 	certificateAuthority ca.CertificateAuthority
+	k8sClient            client.Reader
 	namespace            string
 	serviceSettings      types.NamespacedName
 	caSecretSettings     types.NamespacedName
@@ -121,6 +123,8 @@ func (s *WebhookServer) Run(ctx context.Context, scheme *runtime.Scheme) error {
 		return err
 	}
 
+	s.k8sClient = mgr.GetClient()
+
 	caCertController := cacertcontroller.NewCACertController(
 		mgr.GetClient(),
 		s.logger,
@@ -219,8 +223,8 @@ func (s *WebhookServer) serveHealthz(ctx context.Context) error {
 	errChan := make(chan error)
 	mux := http.NewServeMux()
 
-	mux.Handle(path.ProbesReady, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		if s.certificateAuthority.Ready() {
+	mux.Handle(path.ProbesReady, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if s.certificateAuthority.Ready() && s.areCRDsReady(r.Context()) {
 			_, _ = io.WriteString(w, "Ready!\n")
 			return
 		}
@@ -258,6 +262,40 @@ func (s *WebhookServer) isLeaderElectionEnabled() bool {
 	return s.caMgmtEnabled || s.crdPatchMgmtEnabled
 }
 
+func (s *WebhookServer) areCRDsReady(ctx context.Context) bool {
+	caCert := s.certificateAuthority.GetCACert()
+	if caCert == nil {
+		return false
+	}
+
+	crdList := &apiextv1.CustomResourceDefinitionList{}
+	options := []client.ListOption{
+		client.MatchingLabels{"app.kubernetes.io/part-of": "emissary-apiext"},
+	}
+
+	err := s.k8sClient.List(ctx, crdList, options...)
+	if err != nil {
+		s.logger.Error("ready check unable to list getambassadorio crds", zap.Error(err))
+		return false
+	}
+
+	for _, item := range crdList.Items {
+		if len(item.Spec.Versions) < 2 {
+			continue
+		}
+
+		if item.Spec.Conversion == nil || item.Spec.Conversion.Webhook == nil || item.Spec.Conversion.Webhook.ClientConfig == nil {
+			return false
+		}
+
+		if !bytes.Equal(item.Spec.Conversion.Webhook.ClientConfig.CABundle, caCert.CertificatePEM) {
+			return false
+		}
+	}
+
+	return true
+}
+
 func createCacheOptions(secretNamespace string) cache.Options {
 	return cache.Options{
 		ByObject: map[client.Object]cache.ByObject{