Allow configuration of an external webhook & associated certs

pkg/apiserver/apiserver.go

@@ -289,12 +289,19 @@ func (s *PorchServer) Run(ctx context.Context) error {
 
 	certStorageDir, found := os.LookupEnv("CERT_STORAGE_DIR")
 	if found && strings.TrimSpace(certStorageDir) != "" {
-		if err := setupWebhooks(ctx, webhookNs, certStorageDir); err != nil {
+		// check if we should use cert manager webhook setup
+		useCertManWebhook := false
+		// if we dont find the env var or the var is not true
+		CertManWebhook, found := os.LookupEnv("CERT_MAN_WEBHOOKS")
+		if found || CertManWebhook == "true"{
+			useCertManWebhook = true
+		}
+		if err := setupWebhooks(ctx, webhookNs, certStorageDir, useCertManWebhook); err != nil {
 			klog.Errorf("%v\n", err)
 			return err
 		}
 	} else {
 		klog.Infoln("Cert storage dir not provided, skipping webhook setup")
 	}
 	return s.GenericAPIServer.PrepareRun().Run(ctx.Done())
-}
+}

pkg/apiserver/webhooks.go

@@ -30,8 +30,11 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
+
 	"github.com/nephio-project/porch/api/porch/v1alpha1"
 	"github.com/nephio-project/porch/internal/kpt/util/porch"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -51,15 +54,24 @@ const (
 	serverEndpoint     = "/validate-deletion"
 )
 
-func setupWebhooks(ctx context.Context, webhookNs string, certStorageDir string) error {
-	caBytes, err := createCerts(webhookNs, certStorageDir)
-	if err != nil {
-		return err
-	}
-	if err := createValidatingWebhook(ctx, webhookNs, caBytes); err != nil {
-		return err
+var (
+	mu          sync.Mutex
+	cert        tls.Certificate
+	certModTime time.Time
+)
+
+func setupWebhooks(ctx context.Context, webhookNs string, certStorageDir string, useCertManWebhook bool) error {
+	if !useCertManWebhook {
+		caBytes, err := createCerts(webhookNs, certStorageDir)
+		if err != nil {
+			return err
+		}
+		if err := createValidatingWebhook(ctx, webhookNs, caBytes); err != nil {
+			return err
+		}
+
 	}
-	if err := runWebhookServer(certStorageDir); err != nil {
+	if err := runWebhookServer(certStorageDir, useCertManWebhook); err != nil {
 		return err
 	}
 	return nil
@@ -226,29 +238,130 @@ func createValidatingWebhook(ctx context.Context, webhookNs string, caCert []byt
 	return nil
 }
 
-func runWebhookServer(certStorageDir string) error {
-	certFile := filepath.Join(certStorageDir, "tls.crt")
-	keyFile := filepath.Join(certStorageDir, "tls.key")
-
-	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+// load the certificate from the secret and update when secret cert data changes e.g.
+func loadCertificate(certPath, keyPath string) (tls.Certificate, error) {
+	// get info about cert manager certificate secret mounted as a volume on the porch server pod
+	certInfo, err := os.Stat(certPath)
 	if err != nil {
-		return err
+		return tls.Certificate{}, err
+	}
+	// if the last time this secret was modified was after the last time we loaded its cert files
+	// we lock access to the mount path and load the keypair from the files in our tls then release the lock
+	// set this new loaded cert as our current cert and note the modtime for next reload
+	if certInfo.ModTime().After(certModTime) {
+		mu.Lock()
+		defer mu.Unlock()
+		newCert, err := tls.LoadX509KeyPair(certPath, keyPath)
+		if err != nil {
+			return tls.Certificate{}, err
+		}
+		cert = newCert
+		certModTime = certInfo.ModTime()
 	}
-	klog.Infoln("Starting webhook server")
-	http.HandleFunc(serverEndpoint, validateDeletion)
-	server := http.Server{
-		Addr: fmt.Sprintf(":%d", webhookServicePort),
-		TLSConfig: &tls.Config{
-			Certificates: []tls.Certificate{cert},
-		},
+	return cert, nil
+}
+
+// watch for changes on the mount path of the secret as volume
+func watchCertificates(directory, certFile, keyFile string) {
+	//set up a watcher for the cert
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		klog.Fatalf("failed to start certificate watcher: %v", err)
 	}
+	defer watcher.Close()
+
+	// if the watcher notices any changes on the mount dir of the secret such as creations or writes to the files in this dir
+	// attempt to load tls cert using the new cert and key files provided and log output
+	done := make(chan bool)
 	go func() {
-		err = server.ListenAndServeTLS("", "")
-		if err != nil {
-			klog.Errorf("could not start server: %v", err)
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Op&fsnotify.Create == fsnotify.Create || event.Op&fsnotify.Write == fsnotify.Write {
+					_, err := loadCertificate(certFile, keyFile)
+					if err != nil {
+						klog.Errorf("Failed to load updated certificate: %v", err)
+					} else {
+						klog.Info("Certificate reloaded successfully")
+					}
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return
+				}
+				klog.Errorf("Watcher error: %v", err)
+			}
 		}
 	}()
-	return err
+	// start the watcher with the dir to watch
+	err = watcher.Add(directory)
+	if err != nil {
+		klog.Fatalf("Error in running watcher: %v", err)
+	}
+
+	<-done
+}
+// return current cert
+func getCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	mu.Lock()
+	defer mu.Unlock()
+	return &cert, nil
+}
+
+func runWebhookServer(certStorageDir string, useCertManWebhook bool) error {
+	certFile := filepath.Join(certStorageDir, "tls.crt")
+	keyFile := filepath.Join(certStorageDir, "tls.key")
+	// load the cert for the first time
+
+	if useCertManWebhook {
+		_, err := loadCertificate(certFile, keyFile)
+		if err != nil {
+			klog.Errorf("failed to load certificate: %v", err)
+		}
+		// Start watching the certificate files for changes
+		// watch for changes in directory where secret is mounted
+		go watchCertificates(certStorageDir, certFile, keyFile)
+
+		klog.Infoln("Starting webhook server")
+		http.HandleFunc(serverEndpoint, validateDeletion)
+		server := http.Server{
+			Addr: fmt.Sprintf(":%d", webhookServicePort),
+			TLSConfig: &tls.Config{
+				GetCertificate: getCertificate,
+			},
+		}
+		go func() {
+			err = server.ListenAndServeTLS("", "")
+			if err != nil {
+				klog.Errorf("could not start server: %v", err)
+			}
+		}()
+		return err
+
+	} else {
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return err
+		}
+		klog.Infoln("Starting webhook server")
+		http.HandleFunc(serverEndpoint, validateDeletion)
+		server := http.Server{
+			Addr: fmt.Sprintf(":%d", webhookServicePort),
+			TLSConfig: &tls.Config{
+				Certificates: []tls.Certificate{cert},
+			},
+		}
+		go func() {
+			err = server.ListenAndServeTLS("", "")
+			if err != nil {
+				klog.Errorf("could not start server: %v", err)
+			}
+		}()
+		return err
+	}
 }
 
 func validateDeletion(w http.ResponseWriter, r *http.Request) {