Allow configuration of an external webhook & associated certs

pkg/apiserver/webhooks.go

@@ -33,6 +33,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
+
 	"github.com/nephio-project/porch/api/porch/v1alpha1"
 	"github.com/nephio-project/porch/internal/kpt/util/porch"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -67,6 +69,7 @@ type WebhookConfig struct {
 	Path             string
 	Port             int32
 	CertStorageDir   string
+	CertManWebhook 	 bool
 }
 
 func NewWebhookConfig() *WebhookConfig {
@@ -90,18 +93,27 @@ func NewWebhookConfig() *WebhookConfig {
 	cfg.Path = serverEndpoint
 	cfg.Port = getEnvInt32("WEBHOOK_PORT", 8443)
 	cfg.CertStorageDir = getEnv("CERT_STORAGE_DIR", "/tmp/cert")
+	cfg.CertManWebhook = getEnvBool("USE_CERT_MAN_FOR_WEBHOOK", false)
 	return &cfg
 }
 
+var (
+	cert        tls.Certificate
+	certModTime time.Time
+)
+
 func setupWebhooks(ctx context.Context) error {
 	cfg := NewWebhookConfig()
-	caBytes, err := createCerts(cfg)
-	if err != nil {
-		return err
-	}
-	if err := createValidatingWebhook(ctx, cfg, caBytes); err != nil {
-		return err
+	if !cfg.CertManWebhook {
+		caBytes, err := createCerts(cfg)
+		if err != nil {
+			return err
+		}
+		if err := createValidatingWebhook(ctx, cfg, caBytes); err != nil {
+			return err
+		}
 	}
+
 	if err := runWebhookServer(cfg); err != nil {
 		return err
 	}
@@ -279,14 +291,78 @@ func createValidatingWebhook(ctx context.Context, cfg *WebhookConfig, caCert []b
 	return nil
 }
 
+// load the certificate & keep note of time loaded for reload on new cert details
+func loadCertificate(certPath, keyPath string) (tls.Certificate, error) {
+	certInfo, err := os.Stat(certPath)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	if certInfo.ModTime().After(certModTime) {
+		newCert, err := tls.LoadX509KeyPair(certPath, keyPath)
+		if err != nil {
+			return tls.Certificate{}, err
+		}
+		cert = newCert
+		certModTime = certInfo.ModTime()
+	}
+	return cert, nil
+}
+
+// watch for changes on the mount path of the secret as volume
+func watchCertificates(directory, certFile, keyFile string) {
+	//set up a watcher for the cert
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		klog.Fatalf("failed to start certificate watcher: %v", err)
+	}
+	defer watcher.Close()
+	// start the watcher with the dir to watch
+	err = watcher.Add(directory)
+	if err != nil {
+		klog.Errorf("Error in running watcher: %v", err)
+	}
+	// if the watcher notices any changes on the mount dir of the secret such as creations or writes to the files in this dir
+	// attempt to load tls cert using the new cert and key files provided and log output
+	done := make(chan bool)
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Op&fsnotify.Create == fsnotify.Create || event.Op&fsnotify.Write == fsnotify.Write {
+					_, err := loadCertificate(certFile, keyFile)
+					if err != nil {
+						klog.Errorf("Failed to load updated certificate: %v", err)
+					} else {
+						klog.Info("Certificate reloaded successfully")
+					}
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return
+				}
+				klog.Errorf("Watcher error: %v", err)
+			}
+		}
+	}()
+	<-done
+}
+
 func runWebhookServer(cfg *WebhookConfig) error {
 	certFile := filepath.Join(cfg.CertStorageDir, "tls.crt")
 	keyFile := filepath.Join(cfg.CertStorageDir, "tls.key")
+	// load the cert for the first time
 
-	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	cert, err := loadCertificate(certFile, keyFile)
 	if err != nil {
+		klog.Errorf("failed to load certificate: %v", err)
 		return err
 	}
+	if cfg.CertManWebhook {
+		go watchCertificates(cfg.CertStorageDir, certFile, keyFile)
+	}
 	klog.Infoln("Starting webhook server")
 	http.HandleFunc(cfg.Path, validateDeletion)
 	server := http.Server{
@@ -302,6 +378,7 @@ func runWebhookServer(cfg *WebhookConfig) error {
 		}
 	}()
 	return err
+
 }
 
 func validateDeletion(w http.ResponseWriter, r *http.Request) {
@@ -436,6 +513,18 @@ func getEnv(key string, defaultValue string) string {
 	return value
 }
 
+func getEnvBool(key string, defaultValue bool) bool {
+	value, found := os.LookupEnv(key)
+	if !found {
+		return defaultValue
+	}
+	boolean, err := strconv.ParseBool(value)
+	if err != nil {
+		panic("could not parse boolean from environment variable: " + key)
+	}
+	return boolean
+}
+
 func getEnvInt32(key string, defaultValue int32) int32 {
 	value, found := os.LookupEnv(key)
 	if !found {

pkg/apiserver/webhooks_test.go

@@ -17,13 +17,15 @@ package apiserver
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -62,6 +64,141 @@ func TestCreateCerts(t *testing.T) {
 	require.True(t, strings.HasSuffix(keyStr, "\n-----END RSA PRIVATE KEY-----"))
 }
 
+func TestLoadCertificate(t *testing.T) {
+
+	//what do i need to test.
+	// first create dummy certs for testing
+	webhookCfg := WebhookConfig{
+		Type:           WebhookTypeUrl,
+		Host:           "localhost",
+		CertStorageDir: t.TempDir(),
+	}
+	defer func() {
+		require.NoError(t, os.RemoveAll(webhookCfg.CertStorageDir))
+	}()
+
+	_, err := createCerts(&webhookCfg)
+	require.NoError(t, err)
+
+	//1. test file that cannot be os.stated or causes that function to fail
+	_, err1 := loadCertificate(filepath.Join(webhookCfg.CertStorageDir, "nonexistingcrtfile.key"), filepath.Join(webhookCfg.CertStorageDir, "nonexistingkeyfile.key"))
+	require.Error(t, err1)
+	// reseting back to 0
+	certModTime = time.Time{}
+	//2. test happy path of os.stat and continue to next error
+	//3. test loading good cert happy path
+	keypath := filepath.Join(webhookCfg.CertStorageDir, "tls.key")
+	crtpath := filepath.Join(webhookCfg.CertStorageDir, "tls.crt")
+
+	_, err2 := loadCertificate(crtpath, keypath)
+	require.NoError(t, err2)
+	certModTime = time.Time{}
+
+	//4. test loading faulty cert error
+	data := []byte("Hello, World!")
+
+	writeErr := os.WriteFile(keypath, data, 0644)
+	require.NoError(t, writeErr)
+	writeErr2 := os.WriteFile(crtpath, data, 0644)
+	require.NoError(t, writeErr2)
+
+	_, err3 := loadCertificate(filepath.Join(webhookCfg.CertStorageDir, "tls.crt"), filepath.Join(webhookCfg.CertStorageDir, "tls.key"))
+	require.Error(t, err3)
+	certModTime = time.Time{}
+}
+
+// method for capturing klog error's
+func captureStderr(f func()) string {
+	read, write, _ := os.Pipe()
+	stderr := os.Stderr
+	os.Stderr = write
+	outputChannel := make(chan string)
+
+	// Copy the output in a separate goroutine so printing can't block indefinitely.
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, read)
+		outputChannel <- buf.String()
+	}()
+
+	// Run the provided function and capture the stderr output
+	f()
+
+	// Restore the original stderr and close the write-end of the pipe so the goroutine will exit
+	os.Stderr = stderr
+	write.Close()
+	out := <-outputChannel
+
+	return out
+}
+
+func TestWatchCertificates(t *testing.T) {
+	// method for processing klog output
+	assertLogMessages := func(log string) error {
+		if len(log) > 0 {
+			if log[0] == 'E' || log[0] == 'W' || log[0] == 'F' {
+				return errors.New("Error Occured in Watcher")
+			}
+		}
+		return nil
+	}
+	// Set up the temp directory with dummy certificate files
+	webhookCfg := WebhookConfig{
+		Type:           WebhookTypeUrl,
+		Host:           "localhost",
+		CertStorageDir: t.TempDir(),
+	}
+	defer func() {
+		require.NoError(t, os.RemoveAll(webhookCfg.CertStorageDir))
+	}()
+
+	_, err := createCerts(&webhookCfg)
+	require.NoError(t, err)
+
+	keyFile := filepath.Join(webhookCfg.CertStorageDir, "tls.key")
+	certFile := filepath.Join(webhookCfg.CertStorageDir, "tls.crt")
+
+	// firstly test error occuring from invalid entity for watcher to watch. aka invalid dir
+	// we expect an error
+	go watchCertificates("Dummy Directory that does not exist", certFile, keyFile)
+
+	invalid_watch_entity_logs := captureStderr(func() {
+		time.Sleep(100 * time.Millisecond) // Give some time for the logs to be flushed
+	})
+	t.Log(invalid_watch_entity_logs)
+	err = assertLogMessages(invalid_watch_entity_logs)
+	require.Error(t, err)
+
+	go watchCertificates(webhookCfg.CertStorageDir, certFile, keyFile)
+	time.Sleep(1 * time.Second)
+
+	//create file to trigger change but not alter the certificate contents
+	//should trigger reload and certificate reloaded successfully
+	newFilePath := filepath.Join(webhookCfg.CertStorageDir, "new_temp_file.txt")
+	_, err = os.Create(newFilePath)
+	require.NoError(t, err)
+
+	valid_reload_logs := captureStderr(func() {
+		time.Sleep(100 * time.Millisecond) // Give some time for the logs to be flushed
+	})
+	t.Log(valid_reload_logs)
+	err = assertLogMessages(valid_reload_logs)
+	require.NoError(t, err)
+
+	// Modify the certificate file to trigger a file system event
+	// should cause an error log since cert contents are not valid anymore
+	certModTime = time.Time{}
+	err = os.WriteFile(certFile, []byte("dummy text"), 0660)
+	require.NoError(t, err)
+
+	invalid_reload_logs := captureStderr(func() {
+		time.Sleep(100 * time.Millisecond) // Give some time for the logs to be flushed
+	})
+	t.Log(invalid_reload_logs)
+	err = assertLogMessages(invalid_reload_logs)
+	require.Error(t, err)
+}
+
 func TestValidateDeletion(t *testing.T) {
 	t.Run("invalid content-type", func(t *testing.T) {
 		request, err := http.NewRequest(http.MethodPost, serverEndpoint, nil)