Merge pull request #57 from nokia/local-e2e-test

Make end-to-end tests work with a local porch-server

.gitignore

@@ -24,6 +24,7 @@ coverage_unit.html
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
+__debug*
 
 ### VisualStudioCode Patch ###
 # Ignore all local history of files

.vscode/launch.json

@@ -4,17 +4,6 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Launch test function",
-            "type": "go",
-            "request": "launch",
-            "mode": "test",
-            "program": "${workspaceFolder}/apiserver/pkg/e2e",
-            "args": [
-                "-test.run",
-                "TestE2E/PorchSuite/TestCloneIntoDeploymentRepository"
-            ]
-        },
         {
             "name": "Launch Server",
             "type": "go",
@@ -23,14 +12,31 @@
             "program": "${workspaceFolder}/cmd/porch/main.go",
             "args": [
                 "--secure-port=4443",
-                "--v=7",
-                "--standalone-debug-mode",
+                // "--v=7",
+                // "--standalone-debug-mode",
                 "--kubeconfig=${env:KUBECONFIG}",
                 "--cache-directory=${workspaceFolder}/.cache",
                 "--function-runner=172.18.255.201:9445",
                 "--repo-sync-frequency=60s"
             ],
             "cwd": "${workspaceFolder}",
+            "env": { 
+                "CERT_STORAGE_DIR": "${workspaceFolder}/.build/pki/tmp", 
+                "WEBHOOK_HOST": "localhost" 
+            }
+        },
+        {
+            "name": "Launch test function",
+            "type": "go",
+            "request": "launch",
+            "mode": "test",
+            "program": "${workspaceFolder}/test/e2e",
+            "args": [
+                "-test.v",
+                "-test.run",
+                "TestE2E/PorchSuite/TestGitRepositoryWithReleaseTagsAndDirectory"
+            ],
+            "env": { "E2E": "1"}
         },
         {
             "name": "Launch Func Client",

Makefile

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 MYGOBIN := $(shell go env GOPATH)/bin
-KUBECONFIG=$(CURDIR)/deployments/local/kubeconfig
 BUILDDIR=$(CURDIR)/.build
 CACHEDIR=$(CURDIR)/.cache
 DEPLOYCONFIGDIR=$(BUILDDIR)/deploy
@@ -104,6 +103,7 @@ stop:
 .PHONY: start-etcd
 start-etcd:
 	docker buildx build -t etcd --output=type=docker -f ./build/Dockerfile.etcd ./build
+	rm -rf $(BUILDDIR)/data/etcd || true
 	mkdir -p $(BUILDDIR)/data/etcd
 	docker stop etcd || true
 	docker rm etcd || true
@@ -161,7 +161,12 @@ tidy:
 
 .PHONY: test-e2e
 test-e2e:
-	E2E=1 go test -v -race --count=1 -failfast ./test/e2e 
+	E2E=1 go test -v -failfast ./test/e2e 
+	E2E=1 go test -v -failfast ./test/e2e/cli
+
+.PHONY: test-e2e-clean
+test-e2e-clean:
+	./scripts/clean-kind-only-e2e-test.sh
 
 .PHONY: configure-git
 configure-git:
@@ -177,13 +182,13 @@ PORCHCTL = $(BUILDDIR)/porchctl
 
 .PHONY: run-local
 run-local: porch
-	KUBECONFIG=$(KUBECONFIG) kubectl apply -f deployments/local/localconfig.yaml
-	KUBECONFIG=$(KUBECONFIG) kubectl apply -f api/porchconfig/v1alpha1/
-	KUBECONFIG=$(KUBECONFIG) kubectl apply -f internal/api/porchinternal/v1alpha1/
+	KUBECONFIG=$(CURDIR)/deployments/local/kubeconfig kubectl apply -f deployments/local/localconfig.yaml
+	KUBECONFIG=$(CURDIR)/deployments/local/kubeconfig kubectl apply -f api/porchconfig/v1alpha1/
+	KUBECONFIG=$(CURDIR)/deployments/local/kubeconfig kubectl apply -f internal/api/porchinternal/v1alpha1/
 	$(PORCH) \
 	--secure-port 9443 \
 	--standalone-debug-mode \
-	--kubeconfig="$(KUBECONFIG)" \
+	--kubeconfig="$(CURDIR)/deployments/local/kubeconfig" \
 	--cache-directory="$(CACHEDIR)" \
 	--function-runner 192.168.8.202:9445 \
 	--repo-sync-frequency=60s

build/Dockerfile.apiserver

@@ -15,14 +15,13 @@
 FROM golang:1.22.2-bookworm as builder
 
 WORKDIR /workspace/src
-RUN git clone https://github.com/kubernetes/kubernetes --branch v1.23.2 --depth=1
+RUN git clone https://github.com/kubernetes/kubernetes --branch v1.30.1 --depth=1
 WORKDIR /workspace/src/kubernetes
 RUN apt-get update && apt-get install --yes rsync
-RUN make generated_files
-RUN CGO_ENABLED=0 go build -o /workspace/artifacts/kube-apiserver ./cmd/kube-apiserver
+RUN make kube-apiserver
 
 FROM gcr.io/distroless/static
-COPY --from=builder /workspace/artifacts/kube-apiserver /kube-apiserver
+COPY --from=builder /workspace/src/kubernetes/_output/local/bin/linux/amd64/kube-apiserver /kube-apiserver
 
 #USER 65532:65532
 

pkg/apiserver/apiserver.go

@@ -284,14 +284,12 @@ func (c completedConfig) New() (*PorchServer, error) {
 
 func (s *PorchServer) Run(ctx context.Context) error {
 	porch.RunBackground(ctx, s.coreClient, s.cache)
-	webhookNs, found := os.LookupEnv("CERT_NAMESPACE")
-	if !found || strings.TrimSpace(webhookNs) == "" {
-		webhookNs = "porch-system"
-	}
 
+	// TODO: Reconsider if the existence of CERT_STORAGE_DIR was a good inidcator for webhook setup,
+	// but for now we keep backward compatiblity
 	certStorageDir, found := os.LookupEnv("CERT_STORAGE_DIR")
 	if found && strings.TrimSpace(certStorageDir) != "" {
-		if err := setupWebhooks(ctx, webhookNs, certStorageDir); err != nil {
+		if err := setupWebhooks(ctx); err != nil {
 			klog.Errorf("%v\n", err)
 			return err
 		}

pkg/apiserver/webhooks.go

@@ -30,6 +30,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/nephio-project/porch/api/porch/v1alpha1"
@@ -47,29 +48,76 @@ import (
 )
 
 const (
-	webhookServicePort = 8443
-	serverEndpoint     = "/validate-deletion"
+	serverEndpoint = "/validate-deletion"
 )
 
-func setupWebhooks(ctx context.Context, webhookNs string, certStorageDir string) error {
-	caBytes, err := createCerts(webhookNs, certStorageDir)
+type WebhookType string
+
+const (
+	WebhookTypeService WebhookType = "service"
+	WebhookTypeUrl     WebhookType = "url"
+)
+
+// WebhookConfig defines the configuration for the PackageRevision deletion webhook
+type WebhookConfig struct {
+	Type             WebhookType
+	ServiceName      string // only used if Type == WebhookTypeService
+	ServiceNamespace string // only used if Type == WebhookTypeService
+	Host             string // only used if Type == WebhookTypeUrl
+	Path             string
+	Port             int32
+	CertStorageDir   string
+}
+
+func NewWebhookConfig() *WebhookConfig {
+	var cfg WebhookConfig
+	// NOTE: CERT_NAMESPACE is supported for backward compatibility.
+	// TODO: We may consider using only WEBHOOK_SERVICE_NAMESPACE instead.
+	if hasEnv("CERT_NAMESPACE") ||
+		hasEnv("WEBHOOK_SERVICE_NAME") ||
+		hasEnv("WEBHOOK_SERVICE_NAMESPACE") ||
+		!hasEnv("WEBHOOK_HOST") {
+
+		cfg.Type = WebhookTypeService
+		cfg.ServiceName = getEnv("WEBHOOK_SERVICE_NAME", "api")
+		cfg.ServiceNamespace = getEnv("WEBHOOK_SERVICE_NAMESPACE", "porch-system")
+		cfg.ServiceNamespace = getEnv("CERT_NAMESPACE", cfg.ServiceNamespace)
+		cfg.Host = fmt.Sprintf("%s.%s.svc", cfg.ServiceName, cfg.ServiceNamespace)
+	} else {
+		cfg.Type = WebhookTypeUrl
+		cfg.Host = getEnv("WEBHOOK_HOST", "localhost")
+	}
+	cfg.Path = serverEndpoint
+	cfg.Port = getEnvInt32("WEBHOOK_PORT", 8443)
+	cfg.CertStorageDir = getEnv("CERT_STORAGE_DIR", "/tmp/cert")
+	return &cfg
+}
+
+func setupWebhooks(ctx context.Context) error {
+	cfg := NewWebhookConfig()
+	caBytes, err := createCerts(cfg)
 	if err != nil {
 		return err
 	}
-	if err := createValidatingWebhook(ctx, webhookNs, caBytes); err != nil {
+	if err := createValidatingWebhook(ctx, cfg, caBytes); err != nil {
 		return err
 	}
-	if err := runWebhookServer(certStorageDir); err != nil {
+	if err := runWebhookServer(cfg); err != nil {
 		return err
 	}
 	return nil
 }
 
-func createCerts(webhookNs string, certStorageDir string) ([]byte, error) {
-	klog.Infoln("creating self-signing TLS cert and key with namespace " + webhookNs + " in directory " + certStorageDir)
-	dnsNames := []string{"api",
-		"api." + webhookNs, "api." + webhookNs + ".svc"}
-	commonName := "api." + webhookNs + ".svc"
+func createCerts(cfg *WebhookConfig) ([]byte, error) {
+	klog.Infof("creating self-signing TLS cert and key for %q in directory %s", cfg.Host, cfg.CertStorageDir)
+	commonName := cfg.Host
+	dnsNames := []string{commonName}
+	if cfg.Type == WebhookTypeService {
+		dnsNames = append(dnsNames, cfg.ServiceName)
+		dnsNames = append(dnsNames, fmt.Sprintf("%s.%s", cfg.ServiceName, cfg.ServiceNamespace))
+		dnsNames = append(dnsNames, fmt.Sprintf("%s.%s.svc", cfg.ServiceName, cfg.ServiceNamespace))
+		dnsNames = append(dnsNames, fmt.Sprintf("%s.%s.svc.cluster.local", cfg.ServiceName, cfg.ServiceNamespace))
+	}
 
 	var caPEM, serverCertPEM, serverPrivateKeyPEM *bytes.Buffer
 	// CA config
@@ -134,15 +182,15 @@ func createCerts(webhookNs string, certStorageDir string) ([]byte, error) {
 		Bytes: x509.MarshalPKCS1PrivateKey(serverPrivateKey),
 	})
 
-	err = os.MkdirAll(certStorageDir, 0666)
+	err = os.MkdirAll(cfg.CertStorageDir, 0777)
 	if err != nil {
 		return nil, err
 	}
-	err = WriteFile(filepath.Join(certStorageDir, "tls.crt"), serverCertPEM.Bytes())
+	err = WriteFile(filepath.Join(cfg.CertStorageDir, "tls.crt"), serverCertPEM.Bytes())
 	if err != nil {
 		return nil, err
 	}
-	err = WriteFile(filepath.Join(certStorageDir, "tls.key"), serverPrivateKeyPEM.Bytes())
+	err = WriteFile(filepath.Join(cfg.CertStorageDir, "tls.key"), serverPrivateKeyPEM.Bytes())
 	if err != nil {
 		return nil, err
 	}
@@ -165,23 +213,20 @@ func WriteFile(filepath string, c []byte) error {
 	return nil
 }
 
-func createValidatingWebhook(ctx context.Context, webhookNs string, caCert []byte) error {
-	klog.Infoln("Creating validating webhook with namespace " + webhookNs)
+func createValidatingWebhook(ctx context.Context, cfg *WebhookConfig, caCert []byte) error {
+
+	klog.Infof("Creating validating webhook for %s:%d", cfg.Host, cfg.Port)
 
-	cfg := ctrl.GetConfigOrDie()
-	kubeClient, err := kubernetes.NewForConfig(cfg)
+	kubeConfig := ctrl.GetConfigOrDie()
+	kubeClient, err := kubernetes.NewForConfig(kubeConfig)
 	if err != nil {
 		return fmt.Errorf("failed to setup kubeClient: %v", err)
 	}
 
 	var (
-		webhookNamespace  = webhookNs
 		validationCfgName = "packagerev-deletion-validating-webhook"
-		webhookService    = "api"
-		path              = serverEndpoint
 		fail              = admissionregistrationv1.Fail
 		none              = admissionregistrationv1.SideEffectClassNone
-		port              = int32(webhookServicePort)
 	)
 
 	validateConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{
@@ -192,12 +237,6 @@ func createValidatingWebhook(ctx context.Context, webhookNs string, caCert []byt
 			Name: "packagerevdeletion.google.com",
 			ClientConfig: admissionregistrationv1.WebhookClientConfig{
 				CABundle: caCert, // CA bundle created earlier
-				Service: &admissionregistrationv1.ServiceReference{
-					Name:      webhookService,
-					Namespace: webhookNamespace,
-					Path:      &path,
-					Port:      &port,
-				},
 			},
 			Rules: []admissionregistrationv1.RuleWithOperations{{Operations: []admissionregistrationv1.OperationType{
 				admissionregistrationv1.Delete},
@@ -212,6 +251,20 @@ func createValidatingWebhook(ctx context.Context, webhookNs string, caCert []byt
 			FailurePolicy:           &fail,
 		}},
 	}
+	switch cfg.Type {
+	case WebhookTypeService:
+		validateConfig.Webhooks[0].ClientConfig.Service = &admissionregistrationv1.ServiceReference{
+			Name:      cfg.ServiceName,
+			Namespace: cfg.ServiceNamespace,
+			Path:      &cfg.Path,
+			Port:      &cfg.Port,
+		}
+	case WebhookTypeUrl:
+		url := fmt.Sprintf("https://%s:%d%s", cfg.Host, cfg.Port, cfg.Path)
+		validateConfig.Webhooks[0].ClientConfig.URL = &url
+	default:
+		return fmt.Errorf("invalid webhook type: %s", cfg.Type)
+	}
 
 	if err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, validationCfgName, metav1.DeleteOptions{}); err != nil {
 		klog.Error("failed to delete existing webhook: %w", err)
@@ -226,18 +279,18 @@ func createValidatingWebhook(ctx context.Context, webhookNs string, caCert []byt
 	return nil
 }
 
-func runWebhookServer(certStorageDir string) error {
-	certFile := filepath.Join(certStorageDir, "tls.crt")
-	keyFile := filepath.Join(certStorageDir, "tls.key")
+func runWebhookServer(cfg *WebhookConfig) error {
+	certFile := filepath.Join(cfg.CertStorageDir, "tls.crt")
+	keyFile := filepath.Join(cfg.CertStorageDir, "tls.key")
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		return err
 	}
 	klog.Infoln("Starting webhook server")
-	http.HandleFunc(serverEndpoint, validateDeletion)
+	http.HandleFunc(cfg.Path, validateDeletion)
 	server := http.Server{
-		Addr: fmt.Sprintf(":%d", webhookServicePort),
+		Addr: fmt.Sprintf(":%d", cfg.Port),
 		TLSConfig: &tls.Config{
 			Certificates: []tls.Certificate{cert},
 		},
@@ -369,3 +422,28 @@ func writeErr(errMsg string, w *http.ResponseWriter) {
 		klog.Errorf("could not write error message: %v", err)
 	}
 }
+
+func hasEnv(key string) bool {
+	_, found := os.LookupEnv(key)
+	return found
+}
+
+func getEnv(key string, defaultValue string) string {
+	value, found := os.LookupEnv(key)
+	if !found {
+		return defaultValue
+	}
+	return value
+}
+
+func getEnvInt32(key string, defaultValue int32) int32 {
+	value, found := os.LookupEnv(key)
+	if !found {
+		return defaultValue
+	}
+	i64, err := strconv.ParseInt(value, 10, 32)
+	if err != nil {
+		panic("could not parse int32 from environment variable: " + key)
+	}
+	return int32(i64) // this is safe because of the size parameter of the ParseInt call
+}