Various improvements to end-to-end testing capabilities

- start with clean etcd for local tests
- make some problematic test cases more verbose (and faster)
- handle the new testing scenario, when everything is running in-cluster, but the porch-server (that is running locally)

.vscode/launch.json

@@ -4,17 +4,6 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Launch test function",
-            "type": "go",
-            "request": "launch",
-            "mode": "test",
-            "program": "${workspaceFolder}/apiserver/pkg/e2e",
-            "args": [
-                "-test.run",
-                "TestE2E/PorchSuite/TestCloneIntoDeploymentRepository"
-            ]
-        },
         {
             "name": "Launch Server",
             "type": "go",
@@ -28,9 +17,27 @@
                 "--kubeconfig=${env:KUBECONFIG}",
                 "--cache-directory=${workspaceFolder}/.cache",
                 "--function-runner=172.18.255.201:9445",
-                "--repo-sync-frequency=60s"
+                "--repo-sync-frequency=60s",
+                // "--cert-dir=${workspaceFolder}/.build/pki/empty1",
+            ],
+            "cwd": "${workspaceFolder}",
+            "env": { 
+                "CERT_STORAGE_DIR": "${workspaceFolder}/.build/pki/empty2", 
+                "CERT_NAMESPACE": "porch-system" 
+            }
+        },
+        {
+            "name": "Launch test function",
+            "type": "go",
+            "request": "launch",
+            "mode": "test",
+            "program": "${workspaceFolder}/test/e2e",
+            "args": [
+                "-test.v",
+                "-test.run",
+                "TestE2E/PorchSuite/TestDeleteFinal"
             ],
-            "cwd": "${workspaceFolder}"
+            "env": { "E2E": "1"}
         },
         {
             "name": "Launch Func Client",

Makefile

@@ -104,6 +104,7 @@ stop:
 .PHONY: start-etcd
 start-etcd:
 	docker buildx build -t etcd --output=type=docker -f ./build/Dockerfile.etcd ./build
+	rm -rf $(BUILDDIR)/data/etcd || true
 	mkdir -p $(BUILDDIR)/data/etcd
 	docker stop etcd || true
 	docker rm etcd || true

deployments/local/porch-api-endpoints.yaml

@@ -9,8 +9,7 @@ subsets:
 - addresses:
   - ip: 172.17.0.1
   ports:
-  - appProtocol: https
-    name: api
+  - name: api
     port: 4443
     protocol: TCP
   - name: webhooks

scripts/clean-kind-only-e2e-test.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright 2022 The kpt and Nephio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stricter error handling
+set -e # Exit on error
+set -u # Must predefine variables
+set -o pipefail # Check errors in piped commands
+
+PORCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+
+cd $PORCH_DIR
+
+kind delete cluster --name dev || true
+kind create cluster --name dev 
+make run-in-kind-kpt IMAGE_TAG='test' KIND_CONTEXT_NAME='dev' KUBECONFIG=$KUBECONFIG
+kubectl rollout status deployment porch-controllers --namespace porch-system
+kubectl rollout status deployment porch-server --namespace porch-system
+kubectl rollout status deployment function-runner --namespace porch-system
+sleep 2
+E2E=1 go test -v ./test/e2e

test/e2e/e2e_test.go

@@ -1198,7 +1198,7 @@ func (t *PorchSuite) TestDeleteFromMain(ctx context.Context) {
 	// Register the repository
 	t.registerMainGitRepositoryF(ctx, repository)
 
-	// Create the first draft package
+	t.Logf("Create and approve package: %s", packageNameFirst)
 	createdFirst := t.createPackageDraftF(ctx, repository, packageNameFirst, workspace)
 
 	// Check the package exists
@@ -1214,7 +1214,7 @@ func (t *PorchSuite) TestDeleteFromMain(ctx context.Context) {
 
 	t.mustExist(ctx, client.ObjectKey{Namespace: t.namespace, Name: createdFirst.Name}, &pkgFirst)
 
-	// Create the second draft package
+	t.Logf("Create and approve package: %s", packageNameSecond)
 	createdSecond := t.createPackageDraftF(ctx, repository, packageNameSecond, workspace)
 
 	// Check the package exists
@@ -1230,42 +1230,49 @@ func (t *PorchSuite) TestDeleteFromMain(ctx context.Context) {
 
 	t.mustExist(ctx, client.ObjectKey{Namespace: t.namespace, Name: createdSecond.Name}, &pkgSecond)
 
-	// We need to wait for the sync for the "main" revisions to get created
-	time.Sleep(75 * time.Second)
-
-	var list porchapi.PackageRevisionList
-	t.ListE(ctx, &list, client.InNamespace(t.namespace))
-
+	t.Log("Wait for the 'main' revisions to get created")
 	var firstPkgRevFromMain porchapi.PackageRevision
 	var secondPkgRevFromMain porchapi.PackageRevision
-
-	for _, pkgrev := range list.Items {
-		if pkgrev.Spec.PackageName == packageNameFirst && pkgrev.Spec.Revision == "main" {
-			firstPkgRevFromMain = pkgrev
+	var list porchapi.PackageRevisionList
+	finalDeadline := time.Now().Add(5 * time.Minute)
+	for firstPkgRevFromMain.Spec.WorkspaceName == "" || secondPkgRevFromMain.Spec.WorkspaceName == "" {
+		if time.Now().After(finalDeadline) {
+			t.Fatal("'main' package revisions not created in time")
+			return
 		}
-		if pkgrev.Spec.PackageName == packageNameSecond && pkgrev.Spec.Revision == "main" {
-			secondPkgRevFromMain = pkgrev
+		time.Sleep(5 * time.Second)
+
+		t.ListE(ctx, &list, client.InNamespace(t.namespace))
+		for _, pkgrev := range list.Items {
+			if pkgrev.Spec.PackageName == packageNameFirst && pkgrev.Spec.Revision == "main" {
+				firstPkgRevFromMain = pkgrev
+				t.Logf("Found first package revision: %+v", firstPkgRevFromMain.Name)
+			}
+			if pkgrev.Spec.PackageName == packageNameSecond && pkgrev.Spec.Revision == "main" {
+				secondPkgRevFromMain = pkgrev
+				t.Logf("Found second package revision: %+v", secondPkgRevFromMain.Name)
+			}
 		}
 	}
 
-	// Propose deletion of both main packages
+	t.Log("Propose deletion of both main packages")
 	firstPkgRevFromMain.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed
 	t.UpdateApprovalF(ctx, &firstPkgRevFromMain, metav1.UpdateOptions{})
 	secondPkgRevFromMain.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed
 	t.UpdateApprovalF(ctx, &secondPkgRevFromMain, metav1.UpdateOptions{})
 
-	// Delete the first package revision from main
+	t.Log("Delete the first package revision from main")
 	t.DeleteE(ctx, &porchapi.PackageRevision{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: t.namespace,
 			Name:      firstPkgRevFromMain.Name,
 		},
 	})
 
-	// We need to wait for the sync
-	time.Sleep(75 * time.Second)
+	// t.Log("Wait for the sync")
+	// time.Sleep(75 * time.Second)
 
-	// Delete the second package revision from main
+	t.Log("Delete the second package revision from main")
 	t.DeleteE(ctx, &porchapi.PackageRevision{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: t.namespace,
@@ -1276,6 +1283,7 @@ func (t *PorchSuite) TestDeleteFromMain(ctx context.Context) {
 	// Propose and delete the original package revisions (cleanup)
 	t.ListE(ctx, &list, client.InNamespace(t.namespace))
 	for _, pkgrev := range list.Items {
+		t.Logf("Propose deletion and delete package revision: %s", pkgrev.Name)
 		pkgrev.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed
 		t.UpdateApprovalF(ctx, &pkgrev, metav1.UpdateOptions{})
 		t.DeleteE(ctx, &porchapi.PackageRevision{

test/e2e/suite.go

@@ -130,7 +130,7 @@ func (t *TestSuite) Initialize(ctx context.Context) {
 		t.clientset = cs
 	}
 
-	t.local = t.IsUsingDevPorch()
+	_, t.local = t.IsUsingDevPorch()
 
 	namespace := fmt.Sprintf("porch-test-%d", time.Now().UnixMicro())
 	t.CreateF(ctx, &coreapi.Namespace{
@@ -154,7 +154,7 @@ func (t *TestSuite) Initialize(ctx context.Context) {
 	})
 }
 
-func (t *TestSuite) IsUsingDevPorch() bool {
+func (t *TestSuite) IsUsingDevPorch() (bool, bool) {
 	porch := aggregatorv1.APIService{}
 	ctx := context.TODO()
 	t.GetF(ctx, client.ObjectKey{
@@ -166,13 +166,20 @@ func (t *TestSuite) IsUsingDevPorch() bool {
 		Name:      porch.Spec.Service.Name,
 	}, &service)
 
-	return len(service.Spec.Selector) == 0
+	isPorchLocal := len(service.Spec.Selector) == 0
+	areAllLocal := service.Spec.Type == coreapi.ServiceTypeExternalName
+	return isPorchLocal, areAllLocal
 }
 
 func (t *TestSuite) CreateGitRepo() GitConfig {
 	// Deploy Git server via k8s client.
 	t.Logf("creating git server in cluster")
-	return t.createInClusterGitServer(context.TODO(), t.IsUsingDevPorch())
+	isPorchLocal, areAllLocal := t.IsUsingDevPorch()
+	if areAllLocal {
+		return createLocalGitServer(t.T)
+	} else {
+		return t.createInClusterGitServer(context.TODO(), isPorchLocal)
+	}
 }
 
 type ErrorHandler func(format string, args ...interface{})
@@ -549,7 +556,7 @@ func (t *TestSuite) createInClusterGitServer(ctx context.Context, exposeByLoadBa
 	giveUp := time.Now().Add(time.Minute)
 
 	for {
-		time.Sleep(5 * time.Second)
+		time.Sleep(1 * time.Second)
 
 		var deployment appsv1.Deployment
 		t.GetF(ctx, deploymentKey, &deployment)
@@ -599,7 +606,7 @@ func (t *TestSuite) createInClusterGitServer(ctx context.Context, exposeByLoadBa
 	gitUrl := fmt.Sprintf("http://%s.%s.svc.cluster.local:8080", serviceKey.Name, serviceKey.Namespace)
 	giveUp = time.Now().Add(time.Minute)
 	for {
-		time.Sleep(5 * time.Second)
+		time.Sleep(1 * time.Second)
 		if time.Now().After(giveUp) {
 			t.Fatalf("git-server-service not ready on time")
 			return GitConfig{}