Fix propagate warning state (kyma-project#834)

* Include Deleting&Warning State in state mapping for correct propagation * test gh actions e2e setup * Remove ssh step * Check how checkout@3 works * Check how checkout@3 works * Check how checkout@3 works * Check how checkout@3 works * Check how checkout@3 works * Check how checkout@3 works * Check how checkout@3 works * Check how checkout@3 works * Check how checkout@3 works * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Test status propagation * Remove duplicate definition of error var * fetch crds for debug * Make sure kcp context is used * Use matrix to speed up pipeline runtime * Use matrix to speed up pipeline runtime * Use matrix to speed up pipeline runtime
LeelaChacha · Sep 4, 2023 · 70913eb · 70913eb
1 parent efb1c76
commit 70913eb
Show file tree

Hide file tree

Showing 8 changed files with 212 additions and 33 deletions.
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -27,14 +27,15 @@ jobs:
   e2e-integration:
     strategy:
       matrix:
+        e2e-test: ["test-watcher-e2e", "test-kyma-deletion-e2e", "test-status-propagation-e2e"]
         cli-stability: [ "unstable" ]
     name: "Run E2E tests"
     needs: [wait-for-img]
     runs-on: ubuntu-latest
     env:
       K3D_VERSION: v5.4.7
       ISTIO_VERSION: 1.17.1
-      CM_VERSION: v1.12.0
+      CM_VERSION: v1.12.3
       KLM_VERSION_TAG: latest
       KLM_IMAGE_REPO: prod
     steps:
@@ -97,8 +98,10 @@ jobs:
         run: |
           kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$CM_VERSION/cert-manager.yaml
           cmctl check api --wait=2m
-      - name: Checkout repo
+      - name: Checkout lifecycle-manager
         uses: actions/checkout@v3
+        with:
+          path: ./lifecycle-manager/
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
@@ -109,6 +112,7 @@ jobs:
           echo "KLM_VERSION_TAG=PR-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
           echo "KLM_IMAGE_REPO=dev" >> $GITHUB_ENV
       - name: Deploy LM local testing kustomize
+        working-directory: ./lifecycle-manager
         run: |
           maxRetry=5
           for retry in $(seq 1 $maxRetry)
@@ -124,9 +128,29 @@ jobs:
               exit 1
             fi
           done
-      - name: Run Watcher E2E Tests
-        run: |
-          make -C tests/e2e_test test-watcher-e2e
-      - name: Run Kyma CR Deletion Test
-        run: |
-          make -C tests/e2e_test test-kyma-deletion-e2e
+      - name: Checkout template-operator
+        uses: actions/checkout@v3
+        if: ${{ matrix.e2e-test == 'test-status-propagation-e2e' }}
+        with:
+          repository: kyma-project/template-operator
+          path: ./template-operator/
+      - name: Create Template Operator Module
+        working-directory: ./template-operator
+        if: ${{ matrix.e2e-test == 'test-status-propagation-e2e' }}
+        run: |
+          pushd config/default
+          echo "- op: replace
+            path: /spec/template/spec/containers/0/args/1
+            value: --final-state=Warning" >> warning_patch.yaml
+          cat warning_patch.yaml
+          kustomize edit add patch --path warning_patch.yaml --kind Deployment
+          popd
+          kyma alpha create module --kubebuilder-project  --channel=regular --name kyma.project.io/module/template-operator --version 1.1.1 --path . --registry k3d-kcp-registry:5111 --insecure --module-archive-version-overwrite /
+          sed -i 's/k3d-kcp-registry:5111/k3d-kcp-registry:5000/g' ./template.yaml
+          kubectl config use-context k3d-kcp
+          kubectl get crds
+          kubectl apply -f template.yaml
+      - name: Run ${{ matrix.e2e-test }}
+        working-directory: ./lifecycle-manager
+        run: |
+          make -C tests/e2e_test ${{ matrix.e2e-test }}
diff --git a/internal/manifest/ready_check.go b/internal/manifest/ready_check.go
@@ -187,6 +187,16 @@ func parseStateChecks(manifest *v1beta2.Manifest) ([]*v1beta2.CustomStateCheck,
 				Value:       string(v1beta2.StateError),
 				MappedState: v1beta2.StateError,
 			},
+			{
+				JSONPath:    customResourceStatePath,
+				Value:       string(v1beta2.StateDeleting),
+				MappedState: v1beta2.StateDeleting,
+			},
+			{
+				JSONPath:    customResourceStatePath,
+				Value:       string(v1beta2.StateWarning),
+				MappedState: v1beta2.StateWarning,
+			},
 		}, false, nil
 	}
 	var stateCheck []*v1beta2.CustomStateCheck

diff --git a/tests/e2e_test/Makefile b/tests/e2e_test/Makefile
@@ -32,12 +32,16 @@ help: ## Display this help.
 ##@ E2E Tests
 
 .PHONY: test
-test: test-watcher-e2e test-kyma-deletion-e2e
+test: test-watcher-e2e test-kyma-deletion-e2e test-status-propagation-e2e
 
 .PHONY: test-kyma-deletion-e2e
-test-kyma-deletion-e2e:
+test-kyma-deletion-e2e: ## Runs the Kyma Deletion E2E Test
 	go test --tags=deletion_e2e -ginkgo.vv
 
 .PHONY: test-watcher-e2e
 test-watcher-e2e: ## Runs the Watcher E2E Test
 	go test --tags=watcher_e2e -ginkgo.vv
+
+.PHONY: test-status-propagation-e2e
+test-status-propagation-e2e: ## Runs the Status Propagation E2E Test
+	go test --tags=status_propagation_e2e -ginkgo.vv
diff --git a/tests/e2e_test/kyma_deletion_test.go b/tests/e2e_test/kyma_deletion_test.go
@@ -3,22 +3,16 @@
 package e2e_test
 
 import (
-	"context"
-	"errors"
 	"os/exec"
 	"time"
 
 	"github.com/kyma-project/lifecycle-manager/api/v1beta2"
 	"github.com/kyma-project/lifecycle-manager/pkg/testutils"
-	"github.com/kyma-project/lifecycle-manager/pkg/util"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/api/meta"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var errKymaNotDeleted = errors.New("kyma CR not deleted")
-
 const (
 	timeout       = 10 * time.Second
 	statusTimeout = 2 * time.Minute
@@ -119,20 +113,9 @@ var _ = Describe("KCP Kyma CR should be deleted successfully when SKR cluster ge
 		})
 
 		It("Kyma CR should be removed", func() {
-			Eventually(checkKCPKymaCRDeleted, timeout, interval).
+			Eventually(CheckKCPKymaCRDeleted, timeout, interval).
 				WithContext(ctx).
 				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient).
 				Should(Succeed())
 		})
 	})
-
-func checkKCPKymaCRDeleted(ctx context.Context,
-	kymaName string, kymaNamespace string, k8sClient client.Client,
-) error {
-	kyma := &v1beta2.Kyma{}
-	err := k8sClient.Get(ctx, client.ObjectKey{Name: kymaName, Namespace: kymaNamespace}, kyma)
-	if util.IsNotFound(err) {
-		return nil
-	}
-	return errKymaNotDeleted
-}
diff --git a/tests/e2e_test/status_propagation_test.go b/tests/e2e_test/status_propagation_test.go
@@ -0,0 +1,97 @@
+//go:build status_propagation_e2e
+
+package e2e_test
+
+import (
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+
+	"github.com/kyma-project/lifecycle-manager/api/v1beta2"
+	"github.com/kyma-project/lifecycle-manager/pkg/testutils"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	timeout       = 10 * time.Second
+	statusTimeout = 2 * time.Minute
+	interval      = 1 * time.Second
+)
+
+var _ = Describe("Enable Template Operator, Kyma CR should have status `Warning`",
+	Ordered, func() {
+		channel := "regular"
+		kyma := testutils.NewKymaForE2E("kyma-sample", "kcp-system", channel)
+		GinkgoWriter.Printf("kyma before create %v\n", kyma)
+		remoteNamespace := "kyma-system"
+
+		BeforeAll(func() {
+			// make sure we can list Kymas to ensure CRDs have been installed
+			err := controlPlaneClient.List(ctx, &v1beta2.KymaList{})
+			Expect(meta.IsNoMatchError(err)).To(BeFalse())
+		})
+
+		It("Should create empty Kyma CR on remote cluster", func() {
+			Eventually(CreateKymaSecret, timeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient).
+				Should(Succeed())
+			Eventually(controlPlaneClient.Create, timeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma).
+				Should(Succeed())
+			By("verifying kyma is ready")
+			Eventually(CheckKymaIsInState, statusTimeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient, v1beta2.StateReady).
+				Should(Succeed())
+			By("verifying remote kyma is ready")
+			Eventually(CheckRemoteKymaCR, statusTimeout, interval).
+				WithContext(ctx).
+				WithArguments(remoteNamespace, []v1beta2.Module{}, runtimeClient, v1beta2.StateReady).
+				Should(Succeed())
+		})
+
+		It("Should enable Template Operator and Kyma should result in Warning status", func() {
+			By("Enabling Template Operator")
+			Eventually(EnableModule, timeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), "template-operator", "regular", controlPlaneClient).
+				Should(Succeed())
+			By("Checking state of kyma")
+			Eventually(CheckKymaIsInState, statusTimeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient, v1beta2.StateWarning).
+				Should(Succeed())
+		})
+
+		It("Should disable Template Operator and Kyma should result in Ready status", func() {
+			By("Disabling Template Operator")
+			Eventually(DisableModule, timeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), "template-operator", controlPlaneClient).
+				Should(Succeed())
+			By("Checking state of kyma")
+			Eventually(CheckKymaIsInState, statusTimeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient, v1beta2.StateReady).
+				Should(Succeed())
+		})
+
+		It("Should delete KCP Kyma", func() {
+			By("Deleting KCP Kyma")
+			Eventually(controlPlaneClient.Delete, statusTimeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma).
+				Should(Succeed())
+		})
+
+		It("Kyma CR should be removed", func() {
+			Eventually(CheckKCPKymaCRDeleted, timeout, interval).
+				WithContext(ctx).
+				WithArguments(kyma.GetName(), kyma.GetNamespace(), controlPlaneClient).
+				Should(Succeed())
+		})
+
+	})
diff --git a/tests/e2e_test/suite_test.go b/tests/e2e_test/suite_test.go
@@ -1,4 +1,4 @@
-//go:build watcher_e2e || deletion_e2e
+//go:build watcher_e2e || deletion_e2e || status_propagation_e2e
 
 //nolint:gochecknoglobals
 package e2e_test

diff --git a/tests/e2e_test/utils_test.go b/tests/e2e_test/utils_test.go
@@ -1,4 +1,4 @@
-//go:build watcher_e2e || deletion_e2e
+//go:build watcher_e2e || deletion_e2e || status_propagation_e2e
 
 package e2e_test
 
@@ -21,6 +21,7 @@ import (
 var (
 	errKymaNotInExpectedState = errors.New("kyma CR not in expected state")
 	errModuleNotExisting      = errors.New("module does not exists in KymaCR")
+	errKymaNotDeleted         = errors.New("kyma CR not deleted")
 )
 
 const (
@@ -40,12 +41,29 @@ func CheckKymaIsInState(ctx context.Context,
 	}
 	GinkgoWriter.Printf("kyma %v\n", kyma)
 	if kyma.Status.State != expectedState {
-		return fmt.Errorf("%w: expect %s, but in %s",
-			errKymaNotInExpectedState, expectedState, kyma.Status.State)
+
+		logmsg, err := getManifestCRs(ctx, k8sClient)
+		if err != nil {
+			return fmt.Errorf("error getting manifest crs %s", err.Error())
+		}
+		return fmt.Errorf("%w: expect %s, but in %s. Kyma CR: %#v, Manifest CRs: %s",
+			errKymaNotInExpectedState, expectedState, kyma.Status.State, kyma, logmsg)
 	}
 	return nil
 }
 
+func getManifestCRs(ctx context.Context, k8sClient client.Client) (string, error) {
+	manifests := &v1beta2.ManifestList{}
+	if err := k8sClient.List(ctx, manifests); err != nil {
+		return "", err
+	}
+	logmsg := ""
+	for _, m := range manifests.Items {
+		logmsg += fmt.Sprintf("Manifest CR: %#v", m)
+	}
+	return logmsg, nil
+}
+
 func CreateKymaSecret(ctx context.Context, kymaName, kymaNamespace string, k8sClient client.Client) error {
 	patchedRuntimeConfig := strings.ReplaceAll(string(*runtimeConfig), localHostname, k3dHostname)
 	secret := &corev1.Secret{
@@ -100,3 +118,47 @@ func DeleteKymaSecret(ctx context.Context, kymaName, kymaNamespace string, k8sCl
 	Expect(err).ToNot(HaveOccurred())
 	return k8sClient.Delete(ctx, secret)
 }
+
+func CheckKCPKymaCRDeleted(ctx context.Context,
+	kymaName string, kymaNamespace string, k8sClient client.Client,
+) error {
+	kyma := &v1beta2.Kyma{}
+	err := k8sClient.Get(ctx, client.ObjectKey{Name: kymaName, Namespace: kymaNamespace}, kyma)
+	if util.IsNotFound(err) {
+		return nil
+	}
+	return errKymaNotDeleted
+}
+
+func EnableModule(ctx context.Context, kymaName, kymaNamespace, moduleName, moduleChannel string, k8sClient client.Client) error {
+	kyma := &v1beta2.Kyma{}
+	if err := k8sClient.Get(ctx, client.ObjectKey{Name: kymaName, Namespace: kymaNamespace}, kyma); err != nil {
+		return err
+	}
+	GinkgoWriter.Printf("kyma %v\n", kyma)
+	kyma.Spec.Modules = append(kyma.Spec.Modules, v1beta2.Module{
+		Name:    moduleName,
+		Channel: moduleChannel,
+	})
+	return k8sClient.Update(ctx, kyma)
+}
+
+func DisableModule(ctx context.Context, kymaName, kymaNamespace, moduleName string, k8sClient client.Client) error {
+	kyma := &v1beta2.Kyma{}
+	if err := k8sClient.Get(ctx, client.ObjectKey{Name: kymaName, Namespace: kymaNamespace}, kyma); err != nil {
+		return err
+	}
+	GinkgoWriter.Printf("kyma %v\n", kyma)
+
+	for i, module := range kyma.Spec.Modules {
+		if module.Name == moduleName {
+			kyma.Spec.Modules = removeModuleWithIndex(kyma.Spec.Modules, i)
+			break
+		}
+	}
+	return k8sClient.Update(ctx, kyma)
+}
+
+func removeModuleWithIndex(s []v1beta2.Module, index int) []v1beta2.Module {
+	return append(s[:index], s[index+1:]...)
+}
diff --git a/tests/e2e_test/watcher_test.go b/tests/e2e_test/watcher_test.go
@@ -48,7 +48,6 @@ var (
 	errPodNotFound               = errors.New("could not find pod")
 	errWatcherDeploymentNotReady = errors.New("watcher Deployment is not ready")
 	errLogNotFound               = errors.New("logMsg was not found in log")
-	errKymaNotDeleted            = errors.New("kyma CR not deleted")
 )
 
 var _ = Describe("Kyma CR change on runtime cluster triggers new reconciliation using the Watcher",