From 91926bee14d8ef79739b4c2126877f4dd0e33b5e Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Mon, 19 Dec 2022 17:49:39 +0000 Subject: [PATCH 01/13] Initial e2e workflow --- .github/workflows/e2e.yaml | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/e2e.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..e8764f1 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,51 @@ +name: e2e + +on: + push: + pull_request: + +env: + GO_VERSION: 1.18 + GOFLAGS: -mod=readonly + +jobs: + build: + name: Build project + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Cache Go module dependencies + id: cache-go-module-dependencies + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: go-mod-cache-${{ runner.os }}-${{ env.GO_VERSION }}-${{ hashFiles('go.sum') }} + restore-keys: | + go-mod-cache-${{ runner.os }}-${{ env.GO_VERSION }} + go-mod-cache-${{ runner.os }} + go-mod-cache + + - name: Set Git refname + id: set-git-refname + run: echo ::set-output name=git_refname::$(echo "${{ github.ref }}" | sed -r 's@refs/(heads|pull|tags)/@@g' ) + + - name: Cache build dependencies + id: cache-build-dependencies + uses: actions/cache@v2 + with: + path: bin/ + key: build-deps-v2-${{ steps.set-git-refname.outputs.git_refname }}-{{ hashFiles('scripts/download-deps.sh') }} + restore-keys: | + build-deps-v2-${{ steps.set-git-refname.outputs.git_refname }} + build-deps-v2 + + - name: Run build + run: make docker-build From 40f38f192957e93de4f0914aa10c99e5f2dc2694 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Mon, 19 Dec 2022 20:40:32 +0000 Subject: [PATCH 02/13] Add kind clusters to e2e job --- .github/workflows/e2e.yaml | 26 ++++++++++++++++++++++++++ test/e2e/cluster-config.yaml | 6 ++++++ 2 files changed, 32 insertions(+) create mode 100644 test/e2e/cluster-config.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e8764f1..b38eb28 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -16,6 +16,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v2 + with: + path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Setup Go uses: actions/setup-go@v2 @@ -49,3 +51,27 @@ jobs: - name: Run build run: make docker-build + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Get kind + run: go get sigs.k8s.io/kind@v0.11.1 + - name: Create kind clusters + run: | + if [[ $KUBERNETES_VERSION=="" ]]; then + KUBERNETES_VERSION="v1.22.1" + fi + for (( i = 1; i <= 2; i++ )); do + kind create cluster --name "kind-${i}" --config test/e2e/cluster-config.yaml --image="kindest/node:$KUBERNETES_VERSION" + configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} + kind get kubeconfig --name "kind-${i}" > ${configPath} + echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV + echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV + done + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: multicluster tests + run: | + kubectl get pods -A --kubeconfig $KUBECONFIG1 + kubectl get pods -A --kubeconfig $KUBECONFIG2 + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Cleanup resources + if: ${{ success() || failure() || cancelled() }} + run: kind delete clusters $(kind get clusters) diff --git a/test/e2e/cluster-config.yaml b/test/e2e/cluster-config.yaml new file mode 100644 index 0000000..f57c67e --- /dev/null +++ b/test/e2e/cluster-config.yaml @@ -0,0 +1,6 @@ +--- +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker From 10c5b8d34fdb238544b5788c16f7dd012dcefa80 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Mon, 19 Dec 2022 20:56:24 +0000 Subject: [PATCH 03/13] go install kind and get helm --- .github/workflows/e2e.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b38eb28..f58c27e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -34,7 +34,11 @@ jobs: go-mod-cache-${{ runner.os }}-${{ env.GO_VERSION }} go-mod-cache-${{ runner.os }} go-mod-cache - + - name: Set go env + run: | + echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + echo GO111MODULE=on >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - name: Set Git refname id: set-git-refname run: echo ::set-output name=git_refname::$(echo "${{ github.ref }}" | sed -r 's@refs/(heads|pull|tags)/@@g' ) @@ -53,7 +57,11 @@ jobs: run: make docker-build working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Get kind - run: go get sigs.k8s.io/kind@v0.11.1 + run: go install sigs.k8s.io/kind@v0.11.1 + - name: Get helm + run: | + wget -q -O - https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz | tar -C /tmp -xzf - + mv /tmp/linux-amd64/helm $GITHUB_WORKSPACE/bin && chmod a+x $GITHUB_WORKSPACE/bin/helm - name: Create kind clusters run: | if [[ $KUBERNETES_VERSION=="" ]]; then From 4cec3530a82daf50fa12067aaa002e55ed5d0d00 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Mon, 19 Dec 2022 21:14:16 +0000 Subject: [PATCH 04/13] e2e: Install cluster-registry-controller, sync, and check each kind cluster --- .github/workflows/e2e.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f58c27e..6cf63ce 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -78,7 +78,24 @@ jobs: - name: multicluster tests run: | kubectl get pods -A --kubeconfig $KUBECONFIG1 + helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c1 --kubeconfig $KUBECONFIG1 + kubectl get pods -A --kubeconfig $KUBECONFIG1 + kubectl get pods -A --kubeconfig $KUBECONFIG2 + helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c2 --kubeconfig $KUBECONFIG2 kubectl get pods -A --kubeconfig $KUBECONFIG2 + sleep 30 + kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 + kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 + echo "Getting cluster c1" + kubectl get cluster c1 --kubeconfig $KUBECONFIG1 -o yaml | kubectl apply --kubeconfig $KUBECONFIG2 -f - + echo "Getting secret c1" + kubectl get secret -n cluster-registry c1 --kubeconfig $KUBECONFIG1 -o yaml | kubectl apply --kubeconfig $KUBECONFIG2 -f - + echo "Getting cluster c2" + kubectl get cluster c2 --kubeconfig $KUBECONFIG2 -o yaml | kubectl apply --kubeconfig $KUBECONFIG1 -f - + echo "Getting secret c2" + kubectl get secret -n cluster-registry c2 --kubeconfig $KUBECONFIG2 -o yaml | kubectl apply --kubeconfig $KUBECONFIG1 -f - + kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 + kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Cleanup resources if: ${{ success() || failure() || cancelled() }} From 2f6df97ea313bd78d792c2e04a7b3ad96c3fa775 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Mon, 19 Dec 2022 21:46:49 +0000 Subject: [PATCH 05/13] e2e: use test built image --- .github/workflows/e2e.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6cf63ce..c69f2bd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -54,7 +54,7 @@ jobs: build-deps-v2 - name: Run build - run: make docker-build + run: IMG=cluster-registry-controller:ci-test make docker-build working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Get kind run: go install sigs.k8s.io/kind@v0.11.1 @@ -73,15 +73,16 @@ jobs: kind get kubeconfig --name "kind-${i}" > ${configPath} echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV + kind load docker-image cluster-registry-controller:ci-test --name "kind-${i}" done working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: multicluster tests run: | kubectl get pods -A --kubeconfig $KUBECONFIG1 - helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c1 --kubeconfig $KUBECONFIG1 + helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c1 --set image.repository=cluster-registry-controller --set image.tag=ci-test --kubeconfig $KUBECONFIG1 kubectl get pods -A --kubeconfig $KUBECONFIG1 kubectl get pods -A --kubeconfig $KUBECONFIG2 - helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c2 --kubeconfig $KUBECONFIG2 + helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c2 --set image.repository=cluster-registry-controller --set image.tag=ci-test --kubeconfig $KUBECONFIG2 kubectl get pods -A --kubeconfig $KUBECONFIG2 sleep 30 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 From 4112abd00377a7c7b812b34dfea4d3980be44795 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Mon, 19 Dec 2022 22:57:11 +0000 Subject: [PATCH 06/13] e2e: wait on cluster-registry pod ready --- .github/workflows/e2e.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c69f2bd..6a7b555 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -84,7 +84,10 @@ jobs: kubectl get pods -A --kubeconfig $KUBECONFIG2 helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c2 --set image.repository=cluster-registry-controller --set image.tag=ci-test --kubeconfig $KUBECONFIG2 kubectl get pods -A --kubeconfig $KUBECONFIG2 - sleep 30 + kubectl describe deployment -n cluster-registry cluster-registry-controller-controller --kubeconfig $KUBECONFIG1 + kubectl describe deployment -n cluster-registry cluster-registry-controller-controller --kubeconfig $KUBECONFIG2 + kubectl wait pods -l app.kubernetes.io/name=cluster-registry-controller --timeout=120s --for condition=Ready -n cluster-registry --kubeconfig $KUBECONFIG1 + kubectl wait pods -l app.kubernetes.io/name=cluster-registry-controller --timeout=120s --for condition=Ready -n cluster-registry --kubeconfig $KUBECONFIG2 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 echo "Getting cluster c1" From 8b39b25824eac21df6754c6ed9aefc4dd5637c8a Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Tue, 20 Dec 2022 02:20:30 +0000 Subject: [PATCH 07/13] add artifacts and sync check --- .github/workflows/e2e.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6a7b555..cd7b443 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -78,6 +78,7 @@ jobs: working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: multicluster tests run: | + mkdir ${ARTIFACTS_DIR} kubectl get pods -A --kubeconfig $KUBECONFIG1 helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c1 --set image.repository=cluster-registry-controller --set image.tag=ci-test --kubeconfig $KUBECONFIG1 kubectl get pods -A --kubeconfig $KUBECONFIG1 @@ -100,7 +101,24 @@ jobs: kubectl get secret -n cluster-registry c2 --kubeconfig $KUBECONFIG2 -o yaml | kubectl apply --kubeconfig $KUBECONFIG1 -f - kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 + kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 > ${ARTIFACTS_DIR}/clusters_c1.yaml + kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 > ${ARTIFACTS_DIR}/clusters_c2.yaml + sync1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') + sync2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') + if [[ "$sync1" == "" || "$sync2" == "" ]]; then + echo "one or more clusters not in sync" + exit 1 + fi working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + env: + ARTIFACTS_DIR: ${{ github.workspace }}/test_artifacts - name: Cleanup resources if: ${{ success() || failure() || cancelled() }} run: kind delete clusters $(kind get clusters) + - name: Upload artifacts + if: ${{ success() || failure() || cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: test_artifacts + path: ${{ github.workspace }}/test_artifacts + From e57a2db34d13f74d362c4a05adca9c8cf2dc313e Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Tue, 20 Dec 2022 16:29:57 +0000 Subject: [PATCH 08/13] e2e: make only run on PR workflow --- .github/workflows/e2e.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index cd7b443..e9436c5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,7 +1,6 @@ name: e2e on: - push: pull_request: env: From b87110d8f9895a60d236584923fbbcb7fadcc573 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Thu, 26 Jan 2023 18:56:49 -0500 Subject: [PATCH 09/13] run ci e2e on push to ci-e2e branches --- .github/workflows/e2e.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e9436c5..abfd044 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,6 +2,9 @@ name: e2e on: pull_request: + push: + branches: + - 'ci-e2e**' env: GO_VERSION: 1.18 From e980f70fd183068f8538aff1970f4ead2b194b9b Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Thu, 26 Jan 2023 19:31:12 -0500 Subject: [PATCH 10/13] e2e: fix sync status check --- .github/workflows/e2e.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index abfd044..31466ba 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -105,12 +105,18 @@ jobs: kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 > ${ARTIFACTS_DIR}/clusters_c1.yaml kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 > ${ARTIFACTS_DIR}/clusters_c2.yaml - sync1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') - sync2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') - if [[ "$sync1" == "" || "$sync2" == "" ]]; then + sync1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[0]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') + echo "Cluster 1 sync status = ${sync1}" + sync2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[1]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') + echo "Cluster 2 sync status = ${sync2}" + if [[ "${sync1}" == "" || "${sync2}" == "" ]]; then echo "one or more clusters not in sync" exit 1 fi + sync_reason1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[0]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') + sync_reason2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[1]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') + echo "Sync status reason 1 = '${sync_reason1}'" + echo "Sync status reason 1 = '${sync_reason2}'" working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} env: ARTIFACTS_DIR: ${{ github.workspace }}/test_artifacts From 0734c1a1bcc854bb62cbc260c61188cd9c764add Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Thu, 26 Jan 2023 21:59:31 -0500 Subject: [PATCH 11/13] e2e: add pods list to artifacts --- .github/workflows/e2e.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 31466ba..234f760 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -91,6 +91,8 @@ jobs: kubectl describe deployment -n cluster-registry cluster-registry-controller-controller --kubeconfig $KUBECONFIG2 kubectl wait pods -l app.kubernetes.io/name=cluster-registry-controller --timeout=120s --for condition=Ready -n cluster-registry --kubeconfig $KUBECONFIG1 kubectl wait pods -l app.kubernetes.io/name=cluster-registry-controller --timeout=120s --for condition=Ready -n cluster-registry --kubeconfig $KUBECONFIG2 + kubectl get pods -A --kubeconfig $KUBECONFIG1 > ${ARTIFACTS_DIR}/pods_c1.txt + kubectl get pods -A --kubeconfig $KUBECONFIG2 > ${ARTIFACTS_DIR}/pods_c2.txt kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 echo "Getting cluster c1" @@ -105,16 +107,16 @@ jobs: kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 > ${ARTIFACTS_DIR}/clusters_c1.yaml kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 > ${ARTIFACTS_DIR}/clusters_c2.yaml - sync1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[0]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') + sync1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') echo "Cluster 1 sync status = ${sync1}" - sync2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[1]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') + sync2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') echo "Cluster 2 sync status = ${sync2}" if [[ "${sync1}" == "" || "${sync2}" == "" ]]; then echo "one or more clusters not in sync" exit 1 fi - sync_reason1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[0]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') - sync_reason2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[1]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') + sync_reason1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') + sync_reason2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') echo "Sync status reason 1 = '${sync_reason1}'" echo "Sync status reason 1 = '${sync_reason2}'" working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} From 2c4c861561f31d1f706ebec9805624c8c122cc55 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Fri, 27 Jan 2023 07:07:19 -0500 Subject: [PATCH 12/13] e2e: add check for actual sync status for both clusters --- .github/workflows/e2e.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 234f760..3041ac7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -93,6 +93,8 @@ jobs: kubectl wait pods -l app.kubernetes.io/name=cluster-registry-controller --timeout=120s --for condition=Ready -n cluster-registry --kubeconfig $KUBECONFIG2 kubectl get pods -A --kubeconfig $KUBECONFIG1 > ${ARTIFACTS_DIR}/pods_c1.txt kubectl get pods -A --kubeconfig $KUBECONFIG2 > ${ARTIFACTS_DIR}/pods_c2.txt + kubectl get secrets -A --kubeconfig $KUBECONFIG1 > ${ARTIFACTS_DIR}/secrets_c1.txt + kubectl get secrets -A --kubeconfig $KUBECONFIG2 > ${ARTIFACTS_DIR}/secrets_c2.txt kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 echo "Getting cluster c1" @@ -112,13 +114,17 @@ jobs: sync2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') echo "Cluster 2 sync status = ${sync2}" if [[ "${sync1}" == "" || "${sync2}" == "" ]]; then - echo "one or more clusters not in sync" + echo "one or more clusters missing sync status" exit 1 fi sync_reason1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') - sync_reason2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') + sync_reason2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].reason}{end}') echo "Sync status reason 1 = '${sync_reason1}'" - echo "Sync status reason 1 = '${sync_reason2}'" + echo "Sync status reason 2 = '${sync_reason2}'" + if [[ "${sync1}" == "False" || "${sync2}" == "False" ]]; then + echo "one or more clusters are not in sync" + exit 1 + fi working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} env: ARTIFACTS_DIR: ${{ github.workspace }}/test_artifacts From 116800e1391ebbb380f01b64ad68f69926b58ae9 Mon Sep 17 00:00:00 2001 From: Tim Swanson Date: Fri, 27 Jan 2023 12:48:03 -0500 Subject: [PATCH 13/13] e2e: sleep before checking cluster sync status to allow time to converge --- .github/workflows/e2e.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3041ac7..42b79de 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -79,8 +79,13 @@ jobs: done working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: multicluster tests + id: test-multicluster run: | mkdir ${ARTIFACTS_DIR} + kubectl config get-contexts --kubeconfig $KUBECONFIG1 + kubectl config get-contexts --kubeconfig $KUBECONFIG2 + kubectl get nodes --kubeconfig $KUBECONFIG1 + kubectl get nodes --kubeconfig $KUBECONFIG2 kubectl get pods -A --kubeconfig $KUBECONFIG1 helm install --namespace=cluster-registry --create-namespace cluster-registry-controller deploy/charts/cluster-registry --set localCluster.name=c1 --set image.repository=cluster-registry-controller --set image.tag=ci-test --kubeconfig $KUBECONFIG1 kubectl get pods -A --kubeconfig $KUBECONFIG1 @@ -109,6 +114,7 @@ jobs: kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 kubectl get clusters -o yaml --kubeconfig $KUBECONFIG1 > ${ARTIFACTS_DIR}/clusters_c1.yaml kubectl get clusters -o yaml --kubeconfig $KUBECONFIG2 > ${ARTIFACTS_DIR}/clusters_c2.yaml + sleep 30 sync1=$(kubectl get clusters -A --kubeconfig $KUBECONFIG1 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}') echo "Cluster 1 sync status = ${sync1}" sync2=$(kubectl get clusters -A --kubeconfig $KUBECONFIG2 -o jsonpath='{range .items[*]}{@.status.conditions[?(@.type=="ClustersSynced")].status}{end}')