feature: VPC Egress Gateway (#4692)

Signed-off-by: zhangzujian <[email protected]>
kubeovn · Dec 3, 2024 · e34d562 · e34d562
1 parent ae0fc89
commit e34d562
Show file tree

Hide file tree

Showing 57 changed files with 3,990 additions and 181 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -2002,7 +2002,7 @@ jobs:
 
       - name: Create kind cluster
         env:
-          k8s_version: v1.23.17
+          k8s_version: v1.29.10
         run: |
           pipx install jinjanator
           make kind-init
@@ -2421,6 +2421,186 @@ jobs:
       - name: Cleanup
         run: timeout -k 10 180 sh -x dist/images/cleanup.sh
 
+  vpc-egress-gateway-e2e:
+    name: VPC Egress Gateway E2E
+    needs:
+      - build-kube-ovn
+      - build-e2e-binaries
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        ip-family:
+          - ipv4
+          - ipv6
+          - dual
+    steps:
+      - uses: jlumbroso/[email protected]
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          docker-images: false
+          large-packages: false
+          tool-cache: false
+          swap-storage: false
+
+      - uses: actions/checkout@v4
+
+      - name: Create the default branch directory
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - uses: actions/setup-go@v5
+        id: setup-go
+        with:
+          go-version-file: ${{ env.E2E_DIR }}/go.mod
+          check-latest: true
+          cache: false
+
+      - name: Export Go full version
+        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
+          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-
+
+      - name: Install kind
+        uses: helm/[email protected]
+        with:
+          version: ${{ env.KIND_VERSION }}
+          install_only: true
+
+      - name: Install ginkgo
+        working-directory: ${{ env.E2E_DIR }}
+        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      - name: Download image
+        uses: actions/download-artifact@v4
+        with:
+          name: kube-ovn
+
+      - name: Load image
+        run: docker load --input kube-ovn.tar
+
+      - name: Create kind cluster
+        run: |
+          pipx install jinjanator
+          make kind-init-${{ matrix.ip-family }}
+
+      - name: Install Kube-OVN
+        id: install
+        run: make kind-install-debug-valgrind-${{ matrix.ip-family }}
+
+      - name: Install Multus
+        run: make kind-install-multus
+
+      - name: Run E2E
+        id: e2e
+        working-directory: ${{ env.E2E_DIR }}
+        env:
+          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
+          E2E_IP_FAMILY: ${{ matrix.ip-family }}
+        run: make vpc-egress-gateway-e2e
+
+      - name: Collect k8s events
+        if: failure() && steps.e2e.conclusion == 'failure'
+        run: |
+          kubectl get events -A -o yaml > kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.yaml
+          tar zcf kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.yaml
+
+      - name: Upload k8s events
+        uses: actions/upload-artifact@v4
+        if: failure() && steps.e2e.conclusion == 'failure'
+        with:
+          name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events
+          path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz
+
+      - name: Collect apiserver audit logs
+        if: failure() && steps.e2e.conclusion == 'failure'
+        run: |
+          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
+          tar zcf kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log
+
+      - name: Upload apiserver audit logs
+        uses: actions/upload-artifact@v4
+        if: failure() && steps.e2e.conclusion == 'failure'
+        with:
+          name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log
+          path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz
+
+      - name: kubectl ko log
+        if: failure() && steps.e2e.conclusion == 'failure'
+        run: |
+          make kubectl-ko-log
+          mv kubectl-ko-log.tar.gz kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
+
+      - name: upload kubectl ko log
+        uses: actions/upload-artifact@v4
+        if: failure() && steps.e2e.conclusion == 'failure'
+        with:
+          name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log
+          path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
+
+      - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
+        run: make check-kube-ovn-pod-restarts
+
+      - name: Check valgrind result
+        run: |
+          kubectl -n kube-system rollout restart ds ovs-ovn
+          kubectl -n kube-system rollout status ds ovs-ovn
+          sleep 10
+          kubectl -n kube-system rollout restart deploy ovn-central
+          kubectl -n kube-system rollout status deploy ovn-central
+          while true; do
+            if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then
+              break
+            fi
+            sleep 1
+          done
+          kubectl ko log ovn
+          kubectl ko log ovs
+
+          for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
+            echo "Checking if valgrind log file for $daemon exists..."
+            find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
+          done
+
+          find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
+            if grep -qw 'definitely lost' "$f"; then
+              echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
+              echo $f
+              cat "$f"
+              exit 1
+            fi;
+          done
+
+      - name: Cleanup
+        run: timeout -k 10 180 sh -x dist/images/cleanup.sh
+
   iptables-vpc-nat-gw-conformance-e2e:
     name: Iptables VPC NAT Gateway E2E
     needs:
@@ -2966,6 +3146,7 @@ jobs:
       - kube-ovn-ic-conformance-e2e
       - kube-ovn-ipsec-e2e
       - multus-conformance-e2e
+      - vpc-egress-gateway-e2e
       - ovn-vpc-nat-gw-conformance-e2e
       - iptables-vpc-nat-gw-conformance-e2e
       - webhook-e2e

diff --git a/Makefile.e2e b/Makefile.e2e
@@ -74,6 +74,7 @@ e2e-build:
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ha
@@ -168,6 +169,14 @@ vip-conformance-e2e:
 	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
 		--focus=CNI:Kube-OVN ./test/e2e/vip/vip.test -- $(TEST_BIN_ARGS)
 
+.PHONY: vpc-egress-gateway-e2e
+vpc-egress-gateway-e2e:
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=30m \
+		--focus=CNI:Kube-OVN ./test/e2e/vpc-egress-gateway/vpc-egress-gateway.test -- $(TEST_BIN_ARGS)
+
 .PHONY: iptables-vpc-nat-gw-conformance-e2e
 iptables-vpc-nat-gw-conformance-e2e:
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw

diff --git a/charts/kube-ovn/Chart.yaml b/charts/kube-ovn/Chart.yaml
@@ -23,4 +23,4 @@ version: v1.14.0
 # It is recommended to use it with quotes.
 appVersion: "1.14.0"
 
-kubeVersion: ">= 1.23.0-0"
+kubeVersion: ">= 1.29.0-0"
diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml
@@ -140,6 +140,7 @@ spec:
           - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }}
           - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }}
           - --enable-live-migration-optimize={{- .Values.func.ENABLE_LIVE_MIGRATION_OPTIMIZE }}
+          - --image={{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
           securityContext:
             runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false