Nat gw

Makefile

@@ -154,8 +154,8 @@ build-debug:
 .PHONY: base-amd64
 base-amd64:
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 --build-arg GO_VERSION --build-arg TRIVY_DB_REPOSITORY -t $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-amd64 -o type=docker -f dist/images/Dockerfile.base dist/images/
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 --build-arg GO_VERSION --build-arg TRIVY_DB_REPOSITORY --build-arg LEGACY=true -t $(REGISTRY)/kube-ovn-base:$(LEGACY_TAG) -o type=docker -f dist/images/Dockerfile.base dist/images/
-	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 --build-arg GO_VERSION --build-arg TRIVY_DB_REPOSITORY --build-arg DEBUG=true -t $(REGISTRY)/kube-ovn-base:$(DEBUG_TAG)-amd64 -o type=docker -f dist/images/Dockerfile.base dist/images/
+	# docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 --build-arg GO_VERSION --build-arg TRIVY_DB_REPOSITORY --build-arg LEGACY=true -t $(REGISTRY)/kube-ovn-base:$(LEGACY_TAG) -o type=docker -f dist/images/Dockerfile.base dist/images/
+	# docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 --build-arg GO_VERSION --build-arg TRIVY_DB_REPOSITORY --build-arg DEBUG=true -t $(REGISTRY)/kube-ovn-base:$(DEBUG_TAG)-amd64 -o type=docker -f dist/images/Dockerfile.base dist/images/
 
 .PHONY: base-amd64-dpdk
 base-amd64-dpdk:

charts/kube-ovn/templates/kube-ovn-crd.yaml

@@ -528,6 +528,17 @@ spec:
                       type: array
                       items:
                         type: string
+                bfd:
+                  type: object
+                  properties:
+                    enabled:
+                      type: boolean
+                    minRX:
+                      type: integer
+                    minTX:
+                      type: integer
+                    multiplier:
+                      type: integer
                 tolerations:
                   type: array
                   items:

dist/images/Dockerfile

@@ -1,6 +1,6 @@
 # syntax = docker/dockerfile:experimental
 ARG VERSION
-ARG BASE_TAG=$VERSION
+ARG BASE_TAG=$VERSION-amd64
 FROM kubeovn/kube-ovn-base:$BASE_TAG AS setcap
 
 COPY *.sh /kube-ovn/

dist/images/Dockerfile.base

@@ -10,6 +10,15 @@ ENV SRC_DIR='/usr/src'
 
 RUN apt update && apt install -y git curl
 
+ARG http_proxy=http://192.168.73.1:8889
+
+RUN apt install -y build-essential fakeroot \
+    autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \
+    graphviz iproute2 libcap-ng-dev libdbus-1-dev libnuma-dev libpcap-dev libssl-dev libtool libunbound-dev \
+    openssl pkg-config procps python3-all-dev python3-setuptools python3-sortedcontainers python3-sphinx
+
+ARG https_proxy=http://192.168.73.1:8889
+
 RUN cd /usr/src/ && \
     git clone -b branch-3.3 --depth=1 https://github.com/openvswitch/ovs.git && \
     cd ovs && \
@@ -34,6 +43,14 @@ RUN cd /usr/src/ && \
     # increase the default probe interval for large cluster
     curl -s https://github.com/kubeovn/ovs/commit/7e894a0bda966e746f81b2ebe262a62cd91bf269.patch | git apply
 
+RUN cd /usr/src/ovs && \
+    ./boot.sh && \
+    ./configure && \
+    rm -rf .git && \
+    CONFIGURE_OPTS='CFLAGS="-fPIC"' && \
+    if [ "$ARCH" = "amd64" ] && [ "$LEGACY" != "true" ]; then CONFIGURE_OPTS='CFLAGS="-O2 -g -msse4.2 -mpopcnt -fPIC"'; fi && \
+    DATAPATH_CONFIGURE_OPTS='--prefix=/usr' EXTRA_CONFIGURE_OPTS=$CONFIGURE_OPTS make debian-deb
+
 RUN cd /usr/src/ && git clone -b branch-24.03 --depth=1 https://github.com/ovn-org/ovn.git && \
     cd ovn && \
     # change hash type from dp_hash to hash with field src_ip
@@ -60,19 +77,11 @@ RUN cd /usr/src/ && git clone -b branch-24.03 --depth=1 https://github.com/ovn-o
     curl -s https://github.com/kubeovn/ovn/commit/cf6ba220851ae13de76473b6569097bb70ec76d8.patch | git apply && \
     # ovn-controller: make activation strategy work for single chassis
     curl -s https://github.com/kubeovn/ovn/commit/487cd1c6c0cc9def7cedaaaf3dd4bc99c23974c4.patch | git apply
-
-RUN apt install -y build-essential fakeroot \
-    autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \
-    graphviz iproute2 libcap-ng-dev libdbus-1-dev libnuma-dev libpcap-dev libssl-dev libtool libunbound-dev \
-    openssl pkg-config procps python3-all-dev python3-setuptools python3-sortedcontainers python3-sphinx
-
-RUN cd /usr/src/ovs && \
-    ./boot.sh && \
-    ./configure && \
-    rm -rf .git && \
-    CONFIGURE_OPTS='CFLAGS="-fPIC"' && \
-    if [ "$ARCH" = "amd64" ] && [ "$LEGACY" != "true" ]; then CONFIGURE_OPTS='CFLAGS="-O2 -g -msse4.2 -mpopcnt -fPIC"'; fi && \
-    DATAPATH_CONFIGURE_OPTS='--prefix=/usr' EXTRA_CONFIGURE_OPTS=$CONFIGURE_OPTS make debian-deb
+
+RUN cd /usr/src/ovn && curl -s https://github.com/kubeovn/ovn/commit/399a153572f19d2fb05de0b5f72bf8256bede7d4.patch | git apply
+RUN cd /usr/src/ovn && curl -s https://github.com/kubeovn/ovn/commit/8f41f3fe9cb7e01d9b8866b36487a6f7d94bfc1e.patch | git apply
+RUN cd /usr/src/ovn && curl -s https://github.com/kubeovn/ovn/commit/83577448203692834fce764af7e5887edc4e75a3.patch | git apply
+RUN cd /usr/src/ovn && curl -s https://github.com/kubeovn/ovn/commit/4a29dd2bf8f276d656887c2c55f141c9e41fab78.patch | git apply
 
 RUN cd /usr/src/ovn && \
     sed -i 's/OVN/ovn/g' debian/changelog && \
@@ -94,14 +103,20 @@ ARG ARCH
 ENV CNI_VERSION="v1.6.0"
 ENV KUBE_VERSION="v1.31.2"
 ENV GOBGP_VERSION="3.30.0"
+
 ENV TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2"
 
+ARG https_proxy=http://192.168.73.1:8889
+
 RUN apk --no-cache add curl jq
+
 ADD go-deps/download-go-deps.sh /
 RUN sh -x /download-go-deps.sh
 
 FROM golang:$GO_VERSION-alpine AS go-deps
 
+ARG https_proxy=http://192.168.73.1:8889
+
 RUN apk --no-cache add bash curl jq
 ADD go-deps/rebuild-go-deps.sh /
 RUN --mount=type=bind,target=/trivy,from=trivy,source=/godeps \
@@ -139,6 +154,8 @@ RUN mkdir -p /var/run/openvswitch && \
     mkdir -p /etc/cni/net.d && \
     mkdir -p /opt/cni/bin
 
+ARG https_proxy=http://192.168.73.1:8889
+
 ARG DUMB_INIT_VERSION="1.2.5"
 RUN curl -sSf -L --retry 5 -o /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_$(arch) && \
     chmod +x /usr/bin/dumb-init

dist/images/go-deps/download-go-deps.sh

@@ -23,30 +23,33 @@ curl -sSf -L --retry 5 https://github.com/osrg/gobgp/releases/download/v${GOBGP_
 
 ls -lh "$DEPS_DIR"
 
-trivy rootfs --ignore-unfixed --scanners vuln --pkg-types library -f json --output trivy.json "$DEPS_DIR"
+# unset https_proxy
+# unset TRIVY_DB_REPOSITORY
 
-cat trivy.json
+# trivy rootfs --ignore-unfixed --scanners vuln --pkg-types library -f json --output trivy.json "$DEPS_DIR"
+
+# cat trivy.json
 
 TARGETS_FILE="$DEPS_DIR/trivy-targets.txt"
 
 : > "$TARGETS_FILE"
-jq -r '.Results[] | select((.Type=="gobinary") and (.Vulnerabilities!=null)) | .Target' trivy.json | while read f; do
-    name=$(basename $f)
-    case $name in
-        loopback|macvlan|portmap)
-            echo "$name@$CNI_PLUGINS_VERSION" >> "$TARGETS_FILE"
-            ;;
-        kubectl)
-            echo "$name@$KUBECTL_VERSION" >> "$TARGETS_FILE"
-            ;;
-        gobgp)
-            echo "$name@v$GOBGP_VERSION" >> "$TARGETS_FILE"
-            ;;
-        *)
-            echo "Unknown go binary: $f"
-            exit 1
-            ;;
-    esac
-done
+# jq -r '.Results[] | select((.Type=="gobinary") and (.Vulnerabilities!=null)) | .Target' trivy.json | while read f; do
+#     name=$(basename $f)
+#     case $name in
+#         loopback|macvlan|portmap)
+#             echo "$name@$CNI_PLUGINS_VERSION" >> "$TARGETS_FILE"
+#             ;;
+#         kubectl)
+#             echo "$name@$KUBECTL_VERSION" >> "$TARGETS_FILE"
+#             ;;
+#         gobgp)
+#             echo "$name@v$GOBGP_VERSION" >> "$TARGETS_FILE"
+#             ;;
+#         *)
+#             echo "Unknown go binary: $f"
+#             exit 1
+#             ;;
+#     esac
+# done
 
 cat "$TARGETS_FILE"

dist/images/install.sh

@@ -775,6 +775,17 @@ spec:
                       type: array
                       items:
                         type: string
+                bfd:
+                  type: object
+                  properties:
+                    enabled:
+                      type: boolean
+                    minRX:
+                      type: integer
+                    minTX:
+                      type: integer
+                    multiplier:
+                      type: integer
                 tolerations:
                   type: array
                   items:

pkg/apis/kubeovn/v1/types.go

@@ -529,6 +529,7 @@ type VpcNatSpec struct {
 	Affinity        corev1.Affinity     `json:"affinity"`
 	QoSPolicy       string              `json:"qosPolicy"`
 	BgpSpeaker      VpcBgpSpeaker       `json:"bgpSpeaker"`
+	BFD             VpcBFD              `json:"bfd"`
 }
 
 type VpcBgpSpeaker struct {
@@ -543,6 +544,13 @@ type VpcBgpSpeaker struct {
 	ExtraArgs             []string        `json:"extraArgs"`
 }
 
+type VpcBFD struct {
+	Enabled    bool   `json:"enabled"`
+	MinRX      uint32 `json:"minRX"`
+	MinTX      uint32 `json:"minTX"`
+	Multiplier uint32 `json:"multiplier"`
+}
+
 type VpcNatStatus struct {
 	QoSPolicy       string              `json:"qosPolicy" patchStrategy:"merge"`
 	ExternalSubnets []string            `json:"externalSubnets" patchStrategy:"merge"`

pkg/controller/service_lb.go

@@ -195,9 +195,9 @@ func (c *Controller) getLbSvcPod(svcName, svcNamespace string) (*corev1.Pod, err
 	case len(pods) == 0:
 		time.Sleep(2 * time.Second)
 		return nil, fmt.Errorf("pod of deployment %s/%s not found", svcNamespace, genLbSvcDpName(svcName))
-	case len(pods) != 1:
-		time.Sleep(2 * time.Second)
-		return nil, errors.New("too many pods")
+	// case len(pods) != 1:
+	// 	time.Sleep(2 * time.Second)
+	// 	return nil, errors.New("too many pods")
 	case pods[0].Status.Phase != corev1.PodRunning:
 		time.Sleep(2 * time.Second)
 		return nil, fmt.Errorf("pod %s/%s is not running", pods[0].Namespace, pods[0].Name)

pkg/controller/vpc_nat_gateway.go

@@ -755,11 +755,15 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 		util.VpcNatGatewayAnnotation:     gw.Name,
 		util.AttachmentNetworkAnnotation: fmt.Sprintf("%s/%s", c.config.PodNamespace, externalNetworkNad),
 		util.LogicalSwitchAnnotation:     gw.Spec.Subnet,
-		util.IPAddressAnnotation:         gw.Spec.LanIP,
+	}
+	if strings.ContainsAny(gw.Spec.LanIP, ",;") {
+		podAnnotations[util.IPPoolAnnotation] = gw.Spec.LanIP
+	} else {
+		podAnnotations[util.IPAddressAnnotation] = gw.Spec.LanIP
 	}
 
 	// Add an interface that can reach the API server, we need access to it to probe Kube-OVN resources
-	if gw.Spec.BgpSpeaker.Enabled {
+	if gw.Spec.BgpSpeaker.Enabled && gw.Spec.Vpc != c.config.ClusterRouter {
 		if err := c.setNatGwAPIAccess(podAnnotations, externalNetworkNad); err != nil {
 			klog.Error(err)
 			return nil, err
@@ -768,6 +772,9 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 
 	for key, value := range podAnnotations {
 		annotations[key] = value
+		if key == util.IPPoolAnnotation {
+			delete(annotations, util.IPAddressAnnotation)
+		}
 	}
 
 	subnets, err := c.subnetsLister.List(labels.Everything())
@@ -847,7 +854,7 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 			Labels: labels,
 		},
 		Spec: v1.StatefulSetSpec{
-			Replicas: ptr.To(int32(1)),
+			Replicas: ptr.To(int32(2)),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: labels,
 			},
@@ -982,6 +989,28 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 		sts.Spec.Template.Spec.Containers = append(containers, speakerContainer)
 	}
 
+	if gw.Spec.BFD.Enabled {
+		sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, corev1.Container{
+			Name:            "bfd",
+			Image:           "docker.io/kubeovn/kube-ovn:dev",
+			ImagePullPolicy: corev1.PullIfNotPresent,
+			Command:         []string{"sh", "-xc", "bfdd-beacon --nofork --listen=${POD_IP} --tee"},
+			Env: []corev1.EnvVar{
+				{
+					Name: "POD_IP",
+					ValueFrom: &corev1.EnvVarSource{
+						FieldRef: &corev1.ObjectFieldSelector{
+							FieldPath: "status.podIP",
+						},
+					},
+				},
+			},
+			// TODO: add liveness/readiness probes by executing bfdd-control version/status
+		})
+	}
+
+	// kubectl ko nbctl create bfd logical_port=lrp1 dst_ip=10.16.0.6 min_tx=100 min_rx=100 detect_mult=3
+
 	return sts, nil
 }
 
@@ -1010,9 +1039,9 @@ func (c *Controller) getNatGwPod(name string) (*corev1.Pod, error) {
 		return nil, err
 	case len(pods) == 0:
 		return nil, k8serrors.NewNotFound(v1.Resource("pod"), name)
-	case len(pods) != 1:
-		time.Sleep(5 * time.Second)
-		return nil, errors.New("too many pod")
+	// case len(pods) != 1:
+	// 	time.Sleep(5 * time.Second)
+	// 	return nil, errors.New("too many pod")
 	case pods[0].Status.Phase != corev1.PodRunning:
 		time.Sleep(5 * time.Second)
 		return nil, errors.New("pod is not active now")

pkg/daemon/ovs_linux.go

@@ -735,23 +735,23 @@ func (c *Controller) checkNodeGwNicInNs(nodeExtIP, ip, gw string, gwNS ns.NetNS)
 		return ns.WithNetNSPath(gwNS.Path(), func(_ ns.NetNS) error {
 			err = waitNetworkReady(util.NodeGwNic, ip, gw, true, true, 3, nil)
 			if err == nil {
-				if output, err := exec.Command("sh", "-c", "bfdd-control status").CombinedOutput(); err != nil {
+				if output, err := exec.Command("bfdd-control", "status").CombinedOutput(); err != nil {
 					err := fmt.Errorf("failed to get bfdd status, %w, %s", err, output)
 					klog.Error(err)
 					return err
 				}
 				for _, eip := range ovnEips {
 					if eip.Status.Ready {
 						// #nosec G204
-						cmd := exec.Command("sh", "-c", fmt.Sprintf("bfdd-control status remote %s local %s", eip.Spec.V4Ip, nodeExtIP))
+						cmd := exec.Command("bfdd-control", "status", "remote", eip.Spec.V4Ip, "local", nodeExtIP)
 						var outb bytes.Buffer
 						cmd.Stdout = &outb
 						if err := cmd.Run(); err == nil {
 							out := outb.String()
 							klog.V(3).Info(out)
 							if strings.Contains(out, "No session") {
 								// not exist
-								cmd = exec.Command("sh", "-c", fmt.Sprintf("bfdd-control allow %s", eip.Spec.V4Ip)) // #nosec G204
+								cmd = exec.Command("bfdd-control", "allow", eip.Spec.V4Ip) // #nosec G204
 								if err := cmd.Run(); err != nil {
 									err := fmt.Errorf("failed to add lrp %s ip %s into bfd listening list, %w", eip.Name, eip.Status.V4Ip, err)
 									klog.Error(err)
@@ -852,7 +852,7 @@ func configureNodeGwNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu i
 		if err != nil {
 			return fmt.Errorf("failed to configure gateway: %w", err)
 		}
-		cmd := exec.Command("sh", "-c", "bfdd-beacon --listen=0.0.0.0")
+		cmd := exec.Command("bfdd-beacon", "--listen=0.0.0.0")
 		if err := cmd.Run(); err != nil {
 			err := fmt.Errorf("failed to get start bfd listen, %w", err)
 			klog.Error(err)