Release cce-network-v2/2.12.6

cce-network-v2/VERSION

@@ -1 +1 @@
-2.12.5
+2.12.6

cce-network-v2/docs/release.md

@@ -7,6 +7,11 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如
 2. 增加 eni 安全组同步功能， 保持CCE ENI 和节点安全组同步。
 3. 增加节点网络配置集功能 NetResourceConfigSet，支持指定节点独立配置网络资源。
 
+#### 2.12.6 [20240913]
+1. [Bug] 修复在 RDMA 场景查询子网为空，打印错误日志的问题
+2. [Bug] 修复在VPC-Route 模式下，开启 RDMA 后，重启 operator 进程会为 RDMA nrs 分配 podCIDR 的问题。
+3. [Optimize] psts 增加对未被 CCE 管理的跨子网 IP 地址的清理逻辑
+
 #### 2.12.5 [20240829]
 1. [Bug] 修复在 2.12.4 版本中同步 ENI 状态时丢失 IPv6 地址的问题。
 2. [Bug] 修复有多个 ENI 都存在待释放 IP 时，多次查询 ENI 顺序不一致影响 IP 标记流程导致无法释放 IP 的问题

cce-network-v2/pkg/bce/bcesync/borrowed_subnet.go

@@ -62,6 +62,10 @@ func resyncBSM() error {
 		return fmt.Errorf("bsm failed to list enis: %v", err)
 	}
 	for _, eni := range enis {
+		// ignore eni without subnetID like RDMA device
+		if eni.Spec.SubnetID == "" {
+			continue
+		}
 		bsbn, ok := subnets[eni.Spec.SubnetID]
 		if !ok {
 			sbn, err := GlobalBSM().GetSubnet(eni.Spec.SubnetID)

cce-network-v2/pkg/bce/vpceni/node_bbc.go

@@ -21,6 +21,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/cache"
 
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/api/v1/models"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/bce/api/metadata"
@@ -319,11 +320,54 @@ func (n *bbcNetworkResourceSet) allocateIPCrossSubnet(ctx context.Context, sbnID
 }
 
 // ReuseIPs implements realNodeInf
-func (n *bbcNetworkResourceSet) reuseIPs(ctx context.Context, ips []*models.PrivateIP, Owner string) (string, error) {
+func (n *bbcNetworkResourceSet) reuseIPs(ctx context.Context, ips []*models.PrivateIP, owner string) (string, error) {
 	if n.tryRefreshBBCENI() == nil {
 		return "", fmt.Errorf("bbc eni %s is not ready", n.instanceID)
 	}
-	scopeLog := n.log.WithField("func", "reuseIPs")
+
+	namespace, name, err := cache.SplitMetaNamespaceKey(owner)
+	if err != nil {
+		return "", fmt.Errorf("invalid owner %s: %v", owner, err)
+	}
+
+	scopeLog := n.log.WithFields(logrus.Fields{
+		"func":      "reuseIPs",
+		"namespace": namespace,
+		"name":      name,
+		"ips":       logfields.Repr(ips),
+	})
+
+	// check ip conflict
+	// should to delete ip from the old eni
+	isLocalIP, err := n.rleaseOldIP(ctx, scopeLog, ips, namespace, name, func(ctx context.Context, scopedLog *logrus.Entry, eniID string, toReleaseIPs []string) error {
+		eni, err := n.manager.enilister.Get(eniID)
+		if err != nil {
+			return fmt.Errorf("fail to release old ip. get eni %s failed: %v", eniID, err)
+		}
+		instanceID := eni.Spec.InstanceID
+		scopedLog.WithFields(logrus.Fields{
+			"oldENI":       eniID,
+			"instanceID":   instanceID,
+			"toReleaseIPs": toReleaseIPs,
+		})
+		err = n.manager.bceclient.BBCBatchDelIP(ctx, &bbc.BatchDelIpArgs{
+			InstanceId: instanceID,
+			PrivateIps: toReleaseIPs,
+		})
+		if err != nil {
+			scopedLog.Warnf("release old ip from bbc eni %s failed: %v", n.instanceID, err)
+		} else {
+			scopedLog.Info("release old ip from bbc eni successfully")
+		}
+		return err
+	})
+	if err != nil {
+		return "", err
+	}
+	if isLocalIP {
+		scopeLog.Info("ip is local, no need to release ip from bbc eni")
+		return n.bbceni.Name, nil
+	}
 
 	// TODO: release ip from bbc/ebc/vpc eni before reuse ip
 	var ipAndSubnets []bbc.IpAndSubnet

cce-network-v2/pkg/bce/vpceni/node_bcc.go

@@ -499,7 +499,16 @@ func (n *bccNetworkResourceSet) reuseIPs(ctx context.Context, ips []*models.Priv
 
 	// check ip conflict
 	// should to delete ip from the old eni
-	isLocalIP, err := n.rleaseOldIP(ctx, scopedLog, ips, namespace, name)
+	isLocalIP, err := n.rleaseOldIP(ctx, scopedLog, ips, namespace, name, func(ctx context.Context, scopedLog *logrus.Entry, eniID string, toReleaseIPs []string) error {
+		scopedLog.WithField("oldENI", eniID).WithField("toReleaseIPs", toReleaseIPs)
+		err = n.manager.bceclient.BatchDeletePrivateIP(ctx, toReleaseIPs, eniID, false)
+		if err != nil {
+			scopedLog.Warnf("release ip %s from eni %s failed: %v", toReleaseIPs, eniID, err)
+		} else {
+			scopedLog.Info("release ip from old eni success")
+		}
+		return err
+	})
 	if err != nil {
 		return
 	}
@@ -562,73 +571,4 @@ func (n *bccNetworkResourceSet) reuseIPs(ctx context.Context, ips []*models.Priv
 
 }
 
-// rleaseOldIP release old ip if it is not used by other endpoint
-// return true: ip is used by current endpoint, do not need to release
-// return false: ip is used by other endpoint, need to release
-func (n *bccNetworkResourceSet) rleaseOldIP(ctx context.Context, scopedLog *logrus.Entry, ips []*models.PrivateIP, namespace string, name string) (bool, error) {
-	for _, privateIP := range ips {
-		ceps, err := n.manager.cepClient.GetByIP(privateIP.PrivateIPAddress)
-		if err == nil && len(ceps) > 0 {
-			for _, cep := range ceps {
-				if cep.Namespace != namespace || cep.Name != name {
-					return false, fmt.Errorf("ip %s has been used by other endpoint %s/%s", privateIP.PrivateIPAddress, cep.Namespace, cep.Name)
-				}
-			}
-		}
-	}
-
-	// if ip is local eni, do not need to release
-	isLocalENI := false
-	for _, privateIP := range ips {
-		enis, err := watchers.ENIClient.GetByIP(privateIP.PrivateIPAddress)
-		if err != nil && len(enis) == 0 {
-			isLocalENI = false
-			break
-		}
-		if err == nil {
-			for _, eni := range enis {
-				if eni.Spec.NodeName == n.k8sObj.Name {
-					isLocalENI = true
-				} else {
-					isLocalENI = false
-					break
-				}
-			}
-		}
-	}
-	if isLocalENI {
-		return true, nil
-	}
-
-	cep, err := n.manager.cepClient.Lister().CCEEndpoints(namespace).Get(name)
-	if err != nil {
-		return false, fmt.Errorf("get endpoint %s/%s failed: %v", namespace, name, err)
-	}
-	if cep.Status.Networking == nil || cep.Status.Networking.NodeIP != n.k8sObj.Name {
-		var (
-			oldENI       string
-			toReleaseIPs []string
-		)
-		scopedLog.WithField("namespace", namespace).WithField("name", name).Debug("try to clean ip from old eni")
-
-		if cep.Status.Networking != nil {
-			for _, addr := range cep.Status.Networking.Addressing {
-				oldENI = addr.Interface
-				toReleaseIPs = append(toReleaseIPs, addr.IP)
-			}
-		}
-		if len(toReleaseIPs) > 0 {
-			releaseLog := scopedLog.WithField("oldENI", oldENI).WithField("toReleaseIPs", toReleaseIPs)
-			err = n.manager.bceclient.BatchDeletePrivateIP(ctx, toReleaseIPs, oldENI, false)
-			if err != nil {
-				releaseLog.Warnf("delete ip %s from eni %s failed: %v", toReleaseIPs, oldENI, err)
-
-			} else {
-				releaseLog.Info("clean ip from old eni success")
-			}
-		}
-	}
-	return false, nil
-}
-
 var _ realNodeInf = &bccNetworkResourceSet{}

cce-network-v2/pkg/bce/vpceni/node_super.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/api/v1/models"
 	operatorOption "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/operator/option"
+	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/operator/watchers"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/bce/api"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/bce/api/metadata"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/bce/bcesync"
@@ -1389,6 +1390,74 @@ func (n *bceNetworkResourceSet) tryBorrowIPs(newENI *ccev2.ENI) error {
 	return nil
 }
 
+type fnReleaseIPsFromCLoud func(ctx context.Context, scopedLog *logrus.Entry, eniID string, toReleaseIPs []string) error
+
+// rleaseOldIP release old ip if it is not used by other endpoint
+// return true: ip is used by current endpoint, do not need to release
+// return false: ip is used by other endpoint, need to release
+func (n *bceNetworkResourceSet) rleaseOldIP(ctx context.Context, scopedLog *logrus.Entry, ips []*models.PrivateIP, namespace string, name string, releaseIPsFromCLoud fnReleaseIPsFromCLoud) (bool, error) {
+	for _, privateIP := range ips {
+		ceps, err := n.manager.cepClient.GetByIP(privateIP.PrivateIPAddress)
+		if err == nil && len(ceps) > 0 {
+			for _, cep := range ceps {
+				if cep.Namespace != namespace || cep.Name != name {
+					return false, fmt.Errorf("ip %s has been used by other endpoint %s/%s", privateIP.PrivateIPAddress, cep.Namespace, cep.Name)
+				}
+			}
+		}
+	}
+
+	// if ip is local eni, do not need to release
+	isLocalENI := false
+	toReleaseEniIPs := make(map[string][]string, 0)
+	for _, privateIP := range ips {
+		enis, err := watchers.ENIClient.GetByIP(privateIP.PrivateIPAddress)
+		if err != nil && len(enis) == 0 {
+			isLocalENI = false
+			break
+		}
+		if err == nil {
+			for _, eni := range enis {
+				if eni.Spec.NodeName == n.k8sObj.Name {
+					isLocalENI = true
+				} else {
+					// do not release ip which use same subnet with eni
+					if eni.Spec.SubnetID == privateIP.SubnetID {
+						return false, fmt.Errorf("ip %s has been used by other eni %s for cached ip", privateIP.PrivateIPAddress, eni.Name)
+					}
+					toReleaseEniIPs[eni.Name] = append(toReleaseEniIPs[eni.Name], privateIP.PrivateIPAddress)
+					isLocalENI = false
+					break
+				}
+			}
+		}
+	}
+	if isLocalENI {
+		return true, nil
+	}
+
+	cep, err := n.manager.cepClient.Lister().CCEEndpoints(namespace).Get(name)
+	if err != nil {
+		return false, fmt.Errorf("get endpoint %s/%s failed: %v", namespace, name, err)
+	}
+	if cep.Status.Networking == nil || cep.Status.Networking.NodeIP != n.k8sObj.Name {
+		scopedLog.WithField("namespace", namespace).WithField("name", name).Debug("try to clean ip from old eni")
+
+		if cep.Status.Networking != nil {
+			for _, addr := range cep.Status.Networking.Addressing {
+				toReleaseEniIPs[addr.Interface] = append(toReleaseEniIPs[addr.Interface], addr.IP)
+			}
+		}
+		for eniid := range toReleaseEniIPs {
+			toReleaseIPs := toReleaseEniIPs[eniid]
+			if len(toReleaseIPs) > 0 {
+				err = releaseIPsFromCLoud(ctx, scopedLog, eniid, toReleaseIPs)
+			}
+		}
+	}
+	return false, err
+}
+
 var (
 	_ endpoint.DirectEndpointOperation = &bceNetworkResourceSet{}
 )

cce-network-v2/pkg/endpoint/operator_psts_manager_provider.go

@@ -97,15 +97,13 @@ func (provider *pstsAllocatorProvider) AllocateIP(ctx context.Context, log *logr
 				log.WithField("step", "allocateFromLocalPool").Error("no available ipv4 ip")
 				return fmt.Errorf("no available ipv4 ip")
 			}
+
 			releaseIPFuncs = append(releaseIPFuncs, release...)
 			log = log.WithField("ipv4", logfields.Repr(ipv4Address)).WithField("ipv6", logfields.Repr(ipv6Address))
 			log.WithField("step", "allocate local ip").Info("allocate local ip success")
 
-			if ipv4Address != nil {
-				action.Addressing = append(action.Addressing, ipv4Address)
-				action.SubnetID = ipv4Address.Subnet
-			}
-
+			action.Addressing = append(action.Addressing, ipv4Address)
+			action.SubnetID = ipv4Address.Subnet
 			if ipv6Address != nil {
 				action.Addressing = append(action.Addressing, ipv6Address)
 				action.SubnetID = ipv6Address.Subnet

cce-network-v2/pkg/ipam/allocator/podcidr/podcidr.go

@@ -30,6 +30,7 @@ import (
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/controller"
 	ipPkg "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ip"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ipam"
+	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s"
 	v2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/lock"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/logging"
@@ -405,7 +406,10 @@ func (n *NodesPodCIDRManager) Resync(context.Context, time.Time) {
 			log.WithError(err).Fatal("Failed to list NetResourceSet")
 		}
 		for _, nrs := range nrsDatas {
-			n.upsertLocked(nrs)
+			// should not allocate podCIDRs for the nrs which use RDMA
+			if k8s.MatchNetworkResourceType(nrs.Annotations, n.ResourceType()) {
+				n.upsertLocked(nrs)
+			}
 		}
 
 		log.Infof("completed to resync %d nrs cidr", len(nrsDatas))

cce-network-v2/pkg/k8s/labels.go

@@ -147,6 +147,16 @@ var (
 	PodLabelOwnerInstance  = "cce.io/ownerInstance"
 )
 
+func MatchNetworkResourceType(annotations map[string]string, networkType string) bool {
+	objType := ccev2.NetResourceSetEventHandlerTypeEth
+	if len(annotations) > 0 {
+		if _, ok := annotations[AnnotationRDMAInfoVifFeatures]; ok {
+			objType = ccev2.NetResourceSetEventHandlerTypeRDMA
+		}
+	}
+	return objType == networkType
+}
+
 // ExtractFixedIPTTLSeconds Extract the expiration time of the fixed
 // IP from the annotation of the pod. The unit is s
 func ExtractFixedIPTTLSeconds(pod *corev1.Pod) int64 {

cce-network-v2/pkg/k8s/labels_test.go

@@ -0,0 +1,60 @@
+package k8s
+
+import (
+	"testing"
+
+	ccev2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMatchResourceType(t *testing.T) {
+	tests := []struct {
+		name          string
+		annotations   map[string]string
+		networkType   string
+		expectedMatch bool
+	}{
+		{
+			name:          "empty annotations",
+			annotations:   map[string]string{},
+			networkType:   ccev2.NetResourceSetEventHandlerTypeEth,
+			expectedMatch: true,
+		},
+		{
+			name:          "empty annotations, expected rdma",
+			annotations:   map[string]string{},
+			networkType:   ccev2.NetResourceSetEventHandlerTypeRDMA,
+			expectedMatch: false,
+		},
+		{
+			name:          "rdma-vif-features present, expected eth",
+			annotations:   map[string]string{AnnotationRDMAInfoVifFeatures: "true"},
+			networkType:   ccev2.NetResourceSetEventHandlerTypeEth,
+			expectedMatch: false,
+		},
+		{
+			name:          "rdma-vif-features present, expected rdma",
+			annotations:   map[string]string{AnnotationRDMAInfoVifFeatures: "true"},
+			networkType:   ccev2.NetResourceSetEventHandlerTypeRDMA,
+			expectedMatch: true,
+		},
+		{
+			name:          "rdma-vif-features absent, expected eth",
+			annotations:   map[string]string{},
+			networkType:   ccev2.NetResourceSetEventHandlerTypeEth,
+			expectedMatch: true,
+		},
+		{
+			name:          "rdma-vif-features absent, expected rdma",
+			annotations:   map[string]string{},
+			networkType:   ccev2.NetResourceSetEventHandlerTypeRDMA,
+			expectedMatch: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			match := MatchNetworkResourceType(test.annotations, test.networkType)
+			assert.Equal(t, test.expectedMatch, match)
+		})
+	}
+}