Release cce-network-v2/2.12.10

cce-network-v2/VERSION

@@ -1 +1 @@
-2.12.9
+2.12.10

cce-network-v2/cmd/agent/cmd/daemon.go

@@ -292,6 +292,10 @@ func NewDaemon(ctx context.Context, cancel context.CancelFunc) (*Daemon, error)
 		bootstrapStats.k8sInit.End(true)
 	}
 
+	// confugure and start ENIM
+	d.configureENIM()
+	d.startENIM()
+
 	// Configure IPAM without using the configuration yet.
 	d.configureIPAM()
 	d.startIPAM()
@@ -303,10 +307,6 @@ func NewDaemon(ctx context.Context, cancel context.CancelFunc) (*Daemon, error)
 		debug.RegisterStatusObject("rdmaIpam["+key+"]", ri)
 	}
 
-	// confugure and start ENIM
-	d.configureENIM()
-	d.startENIM()
-
 	// endpoints handler
 	d.startEndpointHanler()
 

cce-network-v2/deploy/cce-network-v2/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.12.0
+version: 2.12.10
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.12.0"
+appVersion: "2.12.10"

cce-network-v2/docs/release.md

@@ -7,6 +7,17 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如
 2. 增加 eni 安全组同步功能， 保持CCE ENI 和节点安全组同步。
 3. 增加节点网络配置集功能 NetResourceConfigSet，支持指定节点独立配置网络资源。
 
+#### 2.12.10 [20241213]
+1. [Optimize] 优化 VPC-ENI 模式下的 veth 单机路由规则目的地址存在冲突的判断条件，解决残留本地路由规则时创建的 Pod 容器网络不通的问题
+2. [Optimize] 禁用 VPC-ENI 模式下的 IPv6 DHCP 时使用的网卡名修改为udev生成的原始名字，避免生成的网卡配置文件导致虚拟机重启时 network.service 服务启动失败
+3. [Bug] 更新 VPC-ENI 模式下的 ENI 状态机处理逻辑，修复当节点移出集群时，ENI 对象残留的问题
+4. [Bug] 修复 VPC-ENI 模式下的新建 ENI 网卡 borrow IP 地址时，borrow task ENIID 填写错误导致并发创建时可能引起的 borrow IP 计算错误
+5. [Optimize] 优化 VPC-ENI 模式下的 Romote ENI Syncer 逻辑，纳管 CCE 历史创建的 ENI，避免因历史创建的 ENI 无法被使用而占用 Node eniQuota 的问题
+6. [Optimize] 优化 VPC-ENI 模式下的 ENI 状态机处理逻辑，支持非终态 ENI 的处理流程，避免因状态机异常中断、重启丢失内存状态等原因导致的 ENI 状态处于异常的非终态而导致的该 ENI 无法正常使用的问题
+7. [Bug] PSTS 增加对 CEP TTL 未过期时直接移除 Node 或 ENI 导致 CEP 后续 TTL 过期后因无对应的 NetworkResourceSet 或 ENI 而无法正常删除时的清理逻辑
+8. [Bug] 修复 VPC-ENI 模式下 EBC ENI Quota 为 0 时，启用主网卡辅助IP的 ENI 赋值逻辑，避免因主网卡 ENI MacAddress 为空导致 cce-network-agent 无法启动的问题
+9. [Bug] 修改 VPC-ENI 模式下 ENI 对象本地缓存过期状态相关逻辑，解决因 resync nrs timeout 而导致的新增节点初始化慢，大规模集群扩容速度慢的问题
+
 #### 2.12.9 [20241121]
 1. [Bug] 修复 agent 在初始化 ENI 缺少 mac 地址时，会给 lo 网卡重命名的问题
 2. [Optimize] 修复 Node 不存在的异常场景时 operator getEniQuota panic 问题
@@ -19,7 +30,7 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如
 5. [Bug] 修复 EBC 主网卡重复创建 ENI 的问题
 
 #### 2.12.7 [20240923]
-1. [Optimize] psts 增加对 cep ttl 未过期时直接移除 Node 导致 cep 后续 ttl 过期后因无对应 eni 而无法正常删除时的清理逻辑
+1. [Optimize] PSTS 增加对 CEP TTL 未过期时直接移除 ENI 导致 CEP 后续 TTL 过期后因无对应 ENI 而无法正常删除时的清理逻辑
 2. [Optimize] 增加 ENI 同步时不一致信息的差异对比日志，方便出现 ENI 数据不一致时排查问题
 3. [Optimize] 去掉 ERI 的独立同步逻辑，复用 ERI 和 ENI 的同步流程
 4. [Optimize] 去掉 Underlay RDMA 的独立同步逻辑，创建 underlay RDMA 网卡后，状态不再变更
@@ -43,12 +54,12 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如
 3. [Optimize] 优化 prepareIPs 阶段对子网查询的逻辑，当 BCC 实例的 ENI 处于非 inuse 状态时，拒绝执行 IP 预备任务。
 
 #### 2.12.3 [20240730]
-1. [Bug] 修复cpsts配置namespaceSelector时误判断Selector导致没有配置namespaceSelector时的空指针问题及namespaceSelector在没有配置Selector时无法生效的问题
+1. [Bug] 修复 cpsts 配置 namespaceSelector 时误判断 Selector 导致没有配置 namespaceSelector 时的空指针问题及 namespaceSelector 在没有配置 Selector 时无法生效的问题
 2. [Optimize] 优化 psts 在没有填写子网 IP 选择策略时的本地 IP 申请器的默认工作区间，避免没有填写 IP 地址族时无法申请 IP 的问题
 3. [Bug] 修复 cilium ipam 保留 IP 时无法保留首个 IP 的问题
 
 #### 2.12.2 [2024/07/24]
-1. [Feature] 支持borrowed subnet 可观测，新增 cce_subnet_ips_guage 指标代表子网可用 IP 地址数量
+1. [Feature] 支持 borrowed subnet 可观测，新增 cce_subnet_ips_guage 指标代表子网可用 IP 地址数量
 2. [Optimize] borrowed subnet 支持定时同步能力，避免因单次 IP 计算错误，导致错误借用未归还的问题。
 3. [Optimize] 更新子网可用 IP 借用语义，单个 ENI 从子网借用 IP 地址数以最新一次为准
 
@@ -72,6 +83,15 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如
 新特性功能：
 1. 新特性：容器内支持分配 RDMA 子网卡及 RDMA 辅助IP。
 
+#### 2.11.9 [20241213]
+1. [Bug] 修改 VPC-ENI 模式下 ENI 对象本地缓存过期状态相关逻辑，解决因 resync nrs timeout 而导致的新增节点初始化慢，大规模集群扩容速度慢的问题
+
+#### 2.11.8 [20241101]
+1. [Bug] 修复 agent 在初始化 ENI 缺少 mac 地址时，会给 lo 网卡重命名的问题
+
+#### 2.11.7 [20241031]
+1. [Optimize] 增加 ENI 主 IP 获取流程，避免新节点缺少主 IP 无法就绪的问题
+
 #### 2.11.6 [20240924]
 1. [Bug] 修复 ENI 同步不支持 EHC 的问题
 

cce-network-v2/go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.8.2
-	github.com/vishvananda/netlink v1.2.1-beta.2.0.20220608195807-1a118fe229fc
+	github.com/vishvananda/netlink v1.3.0
 	go.uber.org/multierr v1.8.0
 	golang.org/x/net v0.8.0
 	golang.org/x/sync v0.1.0
@@ -121,7 +121,7 @@ require (
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/subosito/gotenv v1.3.0 // indirect
-	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
+	github.com/vishvananda/netns v0.0.4 // indirect
 	go.mongodb.org/mongo-driver v1.11.3 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/mod v0.9.0 // indirect

cce-network-v2/go.sum

@@ -650,11 +650,10 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/vishvananda/netlink v1.2.1-beta.2.0.20220608195807-1a118fe229fc h1:2wzJ1cBcM23GetRJs2y6ETXrFMvp6HefTbFWtqviHZQ=
-github.com/vishvananda/netlink v1.2.1-beta.2.0.20220608195807-1a118fe229fc/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
-github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
+github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
+github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
+github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=

cce-network-v2/operator/flags.go

@@ -259,7 +259,7 @@ func init() {
 	flags.String(operatorOption.CCEK8sNamespace, "kube-system", fmt.Sprintf("Name of the Kubernetes namespace in which CCE is deployed in. Defaults to the same namespace defined in %s", option.K8sNamespaceName))
 	option.BindEnv(operatorOption.CCEK8sNamespace)
 
-	flags.String(operatorOption.CCEPodLabels, "app.cce.baidubce.com=cce-cni-v2-agent", "CCE Pod's labels. Used to detect if a CCE pod is running to remove the node taints where its running and set NetworkUnavailable to false")
+	flags.String(operatorOption.CCEPodLabels, "app.cce.baidubce.com=cce-network-agent", "CCE Pod's labels. Used to detect if a CCE pod is running to remove the node taints where its running and set NetworkUnavailable to false")
 	option.BindEnv(operatorOption.CCEPodLabels)
 
 	flags.Bool(operatorOption.RemoveNetResourceSetTaints, true, fmt.Sprintf("Remove node taint %q from Kubernetes nodes once CCE is up and running", pkgOption.Config.AgentNotReadyNodeTaintValue()))

cce-network-v2/pkg/bce/agent/eni_link.go

@@ -86,16 +86,17 @@ func (ec *eniLink) rename(isPrimary bool) error {
 		return fmt.Errorf("failed to list links: %v", err)
 	}
 
+	udevName := elink.Attrs().Name
 	// rename link to cce-eni-{index}
-	if !strings.HasPrefix(elink.Attrs().Name, ENINamePrefix) {
-		var eniName = fmt.Sprintf("%s-%d", ENINamePrefix, eniIndex)
+	if !strings.HasPrefix(udevName, ENINamePrefix) {
+		var cceName = fmt.Sprintf("%s-%d", ENINamePrefix, eniIndex)
 		if !isPrimary {
 			// find a free index for eni
 			for i := 0; i < maxENIIndex; i++ {
 				findENI := false
-				eniName = fmt.Sprintf("%s-%d", ENINamePrefix, i)
+				cceName = fmt.Sprintf("%s-%d", ENINamePrefix, i)
 				for _, link := range linkList {
-					if link.Attrs().Name == eniName {
+					if link.Attrs().Name == cceName {
 						findENI = true
 						break
 					}
@@ -107,24 +108,24 @@ func (ec *eniLink) rename(isPrimary bool) error {
 			}
 		}
 
-		err = ec.release.HostOS().DisableDHCPv6(eniName)
+		err = ec.release.HostOS().DisableDHCPv6(udevName, cceName)
 		if err != nil {
 			return err
 		}
-		ec.log.WithField("ifname", eniName).Info("generate ifcfg file")
+		ec.log.WithField("ifname", cceName).Info("generate ifcfg file")
 
 		// Devices can be renamed only when down
 		if err = netlink.LinkSetDown(elink); err != nil {
 			return fmt.Errorf("failed to set %q down: %v", elink.Attrs().Name, err)
 		}
 
 		// Rename container device to respect args.IfName
-		if err := netlink.LinkSetName(elink, eniName); err != nil {
-			return fmt.Errorf("failed to rename device %q to %q: %v", elink.Attrs().Name, eniName, err)
+		if err := netlink.LinkSetName(elink, cceName); err != nil {
+			return fmt.Errorf("failed to rename device %q to %q: %v", elink.Attrs().Name, cceName, err)
 		}
-		elink, err = netlink.LinkByName(eniName)
+		elink, err = netlink.LinkByName(cceName)
 		if err != nil {
-			return fmt.Errorf("failed to find device %q: %v", eniName, err)
+			return fmt.Errorf("failed to find device %q: %v", cceName, err)
 		}
 	}
 

cce-network-v2/pkg/bce/agent/eni_provider.go

@@ -58,7 +58,8 @@ func (eh *eniInitFactory) OnAddENI(node *ccev2.ENI) error {
 	return nil
 }
 
-// OnUpdateENI It will be recalled every 30s
+// OnUpdateENI It will be recalled every 30s before cce-network-v2/2.11.5 and
+// cce-network-v2/2.12.7. But it is not necessary, so remove periodic recalled code.
 func (eh *eniInitFactory) OnUpdateENI(oldObj, newObj *ccev2.ENI) error {
 	var err error
 	resource := newObj.DeepCopy()
@@ -128,8 +129,16 @@ func (eh *eniInitFactory) OnUpdateENI(oldObj, newObj *ccev2.ENI) error {
 		return fmt.Errorf("failed to set eni neighbor config: %w", err)
 	}
 
-	// set device and route on the woker machine only when eni bound at bcc
+	isNeedUpdateStatus := false
+	// set device and route on the worker machine only when eni bound at bcc
 	if _, ok := eh.localENIs[resource.Spec.ENI.ID]; !ok {
+		isNeedUpdateStatus = true
+	} else if resource.Status.InterfaceIndex != eniLink.linkIndex ||
+		resource.Status.InterfaceName != eniLink.linkName || resource.Status.ENIIndex != eniLink.eniIndex ||
+		resource.Status.GatewayIPv4 != eniLink.ipv4Gateway || resource.Status.GatewayIPv6 != eniLink.ipv6Gateway {
+		isNeedUpdateStatus = true
+	}
+	if isNeedUpdateStatus {
 		resource.Status.InterfaceIndex = eniLink.linkIndex
 		resource.Status.InterfaceName = eniLink.linkName
 		resource.Status.ENIIndex = eniLink.eniIndex
@@ -139,15 +148,25 @@ func (eh *eniInitFactory) OnUpdateENI(oldObj, newObj *ccev2.ENI) error {
 		if eniLink.ipv6Gateway != "" {
 			resource.Status.GatewayIPv6 = eniLink.ipv6Gateway
 		}
+	}
 
-		if !reflect.DeepEqual(&resource.Status, &newObj.Status) {
-			(&resource.Status).AppendCCEENIStatus(ccev2.ENIStatusReadyOnNode)
+	isNeedUpdateToReadyOnNode := false
+	if !reflect.DeepEqual(&resource.Status, &newObj.Status) {
+		isNeedUpdateToReadyOnNode = true
+		isNeedUpdateStatus = true
+	} else if resource.Status.CCEStatus == ccev2.ENIStatusNone && resource.Status.VPCStatus == ccev2.VPCENIStatusInuse {
+		isNeedUpdateToReadyOnNode = true
+		isNeedUpdateStatus = true
+	}
+	if isNeedUpdateToReadyOnNode {
+		(&resource.Status).AppendCCEENIStatus(ccev2.ENIStatusReadyOnNode)
+	}
 
-			_, err = eh.eniClient.ENIs().UpdateStatus(context.TODO(), resource, metav1.UpdateOptions{})
-			if err != nil {
-				scopedLog.WithError(err).Error("update eni status")
-				return err
-			}
+	if isNeedUpdateStatus {
+		_, err = eh.eniClient.ENIs().UpdateStatus(context.TODO(), resource, metav1.UpdateOptions{})
+		if err != nil {
+			scopedLog.WithError(err).Error("update eni status")
+			return err
 		}
 	}
 

cce-network-v2/pkg/bce/bcesync/borrowed_subnet.go

@@ -197,46 +197,46 @@ func (bs *BorrowedSubnet) update(subnet *ccev1.Subnet) {
 	bs.BorrowedAvailableIPsCount = subnet.Status.AvailableIPNum - bs.BorrowedIPsCount
 }
 
-func (bs *BorrowedSubnet) Borrow(enid string, ipNum int) (borrowedIPNum int) {
+func (bs *BorrowedSubnet) Borrow(eniID string, ipNum int) (borrowedIPNum int) {
 	bs.mutex.Lock()
 	defer bs.mutex.Unlock()
 	if bs.BorrowedAvailableIPsCount < ipNum {
 		bs.logger().WithFields(logrus.Fields{
 			"task":      "borrow",
-			"eniID":     enid,
+			"eniID":     eniID,
 			"needIPNum": ipNum,
 			"tasks":     logfields.Json(bs.tasks),
 		}).Warning("subnet not enough available ips to borrow by eni")
 		return
 	}
 
-	return bs._forceBorrowForENI(enid, ipNum)
+	return bs._forceBorrowForENI(eniID, ipNum)
 }
 
 // forceBorrowForENI borrow ip for eni
 // return borrowed ip num
-func (bs *BorrowedSubnet) forceBorrowForENI(enid string, ipNum int) int {
+func (bs *BorrowedSubnet) forceBorrowForENI(eniID string, ipNum int) int {
 	bs.mutex.Lock()
 	defer bs.mutex.Unlock()
 
-	return bs._forceBorrowForENI(enid, ipNum)
+	return bs._forceBorrowForENI(eniID, ipNum)
 }
 
-func (bs *BorrowedSubnet) _forceBorrowForENI(enid string, ipNum int) int {
+func (bs *BorrowedSubnet) _forceBorrowForENI(eniID string, ipNum int) int {
 	var (
 		eniBorrowedIPNum int
 		sbnAvailBorrowIP int
 	)
-	if task, ok := bs.tasks[enid]; ok {
+	if task, ok := bs.tasks[eniID]; ok {
 		bs.BorrowedIPsCount -= task.IPNum
 		sbnAvailBorrowIP = bs.Status.AvailableIPNum - bs.BorrowedIPsCount
 		eniBorrowedIPNum = math.IntMin(sbnAvailBorrowIP, ipNum)
 		task.IPNum = eniBorrowedIPNum
-		bs.tasks[enid] = task
+		bs.tasks[eniID] = task
 	} else {
 		sbnAvailBorrowIP = bs.Status.AvailableIPNum - bs.BorrowedIPsCount
 		eniBorrowedIPNum = math.IntMin(sbnAvailBorrowIP, ipNum)
-		bs.tasks[enid] = IPBorrowTask{SubnetId: bs.SubnetId, EniID: enid, IPNum: eniBorrowedIPNum}
+		bs.tasks[eniID] = IPBorrowTask{SubnetId: bs.SubnetId, EniID: eniID, IPNum: eniBorrowedIPNum}
 	}
 
 	bs.BorrowedIPsCount += eniBorrowedIPNum
@@ -245,7 +245,7 @@ func (bs *BorrowedSubnet) _forceBorrowForENI(enid string, ipNum int) int {
 	if eniBorrowedIPNum < ipNum {
 		bs.logger().WithFields(logrus.Fields{
 			"task":                            "forceBorrowForENI",
-			"eniID":                           "enid",
+			"eniID":                           eniID,
 			"sbnID":                           bs.SubnetId,
 			"needIPNum":                       ipNum,
 			"eniBorrowedIPNum":                eniBorrowedIPNum,
@@ -258,10 +258,10 @@ func (bs *BorrowedSubnet) _forceBorrowForENI(enid string, ipNum int) int {
 	return eniBorrowedIPNum
 }
 
-func (bs *BorrowedSubnet) Done(enid string, ipNum int) {
+func (bs *BorrowedSubnet) Done(eniID string, ipNum int) {
 	bs.mutex.Lock()
 	defer bs.mutex.Unlock()
-	if task, ok := bs.tasks[enid]; ok {
+	if task, ok := bs.tasks[eniID]; ok {
 		if task.IPNum < ipNum {
 			ipNum = task.IPNum
 			task.IPNum = 0