diff --git a/cce-network-v2/VERSION b/cce-network-v2/VERSION index ffbe9e2..420fbec 100644 --- a/cce-network-v2/VERSION +++ b/cce-network-v2/VERSION @@ -1 +1 @@ -2.12.9 +2.12.10 diff --git a/cce-network-v2/cmd/agent/cmd/daemon.go b/cce-network-v2/cmd/agent/cmd/daemon.go index b99eec1..88353e8 100644 --- a/cce-network-v2/cmd/agent/cmd/daemon.go +++ b/cce-network-v2/cmd/agent/cmd/daemon.go @@ -292,6 +292,10 @@ func NewDaemon(ctx context.Context, cancel context.CancelFunc) (*Daemon, error) bootstrapStats.k8sInit.End(true) } + // confugure and start ENIM + d.configureENIM() + d.startENIM() + // Configure IPAM without using the configuration yet. d.configureIPAM() d.startIPAM() @@ -303,10 +307,6 @@ func NewDaemon(ctx context.Context, cancel context.CancelFunc) (*Daemon, error) debug.RegisterStatusObject("rdmaIpam["+key+"]", ri) } - // confugure and start ENIM - d.configureENIM() - d.startENIM() - // endpoints handler d.startEndpointHanler() diff --git a/cce-network-v2/deploy/cce-network-v2-2.12.tar.gz b/cce-network-v2/deploy/cce-network-v2-2.12.tar.gz index 53449be..7849e50 100644 Binary files a/cce-network-v2/deploy/cce-network-v2-2.12.tar.gz and b/cce-network-v2/deploy/cce-network-v2-2.12.tar.gz differ diff --git a/cce-network-v2/deploy/cce-network-v2/Chart.yaml b/cce-network-v2/deploy/cce-network-v2/Chart.yaml index fb7bae9..e0fd470 100644 --- a/cce-network-v2/deploy/cce-network-v2/Chart.yaml +++ b/cce-network-v2/deploy/cce-network-v2/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.12.0 +version: 2.12.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.12.0" +appVersion: "2.12.10" diff --git a/cce-network-v2/docs/release.md b/cce-network-v2/docs/release.md index e0617e3..ca72a09 100644 --- a/cce-network-v2/docs/release.md +++ b/cce-network-v2/docs/release.md @@ -7,6 +7,17 @@ v2 版本新架构,支持VPC-ENI 辅助IP和vpc路由。版本发布历史如 2. 增加 eni 安全组同步功能, 保持CCE ENI 和节点安全组同步。 3. 增加节点网络配置集功能 NetResourceConfigSet,支持指定节点独立配置网络资源。 +#### 2.12.10 [20241213] +1. [Optimize] 优化 VPC-ENI 模式下的 veth 单机路由规则目的地址存在冲突的判断条件,解决残留本地路由规则时创建的 Pod 容器网络不通的问题 +2. [Optimize] 禁用 VPC-ENI 模式下的 IPv6 DHCP 时使用的网卡名修改为udev生成的原始名字,避免生成的网卡配置文件导致虚拟机重启时 network.service 服务启动失败 +3. [Bug] 更新 VPC-ENI 模式下的 ENI 状态机处理逻辑,修复当节点移出集群时,ENI 对象残留的问题 +4. [Bug] 修复 VPC-ENI 模式下的新建 ENI 网卡 borrow IP 地址时,borrow task ENIID 填写错误导致并发创建时可能引起的 borrow IP 计算错误 +5. [Optimize] 优化 VPC-ENI 模式下的 Romote ENI Syncer 逻辑,纳管 CCE 历史创建的 ENI,避免因历史创建的 ENI 无法被使用而占用 Node eniQuota 的问题 +6. [Optimize] 优化 VPC-ENI 模式下的 ENI 状态机处理逻辑,支持非终态 ENI 的处理流程,避免因状态机异常中断、重启丢失内存状态等原因导致的 ENI 状态处于异常的非终态而导致的该 ENI 无法正常使用的问题 +7. [Bug] PSTS 增加对 CEP TTL 未过期时直接移除 Node 或 ENI 导致 CEP 后续 TTL 过期后因无对应的 NetworkResourceSet 或 ENI 而无法正常删除时的清理逻辑 +8. [Bug] 修复 VPC-ENI 模式下 EBC ENI Quota 为 0 时,启用主网卡辅助IP的 ENI 赋值逻辑,避免因主网卡 ENI MacAddress 为空导致 cce-network-agent 无法启动的问题 +9. [Bug] 修改 VPC-ENI 模式下 ENI 对象本地缓存过期状态相关逻辑,解决因 resync nrs timeout 而导致的新增节点初始化慢,大规模集群扩容速度慢的问题 + #### 2.12.9 [20241121] 1. [Bug] 修复 agent 在初始化 ENI 缺少 mac 地址时,会给 lo 网卡重命名的问题 2. [Optimize] 修复 Node 不存在的异常场景时 operator getEniQuota panic 问题 @@ -19,7 +30,7 @@ v2 版本新架构,支持VPC-ENI 辅助IP和vpc路由。版本发布历史如 5. [Bug] 修复 EBC 主网卡重复创建 ENI 的问题 #### 2.12.7 [20240923] -1. [Optimize] psts 增加对 cep ttl 未过期时直接移除 Node 导致 cep 后续 ttl 过期后因无对应 eni 而无法正常删除时的清理逻辑 +1. [Optimize] PSTS 增加对 CEP TTL 未过期时直接移除 ENI 导致 CEP 后续 TTL 过期后因无对应 ENI 而无法正常删除时的清理逻辑 2. [Optimize] 增加 ENI 同步时不一致信息的差异对比日志,方便出现 ENI 数据不一致时排查问题 3. [Optimize] 去掉 ERI 的独立同步逻辑,复用 ERI 和 ENI 的同步流程 4. [Optimize] 去掉 Underlay RDMA 的独立同步逻辑,创建 underlay RDMA 网卡后,状态不再变更 @@ -43,12 +54,12 @@ v2 版本新架构,支持VPC-ENI 辅助IP和vpc路由。版本发布历史如 3. [Optimize] 优化 prepareIPs 阶段对子网查询的逻辑,当 BCC 实例的 ENI 处于非 inuse 状态时,拒绝执行 IP 预备任务。 #### 2.12.3 [20240730] -1. [Bug] 修复cpsts配置namespaceSelector时误判断Selector导致没有配置namespaceSelector时的空指针问题及namespaceSelector在没有配置Selector时无法生效的问题 +1. [Bug] 修复 cpsts 配置 namespaceSelector 时误判断 Selector 导致没有配置 namespaceSelector 时的空指针问题及 namespaceSelector 在没有配置 Selector 时无法生效的问题 2. [Optimize] 优化 psts 在没有填写子网 IP 选择策略时的本地 IP 申请器的默认工作区间,避免没有填写 IP 地址族时无法申请 IP 的问题 3. [Bug] 修复 cilium ipam 保留 IP 时无法保留首个 IP 的问题 #### 2.12.2 [2024/07/24] -1. [Feature] 支持borrowed subnet 可观测,新增 cce_subnet_ips_guage 指标代表子网可用 IP 地址数量 +1. [Feature] 支持 borrowed subnet 可观测,新增 cce_subnet_ips_guage 指标代表子网可用 IP 地址数量 2. [Optimize] borrowed subnet 支持定时同步能力,避免因单次 IP 计算错误,导致错误借用未归还的问题。 3. [Optimize] 更新子网可用 IP 借用语义,单个 ENI 从子网借用 IP 地址数以最新一次为准 @@ -72,6 +83,15 @@ v2 版本新架构,支持VPC-ENI 辅助IP和vpc路由。版本发布历史如 新特性功能: 1. 新特性:容器内支持分配 RDMA 子网卡及 RDMA 辅助IP。 +#### 2.11.9 [20241213] +1. [Bug] 修改 VPC-ENI 模式下 ENI 对象本地缓存过期状态相关逻辑,解决因 resync nrs timeout 而导致的新增节点初始化慢,大规模集群扩容速度慢的问题 + +#### 2.11.8 [20241101] +1. [Bug] 修复 agent 在初始化 ENI 缺少 mac 地址时,会给 lo 网卡重命名的问题 + +#### 2.11.7 [20241031] +1. [Optimize] 增加 ENI 主 IP 获取流程,避免新节点缺少主 IP 无法就绪的问题 + #### 2.11.6 [20240924] 1. [Bug] 修复 ENI 同步不支持 EHC 的问题 diff --git a/cce-network-v2/go.mod b/cce-network-v2/go.mod index d3d9044..8ae931f 100644 --- a/cce-network-v2/go.mod +++ b/cce-network-v2/go.mod @@ -38,7 +38,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.8.2 - github.com/vishvananda/netlink v1.2.1-beta.2.0.20220608195807-1a118fe229fc + github.com/vishvananda/netlink v1.3.0 go.uber.org/multierr v1.8.0 golang.org/x/net v0.8.0 golang.org/x/sync v0.1.0 @@ -121,7 +121,7 @@ require ( github.com/spf13/afero v1.8.2 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect - github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect + github.com/vishvananda/netns v0.0.4 // indirect go.mongodb.org/mongo-driver v1.11.3 // indirect go.uber.org/atomic v1.9.0 // indirect golang.org/x/mod v0.9.0 // indirect diff --git a/cce-network-v2/go.sum b/cce-network-v2/go.sum index e375251..2e3a957 100644 --- a/cce-network-v2/go.sum +++ b/cce-network-v2/go.sum @@ -650,11 +650,10 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/vishvananda/netlink v1.2.1-beta.2.0.20220608195807-1a118fe229fc h1:2wzJ1cBcM23GetRJs2y6ETXrFMvp6HefTbFWtqviHZQ= -github.com/vishvananda/netlink v1.2.1-beta.2.0.20220608195807-1a118fe229fc/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= -github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= diff --git a/cce-network-v2/operator/flags.go b/cce-network-v2/operator/flags.go index 17c2619..e597c0f 100644 --- a/cce-network-v2/operator/flags.go +++ b/cce-network-v2/operator/flags.go @@ -259,7 +259,7 @@ func init() { flags.String(operatorOption.CCEK8sNamespace, "kube-system", fmt.Sprintf("Name of the Kubernetes namespace in which CCE is deployed in. Defaults to the same namespace defined in %s", option.K8sNamespaceName)) option.BindEnv(operatorOption.CCEK8sNamespace) - flags.String(operatorOption.CCEPodLabels, "app.cce.baidubce.com=cce-cni-v2-agent", "CCE Pod's labels. Used to detect if a CCE pod is running to remove the node taints where its running and set NetworkUnavailable to false") + flags.String(operatorOption.CCEPodLabels, "app.cce.baidubce.com=cce-network-agent", "CCE Pod's labels. Used to detect if a CCE pod is running to remove the node taints where its running and set NetworkUnavailable to false") option.BindEnv(operatorOption.CCEPodLabels) flags.Bool(operatorOption.RemoveNetResourceSetTaints, true, fmt.Sprintf("Remove node taint %q from Kubernetes nodes once CCE is up and running", pkgOption.Config.AgentNotReadyNodeTaintValue())) diff --git a/cce-network-v2/pkg/bce/agent/eni_link.go b/cce-network-v2/pkg/bce/agent/eni_link.go index 4854242..ed36f26 100644 --- a/cce-network-v2/pkg/bce/agent/eni_link.go +++ b/cce-network-v2/pkg/bce/agent/eni_link.go @@ -86,16 +86,17 @@ func (ec *eniLink) rename(isPrimary bool) error { return fmt.Errorf("failed to list links: %v", err) } + udevName := elink.Attrs().Name // rename link to cce-eni-{index} - if !strings.HasPrefix(elink.Attrs().Name, ENINamePrefix) { - var eniName = fmt.Sprintf("%s-%d", ENINamePrefix, eniIndex) + if !strings.HasPrefix(udevName, ENINamePrefix) { + var cceName = fmt.Sprintf("%s-%d", ENINamePrefix, eniIndex) if !isPrimary { // find a free index for eni for i := 0; i < maxENIIndex; i++ { findENI := false - eniName = fmt.Sprintf("%s-%d", ENINamePrefix, i) + cceName = fmt.Sprintf("%s-%d", ENINamePrefix, i) for _, link := range linkList { - if link.Attrs().Name == eniName { + if link.Attrs().Name == cceName { findENI = true break } @@ -107,11 +108,11 @@ func (ec *eniLink) rename(isPrimary bool) error { } } - err = ec.release.HostOS().DisableDHCPv6(eniName) + err = ec.release.HostOS().DisableDHCPv6(udevName, cceName) if err != nil { return err } - ec.log.WithField("ifname", eniName).Info("generate ifcfg file") + ec.log.WithField("ifname", cceName).Info("generate ifcfg file") // Devices can be renamed only when down if err = netlink.LinkSetDown(elink); err != nil { @@ -119,12 +120,12 @@ func (ec *eniLink) rename(isPrimary bool) error { } // Rename container device to respect args.IfName - if err := netlink.LinkSetName(elink, eniName); err != nil { - return fmt.Errorf("failed to rename device %q to %q: %v", elink.Attrs().Name, eniName, err) + if err := netlink.LinkSetName(elink, cceName); err != nil { + return fmt.Errorf("failed to rename device %q to %q: %v", elink.Attrs().Name, cceName, err) } - elink, err = netlink.LinkByName(eniName) + elink, err = netlink.LinkByName(cceName) if err != nil { - return fmt.Errorf("failed to find device %q: %v", eniName, err) + return fmt.Errorf("failed to find device %q: %v", cceName, err) } } diff --git a/cce-network-v2/pkg/bce/agent/eni_provider.go b/cce-network-v2/pkg/bce/agent/eni_provider.go index 1f3cc73..c0add09 100644 --- a/cce-network-v2/pkg/bce/agent/eni_provider.go +++ b/cce-network-v2/pkg/bce/agent/eni_provider.go @@ -58,7 +58,8 @@ func (eh *eniInitFactory) OnAddENI(node *ccev2.ENI) error { return nil } -// OnUpdateENI It will be recalled every 30s +// OnUpdateENI It will be recalled every 30s before cce-network-v2/2.11.5 and +// cce-network-v2/2.12.7. But it is not necessary, so remove periodic recalled code. func (eh *eniInitFactory) OnUpdateENI(oldObj, newObj *ccev2.ENI) error { var err error resource := newObj.DeepCopy() @@ -128,8 +129,16 @@ func (eh *eniInitFactory) OnUpdateENI(oldObj, newObj *ccev2.ENI) error { return fmt.Errorf("failed to set eni neighbor config: %w", err) } - // set device and route on the woker machine only when eni bound at bcc + isNeedUpdateStatus := false + // set device and route on the worker machine only when eni bound at bcc if _, ok := eh.localENIs[resource.Spec.ENI.ID]; !ok { + isNeedUpdateStatus = true + } else if resource.Status.InterfaceIndex != eniLink.linkIndex || + resource.Status.InterfaceName != eniLink.linkName || resource.Status.ENIIndex != eniLink.eniIndex || + resource.Status.GatewayIPv4 != eniLink.ipv4Gateway || resource.Status.GatewayIPv6 != eniLink.ipv6Gateway { + isNeedUpdateStatus = true + } + if isNeedUpdateStatus { resource.Status.InterfaceIndex = eniLink.linkIndex resource.Status.InterfaceName = eniLink.linkName resource.Status.ENIIndex = eniLink.eniIndex @@ -139,15 +148,25 @@ func (eh *eniInitFactory) OnUpdateENI(oldObj, newObj *ccev2.ENI) error { if eniLink.ipv6Gateway != "" { resource.Status.GatewayIPv6 = eniLink.ipv6Gateway } + } - if !reflect.DeepEqual(&resource.Status, &newObj.Status) { - (&resource.Status).AppendCCEENIStatus(ccev2.ENIStatusReadyOnNode) + isNeedUpdateToReadyOnNode := false + if !reflect.DeepEqual(&resource.Status, &newObj.Status) { + isNeedUpdateToReadyOnNode = true + isNeedUpdateStatus = true + } else if resource.Status.CCEStatus == ccev2.ENIStatusNone && resource.Status.VPCStatus == ccev2.VPCENIStatusInuse { + isNeedUpdateToReadyOnNode = true + isNeedUpdateStatus = true + } + if isNeedUpdateToReadyOnNode { + (&resource.Status).AppendCCEENIStatus(ccev2.ENIStatusReadyOnNode) + } - _, err = eh.eniClient.ENIs().UpdateStatus(context.TODO(), resource, metav1.UpdateOptions{}) - if err != nil { - scopedLog.WithError(err).Error("update eni status") - return err - } + if isNeedUpdateStatus { + _, err = eh.eniClient.ENIs().UpdateStatus(context.TODO(), resource, metav1.UpdateOptions{}) + if err != nil { + scopedLog.WithError(err).Error("update eni status") + return err } } diff --git a/cce-network-v2/pkg/bce/bcesync/borrowed_subnet.go b/cce-network-v2/pkg/bce/bcesync/borrowed_subnet.go index adea71e..a03982f 100644 --- a/cce-network-v2/pkg/bce/bcesync/borrowed_subnet.go +++ b/cce-network-v2/pkg/bce/bcesync/borrowed_subnet.go @@ -197,46 +197,46 @@ func (bs *BorrowedSubnet) update(subnet *ccev1.Subnet) { bs.BorrowedAvailableIPsCount = subnet.Status.AvailableIPNum - bs.BorrowedIPsCount } -func (bs *BorrowedSubnet) Borrow(enid string, ipNum int) (borrowedIPNum int) { +func (bs *BorrowedSubnet) Borrow(eniID string, ipNum int) (borrowedIPNum int) { bs.mutex.Lock() defer bs.mutex.Unlock() if bs.BorrowedAvailableIPsCount < ipNum { bs.logger().WithFields(logrus.Fields{ "task": "borrow", - "eniID": enid, + "eniID": eniID, "needIPNum": ipNum, "tasks": logfields.Json(bs.tasks), }).Warning("subnet not enough available ips to borrow by eni") return } - return bs._forceBorrowForENI(enid, ipNum) + return bs._forceBorrowForENI(eniID, ipNum) } // forceBorrowForENI borrow ip for eni // return borrowed ip num -func (bs *BorrowedSubnet) forceBorrowForENI(enid string, ipNum int) int { +func (bs *BorrowedSubnet) forceBorrowForENI(eniID string, ipNum int) int { bs.mutex.Lock() defer bs.mutex.Unlock() - return bs._forceBorrowForENI(enid, ipNum) + return bs._forceBorrowForENI(eniID, ipNum) } -func (bs *BorrowedSubnet) _forceBorrowForENI(enid string, ipNum int) int { +func (bs *BorrowedSubnet) _forceBorrowForENI(eniID string, ipNum int) int { var ( eniBorrowedIPNum int sbnAvailBorrowIP int ) - if task, ok := bs.tasks[enid]; ok { + if task, ok := bs.tasks[eniID]; ok { bs.BorrowedIPsCount -= task.IPNum sbnAvailBorrowIP = bs.Status.AvailableIPNum - bs.BorrowedIPsCount eniBorrowedIPNum = math.IntMin(sbnAvailBorrowIP, ipNum) task.IPNum = eniBorrowedIPNum - bs.tasks[enid] = task + bs.tasks[eniID] = task } else { sbnAvailBorrowIP = bs.Status.AvailableIPNum - bs.BorrowedIPsCount eniBorrowedIPNum = math.IntMin(sbnAvailBorrowIP, ipNum) - bs.tasks[enid] = IPBorrowTask{SubnetId: bs.SubnetId, EniID: enid, IPNum: eniBorrowedIPNum} + bs.tasks[eniID] = IPBorrowTask{SubnetId: bs.SubnetId, EniID: eniID, IPNum: eniBorrowedIPNum} } bs.BorrowedIPsCount += eniBorrowedIPNum @@ -245,7 +245,7 @@ func (bs *BorrowedSubnet) _forceBorrowForENI(enid string, ipNum int) int { if eniBorrowedIPNum < ipNum { bs.logger().WithFields(logrus.Fields{ "task": "forceBorrowForENI", - "eniID": "enid", + "eniID": eniID, "sbnID": bs.SubnetId, "needIPNum": ipNum, "eniBorrowedIPNum": eniBorrowedIPNum, @@ -258,10 +258,10 @@ func (bs *BorrowedSubnet) _forceBorrowForENI(enid string, ipNum int) int { return eniBorrowedIPNum } -func (bs *BorrowedSubnet) Done(enid string, ipNum int) { +func (bs *BorrowedSubnet) Done(eniID string, ipNum int) { bs.mutex.Lock() defer bs.mutex.Unlock() - if task, ok := bs.tasks[enid]; ok { + if task, ok := bs.tasks[eniID]; ok { if task.IPNum < ipNum { ipNum = task.IPNum task.IPNum = 0 diff --git a/cce-network-v2/pkg/bce/bcesync/eni.go b/cce-network-v2/pkg/bce/bcesync/eni.go index 8d96a62..3b565a0 100644 --- a/cce-network-v2/pkg/bce/bcesync/eni.go +++ b/cce-network-v2/pkg/bce/bcesync/eni.go @@ -210,27 +210,25 @@ func (es *eniSyncher) handleENIUpdate(resource *ccev2.ENI, scopeLog *logrus.Entr return nil } - skipRefresh := es.mangeFinalizer(newObj) - if !skipRefresh { - if es.remoteSyncer.useENIMachine() { - scopeLog.Debug("start eni machine") - // start machine - machine := eniStateMachine{ - es: es, - ctx: ctx, - resource: newObj, - scopeLog: scopeLog, - } + isNeedUpdate := es.mangeFinalizer(newObj) + if es.remoteSyncer.useENIMachine() && isNeedUpdate { + scopeLog.Debug("start eni machine") + // start machine + machine := eniStateMachine{ + es: es, + ctx: ctx, + resource: newObj, + scopeLog: scopeLog, + } - err = machine.start() - eniStatus = &newObj.Status - _, isDelayError := err.(*cm.DelayEvent) - if isDelayError { - goto updateStatus - } else if err != nil { - scopeLog.WithError(err).Error("eni machine failed") - return err - } + err = machine.start() + eniStatus = &newObj.Status + _, isDelayError := err.(*cm.DelayEvent) + if isDelayError { + goto updateStatus + } else if err != nil { + scopeLog.WithError(err).Error("eni machine failed") + return err } } @@ -254,10 +252,12 @@ updateStatus: } // mangeFinalizer except for node deletion, direct deletion of ENI objects is prohibited -// return true: should delete this object -func (*eniSyncher) mangeFinalizer(newObj *ccev2.ENI) bool { +// return true: should update this object (*ccev2.ENI) +func (*eniSyncher) mangeFinalizer(newObj *ccev2.ENI) (isNeedUpdate bool) { + isNeedUpdate = false if newObj.DeletionTimestamp == nil && len(newObj.Finalizers) == 0 { newObj.Finalizers = append(newObj.Finalizers, FinalizerENI) + isNeedUpdate = true } var finalizers []string @@ -279,7 +279,7 @@ func (*eniSyncher) mangeFinalizer(newObj *ccev2.ENI) bool { goto removeFinalizer } } - return false + return removeFinalizer: for _, f := range newObj.Finalizers { @@ -290,7 +290,8 @@ removeFinalizer: } newObj.Finalizers = finalizers log.Infof("remove finalizer from deletable ENI %s on NetResourceSet %s ", newObj.Name, newObj.Spec.NodeName) - return true + isNeedUpdate = true + return } func (es *eniSyncher) Delete(name string) error { @@ -351,6 +352,20 @@ func (esm *eniStateMachine) start() error { return updateError } esm.scopeLog.Info("update eni spec success") + } else { + _, err = esm.es.remoteSyncer.statENI(esm.ctx, esm.resource.Name) + if err != nil { + esm.scopeLog.Infof("eni state machine failed to get inuse eni(%s): %v", esm.resource.Name, err) + (&esm.resource.Status).AppendVPCStatus(ccev2.VPCENIStatusNone) + // update spec + _, updateError := esm.es.updater.Update(esm.resource) + if updateError != nil { + esm.scopeLog.WithError(updateError).Error("update eni spec failed") + return updateError + } + esm.scopeLog.Info("update eni spec success") + return nil + } } } else if esm.resource.Status.VPCStatus != ccev2.VPCENIStatusDeleted { // refresh status of ENI diff --git a/cce-network-v2/pkg/bce/bcesync/remote_eni_syncer.go b/cce-network-v2/pkg/bce/bcesync/remote_eni_syncer.go index 26e47a2..7bf0dcf 100644 --- a/cce-network-v2/pkg/bce/bcesync/remote_eni_syncer.go +++ b/cce-network-v2/pkg/bce/bcesync/remote_eni_syncer.go @@ -15,6 +15,7 @@ import ( "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/operator/watchers" "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/bce/api/cloud" "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/bce/api/eni" + "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/defaults" "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s" ccev2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2" "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/logging/logfields" @@ -57,17 +58,27 @@ func (es *remoteVpcEniSyncher) syncENI(ctx context.Context) (result []eni.Eni, e VpcId: es.VPCIDs, Name: fmt.Sprintf("%s/", es.ClusterID), } + scopedLog := log.WithField(taskLogField, eniControllerName). + WithField("request", logfields.Json(listArgs)) enis, err := es.bceclient.ListENIs(context.TODO(), listArgs) if err != nil { - log.WithField(taskLogField, eniControllerName). - WithField("request", logfields.Json(listArgs)). - WithError(err).Errorf("sync eni failed") + scopedLog.WithError(err).Errorf("sync eni failed") return result, err } for i := 0; i < len(enis); i++ { result = append(result, eni.Eni{Eni: enis[i]}) - es.createExternalENI(&enis[i]) + isExisted, objEni := es.createExternalENI(&enis[i]) + if isExisted && objEni != nil { + if objEni.Status.CCEStatus != ccev2.ENIStatusReadyOnNode && objEni.Status.VPCStatus == ccev2.VPCENIStatusInuse { + (&objEni.Status).AppendCCEENIStatus(ccev2.ENIStatusReadyOnNode) + } + _, err = es.updater.UpdateStatus(objEni) + if err != nil { + scopedLog.WithError(err).Error("update eni status") + return + } + } } return } @@ -80,13 +91,17 @@ func (es *remoteVpcEniSyncher) useENIMachine() bool { // If this ENI is missing, CCE will continue to try to create new ENIs. // This will result in the inability to properly identify the capacity // of the ENI -func (es *remoteVpcEniSyncher) createExternalENI(eni *enisdk.Eni) { +// returns: +// +// isExisted: true: eni object is existed, false: eni object is not existed. +// result: the ENI object if it is created or existed, nil otherwise. +func (es *remoteVpcEniSyncher) createExternalENI(eni *enisdk.Eni) (isExisted bool, result *ccev2.ENI) { old, err := es.updater.Lister().Get(eni.EniId) if err != nil && !kerrors.IsNotFound(err) { - return + return false, nil } if old != nil { - return + return true, old } if eni.Status == string(ccev2.VPCENIStatusDetaching) || eni.Status == string(ccev2.VPCENIStatusDeleted) { @@ -102,10 +117,10 @@ func (es *remoteVpcEniSyncher) createExternalENI(eni *enisdk.Eni) { // find node by instanceID nrsList, err := watchers.NetResourceSetClient.GetByInstanceID(eni.InstanceId) - - if len(nrsList) == 0 { + if err != nil || len(nrsList) == 0 { return } + resource := nrsList[0] scopeLog = scopeLog.WithField("nodeName", resource.Name) scopeLog.Debugf("find node by instanceID success") @@ -146,10 +161,14 @@ func (es *remoteVpcEniSyncher) createExternalENI(eni *enisdk.Eni) { InstallSourceBasedRouting: resource.Spec.ENI.InstallSourceBasedRouting, }, } - _, err = es.updater.Create(newENI) + if eni.Description == defaults.DefaultENIDescription && eni.Status == string(ccev2.VPCENIStatusInuse) { + (&newENI.Status).AppendCCEENIStatus(ccev2.ENIStatusReadyOnNode) + } + result, err = es.updater.Create(newENI) if err != nil { es.eventRecorder.Eventf(resource, corev1.EventTypeWarning, "FailedCreateExternalENI", "failed to create external ENI on nrs %s: %s", resource.Name, err) - return + return false, nil } es.eventRecorder.Eventf(resource, corev1.EventTypeNormal, "CreateExternalENISuccess", "create external ENI %s on nrs %s success", eni.EniId, resource.Name) + return true, result } diff --git a/cce-network-v2/pkg/bce/vpceni/instances.go b/cce-network-v2/pkg/bce/vpceni/instances.go index 3899b8e..9605bd5 100644 --- a/cce-network-v2/pkg/bce/vpceni/instances.go +++ b/cce-network-v2/pkg/bce/vpceni/instances.go @@ -16,9 +16,13 @@ package vpceni import ( "context" - "fmt" "time" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/operator/watchers" "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/bce/api/cloud" "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/endpoint" @@ -26,13 +30,10 @@ import ( ipamTypes "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ipam/types" "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s" ccev2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2" + v2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2" listv1 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/client/listers/cce.baidubce.com/v1" listv2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/client/listers/cce.baidubce.com/v2" - "github.com/sirupsen/logrus" - "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/lock" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" ) // InstancesManager maintains the list of instances. It must be kept up to date @@ -135,16 +136,16 @@ func (m *InstancesManager) Resync(ctx context.Context) time.Time { // NodeEndpoint implements endpoint.DirectIPAllocator func (m *InstancesManager) NodeEndpoint(cep *ccev2.CCEEndpoint) (endpoint.DirectEndpointOperation, error) { nodeIP := cep.Spec.Network.IPAllocation.NodeIP - obj, err := k8s.CCEClient().Informers.Cce().V2().NetResourceSets().Lister().Get(nodeIP) - if err != nil || obj == nil { - return nil, fmt.Errorf("endpoint nrs %s not found, recommend using IP as the node name", nodeIP) + _, err := k8s.CCEClient().Informers.Cce().V2().NetResourceSets().Lister().Get(nodeIP) + if err != nil { + return nil, err } m.mutex.Lock() - bceNrs, ok := m.bceNetworkResourceSetMap[cep.Spec.Network.IPAllocation.NodeIP] + bceNrs, ok := m.bceNetworkResourceSetMap[nodeIP] m.mutex.Unlock() if !ok { - return nil, fmt.Errorf("node %s not found", cep.Spec.Network.IPAllocation.NodeIP) + return nil, errors.NewNotFound(v2.Resource("netresourceset"), nodeIP) } return bceNrs, nil } diff --git a/cce-network-v2/pkg/bce/vpceni/node_bcc.go b/cce-network-v2/pkg/bce/vpceni/node_bcc.go index d52b4db..62a9ae5 100644 --- a/cce-network-v2/pkg/bce/vpceni/node_bcc.go +++ b/cce-network-v2/pkg/bce/vpceni/node_bcc.go @@ -73,13 +73,6 @@ func (n *bccNetworkResourceSet) refreshBCCInfo() (*bccapi.InstanceModel, error) return n.bccInfo, err } n.log.WithField("bccInfo", logfields.Repr(bccInfo)).Infof("Get bcc instance detail") - // TODO delete this test code block after bcc instance eniQuota is fixed - // The tentative plan is for April 15th - { - if bccInfo.Spec == "ebc.la2.c256m1024.2d" { - bccInfo.EniQuota = 0 - } - } if bccInfo.EniQuota == 0 { n.usePrimaryENIWithSecondaryMode = true diff --git a/cce-network-v2/pkg/bce/vpceni/node_bcc_eni.go b/cce-network-v2/pkg/bce/vpceni/node_bcc_eni.go index c6d2cc3..b90474e 100644 --- a/cce-network-v2/pkg/bce/vpceni/node_bcc_eni.go +++ b/cce-network-v2/pkg/bce/vpceni/node_bcc_eni.go @@ -101,7 +101,12 @@ func (m *InstancesManager) ForeachInstance(instanceID, nodeName string, fn ipamT return enis[i].CreationTimestamp.After(enis[j].CreationTimestamp.Time) }) for i := 0; i < len(enis); i++ { - if enis[i].DeletionTimestamp != nil || enis[i].Status.VPCStatus == ccev2.VPCENIStatusDeleted { + eni := enis[i] + vpcStatus := eni.Status.VPCStatus + // Do not process the ENI which is being deleted or in attaching/detaching status. + // It is not useful to process it, because the IPs are not assigned to the ENI. + if eni.DeletionTimestamp != nil || vpcStatus == ccev2.VPCENIStatusAttaching || + vpcStatus == ccev2.VPCENIStatusDetaching || vpcStatus == ccev2.VPCENIStatusDeleted { continue } err = fn(instanceID, enis[i].Spec.ENI.ID, ipamTypes.InterfaceRevision{ @@ -117,8 +122,9 @@ func (m *InstancesManager) ForeachInstance(instanceID, nodeName string, fn ipamT // waitForENISynced wait for eni synced // this method should not lock the mutex of bceNode before calling func (n *bceNetworkResourceSet) waitForENISynced(ctx context.Context) { - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() + wait.PollImmediateUntilWithContext(ctx, 200*time.Millisecond, func(ctx context.Context) (done bool, err error) { haveSynced := true n.manager.ForeachInstance(n.instanceID, n.k8sObj.Name, @@ -141,7 +147,6 @@ func (n *bceNetworkResourceSet) waitForENISynced(ctx context.Context) { }) return haveSynced, nil }) - } // CreateInterface create a new ENI @@ -276,10 +281,11 @@ func (n *bccNetworkResourceSet) createENIOnCluster(ctx context.Context, scopedLo scopedLog = scopedLog.WithField("eniName", eniName).WithField("securityGroupIDs", resource.Spec.ENI.SecurityGroups) eniID, err := n.createENI(ctx, newENI, scopedLog) if err != nil { - n.eventRecorder.Eventf(resource, corev1.EventTypeWarning, "FailedCreateENI", "failed to create ENI on nrs %s: %s", resource.Name, err) + n.eventRecorder.Eventf(resource, corev1.EventTypeWarning, "FailedCreateENI", "failed to create ENI on node %s: %s", resource.Name, err) return err } - + newENI.Spec.ENI.ID = eniID + newENI.Name = eniID err = n.tryBorrowIPs(newENI) if err != nil { n.deleteENI(ctx, eniID, scopedLog) @@ -288,8 +294,6 @@ func (n *bccNetworkResourceSet) createENIOnCluster(ctx context.Context, scopedLo n.eventRecorder.Eventf(resource, corev1.EventTypeNormal, "CreateENISuccess", "create new ENI %s on nrs %s success", eniID, resource.Name) scopedLog = scopedLog.WithField("eniID", eniID) - newENI.Spec.ENI.ID = eniID - newENI.Name = eniID n.creatingEni.addCreatingENI(newENI.Name, time.Now()) diff --git a/cce-network-v2/pkg/bce/vpceni/node_ebc.go b/cce-network-v2/pkg/bce/vpceni/node_ebc.go index c499382..47a7be1 100644 --- a/cce-network-v2/pkg/bce/vpceni/node_ebc.go +++ b/cce-network-v2/pkg/bce/vpceni/node_ebc.go @@ -140,6 +140,7 @@ func (n *ebcNetworkResourceSet) createPrimaryENIOnCluster(ctx context.Context, s VpcID: bccInfo.NicInfo.VpcId, SubnetID: bccInfo.NicInfo.SubnetId, SecurityGroupIds: bccInfo.NicInfo.SecurityGroups, + MacAddress: bccInfo.NicInfo.MacAddress, }, RouteTableOffset: resource.Spec.ENI.RouteTableOffset, InstallSourceBasedRouting: false, diff --git a/cce-network-v2/pkg/bce/vpceni/node_super.go b/cce-network-v2/pkg/bce/vpceni/node_super.go index cfb6960..0d0c166 100644 --- a/cce-network-v2/pkg/bce/vpceni/node_super.go +++ b/cce-network-v2/pkg/bce/vpceni/node_super.go @@ -1120,6 +1120,8 @@ func (n *bceNetworkResourceSet) updateENIWithPoll(ctx context.Context, eni *ccev return false, fmt.Errorf("get eni %s failed: %v", eni.Name, ierr) } eni = eni.DeepCopy() + oldversion = eni.Spec.VPCVersion + eni.Spec.VPCVersion = eni.Spec.VPCVersion + 1 eni = refresh(eni) // update eni @@ -1419,7 +1421,7 @@ func (n *bceNetworkResourceSet) tryBorrowIPs(newENI *ccev2.ENI) error { toBorrowIps = maxAllocateIPs + 1 } - borrowedIPs = subnet.Borrow(newENI.Spec.SubnetID, toBorrowIps) + borrowedIPs = subnet.Borrow(newENI.Spec.ENI.ID, toBorrowIps) if borrowedIPs != toBorrowIps { subnet.Cancel(newENI.Name) errMsg := fmt.Sprintf("Failed to borrow ENI ips (%d/%d) for eni %s, please change subnet of %s instance", borrowedIPs, toBorrowIps, newENI.Spec.SubnetID, n.instanceType) diff --git a/cce-network-v2/pkg/datapath/link/link.go b/cce-network-v2/pkg/datapath/link/link.go index 6271ee8..36d7238 100644 --- a/cce-network-v2/pkg/datapath/link/link.go +++ b/cce-network-v2/pkg/datapath/link/link.go @@ -25,7 +25,7 @@ func DetectDefaultRouteInterface() (netlink.Link, error) { } for _, v := range routeToDstIP { - if v.Dst == nil { + if v.Dst == nil || v.Dst.IP.IsUnspecified() { l, err := netlink.LinkByIndex(v.LinkIndex) if err != nil { return nil, err diff --git a/cce-network-v2/pkg/endpoint/operator_endpoint_manager.go b/cce-network-v2/pkg/endpoint/operator_endpoint_manager.go index 25e8efc..18315cb 100644 --- a/cce-network-v2/pkg/endpoint/operator_endpoint_manager.go +++ b/cce-network-v2/pkg/endpoint/operator_endpoint_manager.go @@ -337,7 +337,7 @@ func (manager *EndpointManager) Delete(namespace, name string) error { } operation, err = manager.directIPAllocator.NodeEndpoint(newCEP) - if err != nil { + if err != nil && !errors.IsNotFound(err) { log.Errorf("failed to get node endpoint %v", err) return err } diff --git a/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2/cce_eni_utils.go b/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2/cce_eni_utils.go index abd048e..0f2ec37 100644 --- a/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2/cce_eni_utils.go +++ b/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2/cce_eni_utils.go @@ -42,27 +42,6 @@ func (s *ENIStatus) AppendVPCStatus(vpcStatus VPCENIStatus) { } } -func (s *ENIStatus) AppendCCEStatus(cceStatus CCEENIStatus) { - if s.CCEStatus != cceStatus { - s.CCEStatus = cceStatus - - change := ENIStatusChange{ - StatusChange: StatusChange{ - Code: "ok", - Time: metav1.Now(), - }, - CCEENIStatus: cceStatus, - } - - if len(s.CCEStatusChangeLog) < 20 { - s.CCEStatusChangeLog = append(s.CCEStatusChangeLog, change) - } else { - s.CCEStatusChangeLog = s.CCEStatusChangeLog[1:19] - s.CCEStatusChangeLog = append(s.CCEStatusChangeLog, change) - } - } -} - func (s *ENIStatus) AppendCCEENIStatus(status CCEENIStatus) { if s.CCEStatus != status { s.CCEStatus = status diff --git a/cce-network-v2/pkg/os/centos.go b/cce-network-v2/pkg/os/centos.go index 7d24974..31aa2a8 100644 --- a/cce-network-v2/pkg/os/centos.go +++ b/cce-network-v2/pkg/os/centos.go @@ -51,14 +51,14 @@ type centos struct { // so we need to disable DHCPv6 for the interface. // NetworkManager will use this file to configure the interface in Redhat OS. // CentOS / Fedora / RHEL / Rocky Linux -func (c *centos) DisableDHCPv6(ifname string) error { +func (c *centos) DisableDHCPv6(udevName, cceName string) error { // maybe not redhat os if _, err := os.ReadDir(networkScriptsPath); err != nil { return nil } - path := fmt.Sprintf(sysconfigPath, ifname) - _, err := os.ReadFile(path) + path := fmt.Sprintf(sysconfigPath, udevName) + _, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { goto createNew @@ -68,7 +68,12 @@ func (c *centos) DisableDHCPv6(ifname string) error { return nil createNew: - newCfg := fmt.Sprintf(ifcfgTemplate, ifname) + cceNamePath := fmt.Sprintf(sysconfigPath, cceName) + _, err = os.Stat(cceNamePath) + if err == nil { + os.Remove(cceNamePath) + } + newCfg := fmt.Sprintf(ifcfgTemplate, udevName) err = os.WriteFile(path, []byte(newCfg), 0644) if err != nil { return fmt.Errorf("failed to write %s: %s", path, err) diff --git a/cce-network-v2/pkg/os/os_detect.go b/cce-network-v2/pkg/os/os_detect.go index c557e4d..d80967f 100644 --- a/cce-network-v2/pkg/os/os_detect.go +++ b/cce-network-v2/pkg/os/os_detect.go @@ -22,7 +22,7 @@ var localOS *OSRelease type HostOS interface { // DisableDHCPv6 - DisableDHCPv6(dev string) error + DisableDHCPv6(udevName, cceName string) error DisableMacPersistant() error } diff --git a/cce-network-v2/pkg/os/ubuntu.go b/cce-network-v2/pkg/os/ubuntu.go index bc87d79..8eeaf85 100644 --- a/cce-network-v2/pkg/os/ubuntu.go +++ b/cce-network-v2/pkg/os/ubuntu.go @@ -27,7 +27,7 @@ type ubuntuOS struct { } // DisableDHCPv6 implements HostOS. -func (*ubuntuOS) DisableDHCPv6(dev string) error { +func (*ubuntuOS) DisableDHCPv6(udevName, cceName string) error { return nil } diff --git a/cce-network-v2/plugins/cptp/ptp.go b/cce-network-v2/plugins/cptp/ptp.go index 70bfa92..9e50031 100644 --- a/cce-network-v2/plugins/cptp/ptp.go +++ b/cce-network-v2/plugins/cptp/ptp.go @@ -128,10 +128,31 @@ func setupHostVeth(host *current.Interface, container *current.Interface, result // dst happens to be the same as IP/net of link veth // the host access container can retain the source IP of the host that // use link scope instead of host scope - if err = AddLinkRoute(ipn, nil, veth); err != nil && !os.IsExist(err) { + if err = AddLinkRoute(ipn, nil, veth); err != nil { + if os.IsExist(err) { + isRouteExist := false + routeExist, errGetRoute := netlink.RouteGet(ipn.IP) + if errGetRoute == nil { + for _, route := range routeExist { + // Optimize the judgment conditions for the conflict of the destination + // address of the routing rules of the veth device, and solve the problem + // that the network of the newly created pod container is blocked due to + // the different destination veth devices of the routing rules when the + // nodes have routing rules of the same destination ip. + if route.LinkIndex == veth.Attrs().Index { + isRouteExist = true + break + } + } + } + if isRouteExist { + goto nextStep + } + } return fmt.Errorf("failed to add route on host: %v", err) } + nextStep: hw, err := net.ParseMAC(container.Mac) if err == nil { // add permanent ARP entry for the gateway on the host veth diff --git a/go.work.sum b/go.work.sum index fcc0ce9..4b3d819 100644 --- a/go.work.sum +++ b/go.work.sum @@ -51,8 +51,6 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/baidubce/bce-sdk-go v0.9.165 h1:Erzw2lQ95Np9llvKtPNSZci3dbQK6OHn5kNFxUmBpnQ= -github.com/baidubce/bce-sdk-go v0.9.165/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bketelsen/crypt v0.0.4 h1:w/jqZtC9YD4DS/Vp9GhWfWcCpuAL58oTnLoI8vE9YHU= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -221,8 +219,6 @@ github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQ github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= @@ -267,8 +263,6 @@ github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/sagikazarmark/crypt v0.6.0 h1:REOEXCs/NFY/1jOCEouMuT4zEniE5YoXbvpC5X/TLF8= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= @@ -388,10 +382,10 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c h1:3kC/TjQ+xzIblQv39bCOyRk8fbEeJcDHwbyxPUU2BpA= -golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=