From 7b633257b895fd45383a5f27563c639cabefca15 Mon Sep 17 00:00:00 2001 From: changluyi <47097611+changluyi@users.noreply.github.com> Date: Sat, 30 Nov 2024 11:09:33 +0800 Subject: [PATCH] add kubevirt live migration optimize (#4773) * add kubevirt live migration optimize Signed-off-by: clyi --- .../kube-ovn/templates/controller-deploy.yaml | 1 + charts/kube-ovn/templates/ovn-CR.yaml | 8 + charts/kube-ovn/values.yaml | 1 + dist/images/install.sh | 16 ++ go.mod | 18 ++- go.sum | 20 ++- pkg/controller/config.go | 71 +++++---- pkg/controller/controller.go | 44 +++++- pkg/controller/kubevirt.go | 143 ++++++++++++++++++ pkg/controller/pod.go | 84 +--------- pkg/util/const.go | 10 +- 11 files changed, 280 insertions(+), 136 deletions(-) create mode 100644 pkg/controller/kubevirt.go diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index 06fdba72d95..42f53b2d7a3 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -139,6 +139,7 @@ spec: - --enable-anp={{- .Values.func.ENABLE_ANP }} - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }} - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }} + - --enable-live-migration-optimize={{- .Values.func.ENABLE_LIVE_MIGRATION_OPTIMIZE }} securityContext: runAsUser: {{ include "kubeovn.runAsUser" . }} privileged: false diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index 856c9cd5b86..aeef5b31313 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -206,6 +206,14 @@ rules: verbs: - approve - sign + - apiGroups: + - kubevirt.io + resources: + - virtualmachineinstancemigrations + verbs: + - "list" + - "watch" + - "get" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/charts/kube-ovn/values.yaml b/charts/kube-ovn/values.yaml index 8bc2f0e57d4..b98bc90a850 100644 --- a/charts/kube-ovn/values.yaml +++ b/charts/kube-ovn/values.yaml @@ -77,6 +77,7 @@ func: SET_VXLAN_TX_OFF: false OVSDB_CON_TIMEOUT: 3 OVSDB_INACTIVITY_TIMEOUT: 10 + ENABLE_LIVE_MIGRATION_OPTIMIZE: true ipv4: POD_CIDR: "10.16.0.0/16" diff --git a/dist/images/install.sh b/dist/images/install.sh index 40a78ef0dbb..3b76fe9fa9b 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -44,6 +44,7 @@ ENABLE_ANP=${ENABLE_ANP:-false} SET_VXLAN_TX_OFF=${SET_VXLAN_TX_OFF:-false} OVSDB_CON_TIMEOUT=${OVSDB_CON_TIMEOUT:-3} OVSDB_INACTIVITY_TIMEOUT=${OVSDB_INACTIVITY_TIMEOUT:-10} +ENABLE_LIVE_MIGRATION_OPTIMIZE=${ENABLE_LIVE_MIGRATION_OPTIMIZE:-true} # debug DEBUG_WRAPPER=${DEBUG_WRAPPER:-} @@ -3269,6 +3270,20 @@ rules: verbs: - approve - sign + - apiGroups: + - kubevirt.io + resources: + - virtualmachineinstancemigrations + verbs: + - "list" + - "watch" + - "get" + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -4391,6 +4406,7 @@ spec: - --enable-anp=$ENABLE_ANP - --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT - --ovsdb-inactivity-timeout=$OVSDB_INACTIVITY_TIMEOUT + - --enable-live-migration-optimize=$ENABLE_LIVE_MIGRATION_OPTIMIZE securityContext: runAsUser: ${RUN_AS_USER} privileged: false diff --git a/go.mod b/go.mod index 41c79f95d47..08a2120623c 100644 --- a/go.mod +++ b/go.mod @@ -47,8 +47,9 @@ require ( google.golang.org/protobuf v1.35.2 gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.1.3 k8s.io/api v0.31.3 + k8s.io/apiextensions-apiserver v0.31.3 k8s.io/apimachinery v0.31.3 - k8s.io/client-go v1.5.2 + k8s.io/client-go v12.0.0+incompatible k8s.io/component-base v0.31.3 k8s.io/klog/v2 v2.130.1 k8s.io/kubectl v0.31.3 @@ -57,7 +58,8 @@ require ( k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 kernel.org/pub/linux/libs/security/libcap/cap v1.2.72 kubevirt.io/api v1.4.0 - kubevirt.io/client-go v1.3.1 + kubevirt.io/client-go v1.4.0 + kubevirt.io/kubevirt v1.4.0 sigs.k8s.io/controller-runtime v0.19.2 sigs.k8s.io/network-policy-api v0.1.5 ) @@ -124,6 +126,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -199,7 +202,7 @@ require ( github.com/opencontainers/runc v1.2.2 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opencontainers/selinux v1.11.1 // indirect - github.com/openshift/api v0.0.0-20231207204216-5efc6fca4b2d // indirect + github.com/openshift/api v0.0.0 // indirect github.com/openshift/client-go v3.9.0+incompatible // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect @@ -255,7 +258,7 @@ require ( go.uber.org/zap v1.27.0 // indirect gocv.io/x/gocv v0.39.0 // indirect golang.org/x/crypto v0.29.0 // indirect - golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/net v0.31.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sync v0.9.0 // indirect @@ -273,7 +276,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v1.0.1 // indirect - k8s.io/apiextensions-apiserver v0.31.3 // indirect k8s.io/apiserver v0.31.3 // indirect k8s.io/cli-runtime v0.31.3 // indirect k8s.io/cloud-provider v0.31.3 // indirect @@ -285,6 +287,7 @@ require ( k8s.io/csi-translation-lib v0.31.3 // indirect k8s.io/dynamic-resource-allocation v0.0.0 // indirect k8s.io/kms v0.31.3 // indirect + k8s.io/kube-aggregator v0.26.4 // indirect k8s.io/kube-openapi v0.31.3 // indirect k8s.io/kube-scheduler v0.0.0 // indirect k8s.io/kubelet v0.31.3 // indirect @@ -302,7 +305,8 @@ require ( replace ( github.com/mdlayher/arp => github.com/kubeovn/arp v0.0.0-20240218024213-d9612a263f68 - github.com/openshift/client-go => github.com/openshift/client-go v0.0.1 + github.com/openshift/api => github.com/openshift/api v0.0.0-20191219222812-2987a591a72c + github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 github.com/ovn-org/libovsdb => github.com/kubeovn/libovsdb v0.0.0-20240814054845-978196448fb2 k8s.io/api => k8s.io/api v0.31.3 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.31.3 @@ -333,5 +337,5 @@ replace ( k8s.io/mount-utils => k8s.io/mount-utils v0.31.3 k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.31.3 k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.31.3 - kubevirt.io/client-go => github.com/kubeovn/kubevirt-client-go v0.0.0-20240823060554-65405ba5499d + kubevirt.io/client-go => github.com/kubeovn/kubevirt-client-go v0.0.0-20241128091559-882afb5db2f6 ) diff --git a/go.sum b/go.sum index 136c6d283b7..5dc8211a049 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -380,8 +382,8 @@ github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca h1:fTMjoho2et9 github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca/go.mod h1:jY1XeGzkx8ASNJ+SqQSxTESNXARkjvt+I6IJOTnzIjw= github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589 h1:y9exo1hjCsq7jsGUzt11kxhTiEGrGSQ0ZqibAiZk2PQ= github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230905082151-e28c4d73a589/go.mod h1:49upX+/hUyppWIqu58cumojyIwXdkA8k6reA/mQlKuI= -github.com/kubeovn/kubevirt-client-go v0.0.0-20240823060554-65405ba5499d h1:xytZ7pwEajOoKnu+7P4f/ljD+XsNeVjyG8e839ht1o8= -github.com/kubeovn/kubevirt-client-go v0.0.0-20240823060554-65405ba5499d/go.mod h1:KLjiIn15GHVtlp8DZTngKY5APnPGfvQS7V7kgOTrB5o= +github.com/kubeovn/kubevirt-client-go v0.0.0-20241128091559-882afb5db2f6 h1:IRNIUu06jdpvv+XunRszHH53YaVWgv1zcLyCGj1K4qk= +github.com/kubeovn/kubevirt-client-go v0.0.0-20241128091559-882afb5db2f6/go.mod h1:bMA7q9MnyN7DExWPH8uJhpqwcv+XqiaURV+xeap9TIg= github.com/kubeovn/libovsdb v0.0.0-20240814054845-978196448fb2 h1:jH4yKIJLu2ZBy6fLMrlVa27ccgjzc53rsGDzNvddh0E= github.com/kubeovn/libovsdb v0.0.0-20240814054845-978196448fb2/go.mod h1:od3agzU0e50RPBxap7mMvBWZ+u37kqX0W849BYufdHI= github.com/kubeovn/ovsdb v0.0.0-20240410091831-5dd26006c475 h1:KZba2Kj9TXCUdUSqOR3eiy4VvkkIyhDVImYmYs6GQWU= @@ -541,10 +543,10 @@ github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/openshift/api v0.0.0-20231207204216-5efc6fca4b2d h1:DJxRHF5883tyG1Lf/5kQB4Ut2ycutC7tpGIaXMLKzYk= -github.com/openshift/api v0.0.0-20231207204216-5efc6fca4b2d/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU= -github.com/openshift/client-go v0.0.1 h1:zJ9NsS9rwBtYkYzLCUECkdmrM6jPit3W7Q0+Pxf5gd4= -github.com/openshift/client-go v0.0.1/go.mod h1:I8qTI1lgErsWc6CVukSjP1PYqpafE7fue0ZPy7A2jiw= +github.com/openshift/api v0.0.0-20191219222812-2987a591a72c h1:WRWMmqacvmZDbUat6WYqpuCy2yEfIeDsxFD/Htgp2T0= +github.com/openshift/api v0.0.0-20191219222812-2987a591a72c/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= +github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 h1:+TEY29DK0XhqB7HFC9OfV8qf3wffSyi7MWv3AP28DGQ= +github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47/go.mod h1:u7NRAjtYVAKokiI9LouzTv4mhds8P4S1TwdVAfbjKSk= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -766,8 +768,8 @@ golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -1158,6 +1160,8 @@ kubevirt.io/containerized-data-importer-api v1.58.1 h1:Zbf0pCvxb4fBvtMR6uI2OIJZ4 kubevirt.io/containerized-data-importer-api v1.58.1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= +kubevirt.io/kubevirt v1.4.0 h1:uVSa4YFzkqsv4p+UF+Zyg6976Fis2DQ/It+p26ekvLA= +kubevirt.io/kubevirt v1.4.0/go.mod h1:UmaDMGGrk1BOiLLrGnWkg0KOJaBvy7RB8iFd7FoHXQ4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1 h1:uOuSLOMBWkJH0TWa9X6l+mj5nZdm6Ay6Bli8HL8rNfk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.2 h1:3sPrF58XQEPzbE8T81TN6selQIMGbtYwuaJ6eDssDF8= diff --git a/pkg/controller/config.go b/pkg/controller/config.go index 9963e31445d..069ad88799a 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -10,6 +10,7 @@ import ( attachnetclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" "github.com/spf13/pflag" + extClientSet "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -39,8 +40,8 @@ type Configuration struct { AnpClient anpclientset.Interface AttachNetClient attachnetclientset.Interface KubevirtClient kubecli.KubevirtClient + ExtClient extClientSet.Interface - // with no timeout KubeFactoryClient kubernetes.Interface KubeOvnFactoryClient clientset.Interface @@ -85,16 +86,17 @@ type Configuration struct { LsDnatModDlDst bool LsCtSkipDstLportIPs bool - EnableLb bool - EnableNP bool - EnableEipSnat bool - EnableExternalVpc bool - EnableEcmp bool - EnableKeepVMIP bool - EnableLbSvc bool - EnableMetrics bool - EnableANP bool - EnableOVNIPSec bool + EnableLb bool + EnableNP bool + EnableEipSnat bool + EnableExternalVpc bool + EnableEcmp bool + EnableKeepVMIP bool + EnableLbSvc bool + EnableMetrics bool + EnableANP bool + EnableOVNIPSec bool + EnableLiveMigrationOptimize bool ExternalGatewaySwitch string ExternalGatewayConfigNS string @@ -154,25 +156,26 @@ func ParseFlags() (*Configuration, error) { argSecureServing = pflag.Bool("secure-serving", false, "Enable secure serving") argNodePgProbeTime = pflag.Int("nodepg-probe-time", 1, "The probe interval for node port-group, the unit is minute") - argNetworkType = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type") - argDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") - argDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") - argDefaultExchangeLinkName = pflag.Bool("default-exchange-link-name", false, "exchange link names of OVS bridge and the provider nic in the default provider-network") - argDefaultVlanName = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name") - argDefaultVlanID = pflag.Int("default-vlan-id", 1, "The default vlan id") - argLsDnatModDlDst = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch") - argLsCtSkipDstLportIPs = pflag.Bool("ls-ct-skip-dst-lport-ips", true, "Skip conntrack for direct traffic between lports") - argPodNicType = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type") - argEnableLb = pflag.Bool("enable-lb", true, "Enable load balancer") - argEnableNP = pflag.Bool("enable-np", true, "Enable network policy support") - argEnableEipSnat = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT") - argEnableExternalVpc = pflag.Bool("enable-external-vpc", true, "Enable external vpc support") - argEnableEcmp = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet") - argKeepVMIP = pflag.Bool("keep-vm-ip", true, "Whether to keep ip for kubevirt pod when pod is rebuild") - argEnableLbSvc = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service") - argEnableMetrics = pflag.Bool("enable-metrics", true, "Whether to support metrics query") - argEnableANP = pflag.Bool("enable-anp", false, "Enable support for admin network policy and baseline admin network policy") - argEnableOVNIPSec = pflag.Bool("enable-ovn-ipsec", false, "Whether to enable ovn ipsec") + argNetworkType = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type") + argDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") + argDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") + argDefaultExchangeLinkName = pflag.Bool("default-exchange-link-name", false, "exchange link names of OVS bridge and the provider nic in the default provider-network") + argDefaultVlanName = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name") + argDefaultVlanID = pflag.Int("default-vlan-id", 1, "The default vlan id") + argLsDnatModDlDst = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch") + argLsCtSkipDstLportIPs = pflag.Bool("ls-ct-skip-dst-lport-ips", true, "Skip conntrack for direct traffic between lports") + argPodNicType = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type") + argEnableLb = pflag.Bool("enable-lb", true, "Enable load balancer") + argEnableNP = pflag.Bool("enable-np", true, "Enable network policy support") + argEnableEipSnat = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT") + argEnableExternalVpc = pflag.Bool("enable-external-vpc", true, "Enable external vpc support") + argEnableEcmp = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet") + argKeepVMIP = pflag.Bool("keep-vm-ip", true, "Whether to keep ip for kubevirt pod when pod is rebuild") + argEnableLbSvc = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service") + argEnableMetrics = pflag.Bool("enable-metrics", true, "Whether to support metrics query") + argEnableANP = pflag.Bool("enable-anp", false, "Enable support for admin network policy and baseline admin network policy") + argEnableOVNIPSec = pflag.Bool("enable-ovn-ipsec", false, "Whether to enable ovn ipsec") + argEnableLiveMigrationOptimize = pflag.Bool("enable-live-migration-optimize", true, "Whether to enable kubevirt live migration optimize") argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system") argExternalGatewaySwitch = pflag.String("external-gateway-switch", "external", "The name of the external gateway switch which is a ovs bridge to provide external network, default: external") @@ -265,6 +268,7 @@ func ParseFlags() (*Configuration, error) { EnableLbSvc: *argEnableLbSvc, EnableMetrics: *argEnableMetrics, EnableOVNIPSec: *argEnableOVNIPSec, + EnableLiveMigrationOptimize: *argEnableLiveMigrationOptimize, BfdMinTx: *argBfdMinTx, BfdMinRx: *argBfdMinRx, BfdDetectMult: *argBfdDetectMult, @@ -378,6 +382,13 @@ func (config *Configuration) initKubeClient() error { } config.KubeOvnClient = kubeOvnClient + ExtClient, err := extClientSet.NewForConfig(cfg) + if err != nil { + klog.Errorf("init extentsion client failed %v", err) + return err + } + config.ExtClient = ExtClient + cfg.ContentType = "application/vnd.kubernetes.protobuf" cfg.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" kubeClient, err := kubernetes.NewForConfig(cfg) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7146382f554..57da0d750c5 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" "k8s.io/utils/keymutex" + kubevirtController "kubevirt.io/kubevirt/pkg/controller" v1alpha1 "sigs.k8s.io/network-policy-api/apis/v1alpha1" anpinformer "sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions" anplister "sigs.k8s.io/network-policy-api/pkg/client/listers/apis/v1alpha1" @@ -258,6 +259,11 @@ type Controller struct { csrSynced cache.InformerSynced addOrUpdateCsrQueue workqueue.TypedRateLimitingInterface[string] + vmiMigrationSynced cache.InformerSynced + addOrUpdateVMIMigrationQueue workqueue.TypedRateLimitingInterface[string] + kubevirtInformerFactory kubevirtController.KubeInformerFactory + hasKubevirtVMIMigration bool + recorder record.EventRecorder informerFactory kubeinformers.SharedInformerFactory cmInformerFactory kubeinformers.SharedInformerFactory @@ -301,6 +307,8 @@ func Run(ctx context.Context, config *Configuration) { listOption.AllowWatchBookmarks = true })) + kubevirtInformerFactory := kubevirtController.NewKubeInformerFactory(config.KubevirtClient.RestClient(), config.KubevirtClient, nil, util.KubevirtNamespace) + vpcInformer := kubeovnInformerFactory.Kubeovn().V1().Vpcs() vpcNatGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() @@ -331,6 +339,7 @@ func Run(ctx context.Context, config *Configuration) { anpInformer := anpInformerFactory.Policy().V1alpha1().AdminNetworkPolicies() banpInformer := anpInformerFactory.Policy().V1alpha1().BaselineAdminNetworkPolicies() csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() + vmiMigrationInformer := kubevirtInformerFactory.VirtualMachineInstanceMigration() numKeyLocks := runtime.NumCPU() * 2 if numKeyLocks < config.WorkerNum*2 { @@ -505,7 +514,11 @@ func Run(ctx context.Context, config *Configuration) { csrLister: csrInformer.Lister(), csrSynced: csrInformer.Informer().HasSynced, - addOrUpdateCsrQueue: newTypedRateLimitingQueue[string]("AddOrUpdateCSR", nil), + addOrUpdateCsrQueue: newTypedRateLimitingQueue[string]("AddOrUpdateCSR", custCrdRateLimiter), + + vmiMigrationSynced: vmiMigrationInformer.HasSynced, + addOrUpdateVMIMigrationQueue: newTypedRateLimitingQueue[string]("AddOrUpdateVMIMigration", nil), + kubevirtInformerFactory: kubevirtInformerFactory, recorder: recorder, informerFactory: informerFactory, @@ -591,6 +604,11 @@ func Run(ctx context.Context, config *Configuration) { controller.kubeovnInformerFactory.Start(ctx.Done()) controller.anpInformerFactory.Start(ctx.Done()) + controller.hasKubevirtVMIMigration = controller.isVMIMigrationCRDInstalled() + if controller.config.EnableLiveMigrationOptimize && controller.hasKubevirtVMIMigration { + kubevirtInformerFactory.Start(ctx.Done()) + } + klog.Info("Waiting for informer caches to sync") cacheSyncs := []cache.InformerSynced{ controller.vpcNatGatewaySynced, controller.vpcSynced, controller.subnetSynced, @@ -610,6 +628,11 @@ func Run(ctx context.Context, config *Configuration) { if controller.config.EnableANP { cacheSyncs = append(cacheSyncs, controller.anpsSynced, controller.banpsSynced) } + + if controller.config.EnableLiveMigrationOptimize && controller.hasKubevirtVMIMigration { + cacheSyncs = append(cacheSyncs, controller.vmiMigrationSynced) + } + if !cache.WaitForCacheSync(ctx.Done(), cacheSyncs...) { util.LogFatalAndExit(nil, "failed to wait for caches to sync") } @@ -847,6 +870,16 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add csr event handler") } } + + if config.EnableLiveMigrationOptimize && controller.hasKubevirtVMIMigration { + if _, err = vmiMigrationInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddVMIMigration, + UpdateFunc: controller.enqueueUpdateVMIMigration, + }); err != nil { + util.LogFatalAndExit(err, "failed to add VMI Migration event handler") + } + } + controller.Run(ctx) } @@ -1039,6 +1072,10 @@ func (c *Controller) shutdown() { c.syncSgPortsQueue.ShutDown() c.addOrUpdateCsrQueue.ShutDown() + + if c.config.EnableLiveMigrationOptimize { + c.addOrUpdateVMIMigrationQueue.ShutDown() + } } func (c *Controller) startWorkers(ctx context.Context) { @@ -1055,7 +1092,6 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("update snat for vpc nat gateway", c.updateVpcSnatQueue, c.handleUpdateVpcSnat), time.Second, ctx.Done()) go wait.Until(runWorker("update subnet route for vpc nat gateway", c.updateVpcSubnetQueue, c.handleUpdateNatGwSubnetRoute), time.Second, ctx.Done()) go wait.Until(runWorker("add/update csr", c.addOrUpdateCsrQueue, c.handleAddOrUpdateCsr), time.Second, ctx.Done()) - // add default and join subnet and wait them ready go wait.Until(runWorker("add/update subnet", c.addOrUpdateSubnetQueue, c.handleAddOrUpdateSubnet), time.Second, ctx.Done()) go wait.Until(runWorker("add/update ippool", c.addOrUpdateIPPoolQueue, c.handleAddOrUpdateIPPool), time.Second, ctx.Done()) @@ -1244,6 +1280,10 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("update base admin network policy", c.updateBanpQueue, c.handleUpdateBanp), time.Second, ctx.Done()) go wait.Until(runWorker("delete base admin network policy", c.deleteBanpQueue, c.handleDeleteBanp), time.Second, ctx.Done()) } + + if c.config.EnableLiveMigrationOptimize && c.hasKubevirtVMIMigration { + go wait.Until(runWorker("add/update vmiMigration ", c.addOrUpdateVMIMigrationQueue, c.handleAddOrUpdateVMIMigration), 50*time.Millisecond, ctx.Done()) + } } func (c *Controller) allSubnetReady(subnets ...string) (bool, error) { diff --git a/pkg/controller/kubevirt.go b/pkg/controller/kubevirt.go new file mode 100644 index 00000000000..16f3a1975da --- /dev/null +++ b/pkg/controller/kubevirt.go @@ -0,0 +1,143 @@ +package controller + +import ( + "context" + "fmt" + "reflect" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + kubevirtv1 "kubevirt.io/api/core/v1" + + "github.com/kubeovn/kube-ovn/pkg/ovs" +) + +func (c *Controller) enqueueAddVMIMigration(obj interface{}) { + var ( + key string + err error + ) + + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + utilruntime.HandleError(err) + return + } + + klog.Infof("enqueue add VMI migration %s ", key) + c.addOrUpdateVMIMigrationQueue.Add(key) +} + +func (c *Controller) enqueueUpdateVMIMigration(oldObj, newObj interface{}) { + oldVmi := oldObj.(*kubevirtv1.VirtualMachineInstanceMigration) + newVmi := newObj.(*kubevirtv1.VirtualMachineInstanceMigration) + + if !newVmi.DeletionTimestamp.IsZero() || + !reflect.DeepEqual(oldVmi.Status.Phase, newVmi.Status.Phase) { + key, err := cache.MetaNamespaceKeyFunc(newObj) + if err != nil { + utilruntime.HandleError(err) + return + } + klog.Infof("enqueue update VMI migration %s", key) + c.addOrUpdateVMIMigrationQueue.Add(key) + } +} + +func (c *Controller) handleAddOrUpdateVMIMigration(key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + vmiMigration, err := c.config.KubevirtClient.VirtualMachineInstanceMigration(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + utilruntime.HandleError(fmt.Errorf("failed to get VMI migration by key %s: %w", key, err)) + return err + } + if vmiMigration.Status.MigrationState == nil { + klog.Infof("VirtualMachineInstanceMigration %s migration state is nil, skipping", key) + return nil + } + + vmi, err := c.config.KubevirtClient.VirtualMachineInstance(namespace).Get(context.TODO(), vmiMigration.Spec.VMIName, metav1.GetOptions{}) + if err != nil { + utilruntime.HandleError(fmt.Errorf("failed to get VMI by name %s: %w", vmiMigration.Spec.VMIName, err)) + return err + } + + if vmi.Status.MigrationState == nil { + klog.Infof("VMI instance %s migration state is nil, skipping", key) + return nil + } + + if vmi.Status.MigrationState.SourcePod == "" { + klog.Infof("VMI instance %s source pod is nil, skipping", key) + return nil + } + + // use VirtualMachineInsance's MigrationState because VirtualMachineInsanceMigration's MigrationState is not updated util migration finished + klog.Infof("current vmiMigration %s status %s, target Node %s, source Node %s, target Pod %s, source Pod %s", key, + vmiMigration.Status.Phase, + vmi.Status.MigrationState.TargetNode, + vmi.Status.MigrationState.SourceNode, + vmi.Status.MigrationState.TargetPod, + vmi.Status.MigrationState.SourcePod) + + sourcePodName := vmi.Status.MigrationState.SourcePod + sourcePod, err := c.config.KubeClient.CoreV1().Pods(namespace).Get(context.TODO(), sourcePodName, metav1.GetOptions{}) + if err != nil { + err = fmt.Errorf("failed to get source pod %s, %w", sourcePodName, err) + klog.Error(err) + return err + } + + podNets, err := c.getPodKubeovnNets(sourcePod) + if err != nil { + err = fmt.Errorf("failed to get pod nets %w", err) + klog.Error(err) + return err + } + + needAllocatePodNets := needAllocateSubnets(sourcePod, podNets) + for _, podNet := range needAllocatePodNets { + portName := ovs.PodNameToPortName(vmiMigration.Spec.VMIName, vmiMigration.Namespace, podNet.ProviderName) + srcNodeName := vmi.Status.MigrationState.SourceNode + targetNodeName := vmi.Status.MigrationState.TargetNode + switch vmiMigration.Status.Phase { + case kubevirtv1.MigrationRunning: + klog.Infof("migrate start set options for lsp %s from %s to %s", portName, srcNodeName, targetNodeName) + if err := c.OVNNbClient.SetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName); err != nil { + err = fmt.Errorf("failed to set migrate options for lsp %s, %w", portName, err) + klog.Error(err) + return err + } + case kubevirtv1.MigrationSucceeded: + klog.Infof("migrate end reset options for lsp %s from %s to %s, migrated succeed", portName, srcNodeName, targetNodeName) + if err := c.OVNNbClient.ResetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName, false); err != nil { + err = fmt.Errorf("failed to clean migrate options for lsp %s, %w", portName, err) + klog.Error(err) + return err + } + case kubevirtv1.MigrationFailed: + klog.Infof("migrate end reset options for lsp %s from %s to %s, migrated fail", portName, srcNodeName, targetNodeName) + if err := c.OVNNbClient.ResetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName, true); err != nil { + err = fmt.Errorf("failed to clean migrate options for lsp %s, %w", portName, err) + klog.Error(err) + return err + } + } + } + return nil +} + +func (c *Controller) isVMIMigrationCRDInstalled() bool { + _, err := c.config.ExtClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), "virtualmachineinstancemigrations.kubevirt.io", metav1.GetOptions{}) + if err != nil { + return false + } + klog.Info("Detect VMI Migration CRD") + return true +} diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 8a5285e0bbd..2234b2f7be7 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -513,14 +513,11 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca // todo: isVmPod, getPodType, getNameByPod has duplicated logic var err error - var isMigrate, migrated, migratedFail bool - var vmKey, srcNodeName, targetNodeName string + var vmKey string + // var isMigrate, migrated, migratedFail bool + // var vmKey, srcNodeName, targetNodeName string if isVMPod && c.config.EnableKeepVMIP { vmKey = fmt.Sprintf("%s/%s", namespace, vmName) - if isMigrate, migrated, migratedFail, srcNodeName, targetNodeName, err = c.migrateVM(pod, vmKey); err != nil { - klog.Error(err) - return nil, err - } } // Avoid create lsp for already running pod in ovn-nb when controller restart for _, podNet := range needAllocatePodNets { @@ -604,24 +601,6 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca return nil, err } - if isMigrate { - if migrated { - klog.Infof("migrate end reset options for lsp %s from %s to %s, migrated fail: %t", portName, srcNodeName, targetNodeName, migratedFail) - if err := c.OVNNbClient.ResetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName, migratedFail); err != nil { - err = fmt.Errorf("failed to clean migrate options for lsp %s, %w", portName, err) - klog.Error(err) - return nil, err - } - } else { - klog.Infof("migrate start set options for lsp %s from %s to %s", portName, srcNodeName, targetNodeName) - if err := c.OVNNbClient.SetLogicalSwitchPortMigrateOptions(portName, srcNodeName, targetNodeName); err != nil { - err = fmt.Errorf("failed to set migrate options for lsp %s, %w", portName, err) - klog.Error(err) - return nil, err - } - } - } - if pod.Annotations[fmt.Sprintf(util.Layer2ForwardAnnotationTemplate, podNet.ProviderName)] == "true" { if err := c.OVNNbClient.EnablePortLayer2forward(portName); err != nil { c.recorder.Eventf(pod, v1.EventTypeWarning, "SetOVNPortL2ForwardFailed", err.Error()) @@ -2097,63 +2076,6 @@ func (c *Controller) getVirtualIPs(pod *v1.Pod, podNets []*kubeovnNet) map[strin return vipsMap } -// migrate vm return migrate, migrated, fail, src node, target node, err -func (c *Controller) migrateVM(pod *v1.Pod, vmKey string) (bool, bool, bool, string, string, error) { - // try optimize vm migration, no need return error - // migrate true means need ovn set migrate options - // migrated ok means need set migrate options to target node - // migrated failed means need set migrate options to source node - if _, ok := pod.Annotations[util.MigrationJobAnnotation]; !ok { - return false, false, false, "", "", nil - } - if _, ok := pod.Annotations[util.MigrationSourceAnnotation]; ok { - klog.Infof("will migrate out vm %s pod %s from source node %s", vmKey, pod.Name, pod.Spec.NodeName) - return false, false, false, "", "", nil - } - // ovn set migrator only in the process of target vm pod - if _, ok := pod.Annotations[util.MigrationTargetAnnotation]; !ok { - return false, false, false, "", "", nil - } - srcNode, ok := pod.Annotations[util.MigrationSourceNodeAnnotation] - if !ok || srcNode == "" { - err := fmt.Errorf("vm %s migration source node is not set", vmKey) - klog.Warning(err) - return false, false, false, "", "", nil - } - targetNode := pod.Spec.NodeName - if targetNode == "" { - err := fmt.Errorf("vm %s migration target node is not set", vmKey) - klog.Warning(err) - return false, false, false, "", "", nil - } - migratePhase, ok := pod.Annotations[util.MigrationPhaseAnnotation] - if !ok { - err := fmt.Errorf("vm %s migration phase is not set", vmKey) - klog.Warning(err) - return false, false, false, "", "", nil - } - // check migrate phase - if migratePhase == "" { - err := fmt.Errorf("vm %s migration phase is empty", vmKey) - klog.Warning(err) - return false, false, false, "", "", nil - } - if migratePhase == util.MigrationPhaseStarted { - klog.Infof("start to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) - return true, false, false, srcNode, targetNode, nil - } - if migratePhase == util.MigrationPhaseSucceeded { - klog.Infof("succeed to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) - return true, true, false, srcNode, targetNode, nil - } - if migratePhase == util.MigrationPhaseFailed { - klog.Infof("failed to migrate src vm %s from %s to %s", vmKey, srcNode, targetNode) - return true, true, true, srcNode, targetNode, nil - } - - return false, false, false, "", "", nil -} - func setPodRoutesAnnotation(annotations map[string]string, provider string, routes []request.Route) error { key := fmt.Sprintf(util.RoutesAnnotationTemplate, provider) if len(routes) == 0 { diff --git a/pkg/util/const.go b/pkg/util/const.go index 8acda2daaf0..187fbb7a0b7 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -311,14 +311,8 @@ const ( ConsumptionKubevirt = "kubevirt" VhostUserSocketVolumeName = "vhostuser-sockets" - MigrationJobAnnotation = "kubevirt.io/migrationJobName" // migration job name - MigrationSourceNodeAnnotation = "kubevirt.io/migration-source-node" // target pod has source node name - MigrationSourceAnnotation = "kubevirt.io/migration-source" // migration source vm: true or false - MigrationTargetAnnotation = "kubevirt.io/migration-target" // migration target vm: true or false - MigrationPhaseAnnotation = "kubevirt.io/migration-phase" // migration vm phase: started/succeeded/failed - MigrationPhaseStarted = "started" - MigrationPhaseSucceeded = "succeeded" - MigrationPhaseFailed = "failed" + MigrationJobAnnotation = "kubevirt.io/migrationJobName" // migration job name + KubevirtNamespace = "kubevirt" DefaultOVNIPSecCA = "ovn-ipsec-ca" DefaultOVSCACertPath = "/var/lib/openvswitch/pki/switchca/cacert.pem"