From 67ac450c693129c1e236bededff787eea3a17c25 Mon Sep 17 00:00:00 2001 From: hedi bouattour Date: Wed, 30 Aug 2023 16:38:17 +0000 Subject: [PATCH 1/2] fix multus version used in docs --- Makefile | 4 ++-- docs/kind.md | 2 +- docs/multinet.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3647ea6b..12f165c1 100644 --- a/Makefile +++ b/Makefile @@ -222,7 +222,7 @@ install-multinet: @kubectl apply -f test/yaml/multinet/projectcalico.org_networks.yaml @kubectl apply -f test/yaml/multinet/whereabouts-daemonset-install.yaml @echo "--Installing multus daemonset..." - @kubectl apply -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/master/deployments/multus-daemonset-thick.yml + @kubectl apply -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/99c4481e08a4a8f0a3d0013446f03e4206033cae/deployments/multus-daemonset-thick.yml @echo "--Installing whereabouts daemonset..." @kubectl apply -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml @kubectl apply -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml @@ -233,7 +233,7 @@ delete-multinet: @kubectl delete -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml @kubectl delete -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml @echo "--Deleting multus daemonset..." - @kubectl delete -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/master/deployments/multus-daemonset-thick.yml + @kubectl delete -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/99c4481e08a4a8f0a3d0013446f03e4206033cae/deployments/multus-daemonset-thick.yml @echo "--Deleting network CRD..." @kubectl delete -f test/yaml/multinet/projectcalico.org_networks.yaml @kubectl delete -f test/yaml/multinet/whereabouts-daemonset-install.yaml diff --git a/docs/kind.md b/docs/kind.md index 9598deb8..daa677b0 100644 --- a/docs/kind.md +++ b/docs/kind.md @@ -50,7 +50,7 @@ nodes: # multinet CRDs, multus plugin and ipam kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/test/yaml/multinet/projectcalico.org_networks.yaml kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/test/yaml/multinet/whereabouts-daemonset-install.yaml -kubectl create -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/master/deployments/multus-daemonset-thick.yml +kubectl create -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/99c4481e08a4a8f0a3d0013446f03e4206033cae/deployments/multus-daemonset-thick.yml kubectl create -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml kubectl create -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml diff --git a/docs/multinet.md b/docs/multinet.md index 41ec4484..cbaa5f58 100644 --- a/docs/multinet.md +++ b/docs/multinet.md @@ -186,7 +186,7 @@ kubectl apply -f test/yaml/multinet/whereabouts-daemonset-install.yaml #### Installing multus deamonset ````yaml -kubectl apply -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/master/deployments/multus-daemonset-thick.yml +kubectl apply -f https://github.com/k8snetworkplumbingwg/multus-cni/raw/99c4481e08a4a8f0a3d0013446f03e4206033cae/deployments/multus-daemonset-thick.yml kubectl apply -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml kubectl apply -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml ```` From 7a7ad80bc8a283501f53dff163d390da5dc3bd4c Mon Sep 17 00:00:00 2001 From: hedi bouattour Date: Thu, 31 Aug 2023 15:07:26 +0000 Subject: [PATCH 2/2] refactor vpp-manager and add dynamic interfaces logic --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 6 +- calico-vpp-agent/common/pubsub.go | 5 +- .../watchers/uplink_link_watcher.go | 21 +- .../watchers/uplink_manager_watcher.go | 64 +++ .../watchers/uplink_route_watcher.go | 17 +- config/config.go | 23 +- vpp-manager/main.go | 27 +- vpp-manager/startup/interface_config.go | 48 +- vpp-manager/startup/startup.go | 54 ++- vpp-manager/uplink/af_packet.go | 29 +- vpp-manager/uplink/af_xdp.go | 57 ++- vpp-manager/uplink/avf.go | 45 +- vpp-manager/uplink/common.go | 72 +-- vpp-manager/uplink/default.go | 39 +- vpp-manager/uplink/dpdk.go | 61 ++- vpp-manager/uplink/rdma.go | 23 +- vpp-manager/uplink/virtio.go | 37 +- vpp-manager/uplink/vmxnet3.go | 35 +- vpp-manager/vpp_runner.go | 454 +++++++++++------- 19 files changed, 662 insertions(+), 455 deletions(-) create mode 100644 calico-vpp-agent/watchers/uplink_manager_watcher.go diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index b9d936d2..7848c6d5 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -90,6 +90,8 @@ func main() { if err != nil { log.Fatalf("Cannot create VPP client: %v", err) } + usr2SignalChannel := make(chan os.Signal, 10) + signal.Notify(usr2SignalChannel, syscall.SIGUSR2) // Once we have the api connection, we know vpp & vpp-manager are running and the // state is accurately reported. Wait for vpp-manager to finish the config. common.VppManagerInfo, err = common.WaitForVppManager() @@ -98,6 +100,8 @@ func main() { } common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) + uplinkManagerWatcher := watchers.NewUplinkManagerWatcher(usr2SignalChannel, log.WithFields(logrus.Fields{"subcomponent": "uplink-manager-watcher"})) + Go(uplinkManagerWatcher.WatchUplinks) /** * Create the API clients we need */ @@ -136,7 +140,7 @@ func main() { * Start watching nodes & fetch our BGP spec */ routeWatcher := watchers.NewRouteWatcher(log.WithFields(logrus.Fields{"subcomponent": "host-route-watcher"})) - linkWatcher := watchers.NewLinkWatcher(common.VppManagerInfo.UplinkStatuses, log.WithFields(logrus.Fields{"subcomponent": "host-link-watcher"})) + linkWatcher := watchers.NewLinkWatcher(log.WithFields(logrus.Fields{"subcomponent": "host-link-watcher"})) bgpConfigurationWatcher := watchers.NewBGPConfigurationWatcher(clientv3, log.WithFields(logrus.Fields{"subcomponent": "bgp-conf-watch"})) prefixWatcher := watchers.NewPrefixWatcher(client, log.WithFields(logrus.Fields{"subcomponent": "prefix-watcher"})) peerWatcher := watchers.NewPeerWatcher(clientv3, k8sclient, log.WithFields(logrus.Fields{"subcomponent": "peer-watcher"})) diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index daef558c..5109a1c7 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -64,10 +64,9 @@ const ( NetDeleted CalicoVppEventType = "NetDeleted" NetsSynced CalicoVppEventType = "NetsSynced" - IpamPoolUpdate CalicoVppEventType = "IpamPoolUpdate" - IpamPoolRemove CalicoVppEventType = "IpamPoolRemove" - WireguardPublicKeyChanged CalicoVppEventType = "WireguardPublicKeyChanged" + + UplinksUpdated CalicoVppEventType = "UplinksUpdated" ) var ( diff --git a/calico-vpp-agent/watchers/uplink_link_watcher.go b/calico-vpp-agent/watchers/uplink_link_watcher.go index bd62ad58..0d5d40d8 100644 --- a/calico-vpp-agent/watchers/uplink_link_watcher.go +++ b/calico-vpp-agent/watchers/uplink_link_watcher.go @@ -23,21 +23,20 @@ import ( "github.com/vishvananda/netlink" "gopkg.in/tomb.v2" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" ) type LinkWatcher struct { - UplinkStatuses map[string]config.UplinkStatus - close chan struct{} - netlinkFailed chan struct{} - closeLock sync.Mutex - log *log.Entry + close chan struct{} + netlinkFailed chan struct{} + closeLock sync.Mutex + log *log.Entry } -func NewLinkWatcher(uplinkStatus map[string]config.UplinkStatus, log *log.Entry) *LinkWatcher { +func NewLinkWatcher(log *log.Entry) *LinkWatcher { return &LinkWatcher{ - UplinkStatuses: uplinkStatus, - log: log, + log: log, } } @@ -78,7 +77,7 @@ func (r *LinkWatcher) WatchLinks(t *tomb.Tomb) error { r.safeClose() goto restart } - for _, v := range r.UplinkStatuses { + for _, v := range common.VppManagerInfo.UplinkStatuses { link, err = netlink.LinkByIndex(v.LinkIndex) if err != nil || link.Attrs().Name != v.Name { r.log.Errorf("error getting link to watch: %v %v", link, err) @@ -113,7 +112,7 @@ func (r *LinkWatcher) WatchLinks(t *tomb.Tomb) error { } found := false v := config.UplinkStatus{} - for _, v := range r.UplinkStatuses { + for _, v = range common.VppManagerInfo.UplinkStatuses { if update.Attrs().Index == v.LinkIndex { found = true break @@ -122,7 +121,7 @@ func (r *LinkWatcher) WatchLinks(t *tomb.Tomb) error { if found { if update.Attrs().Name == v.Name { if update.Attrs().MTU != v.Mtu { - if err = netlink.LinkSetMTU(link, v.Mtu); err != nil { + if err = netlink.LinkSetMTU(update.Link, v.Mtu); err != nil { r.log.Warnf("Error resetting link mtu: %v", err) r.safeClose() goto restart diff --git a/calico-vpp-agent/watchers/uplink_manager_watcher.go b/calico-vpp-agent/watchers/uplink_manager_watcher.go new file mode 100644 index 00000000..b8e4088b --- /dev/null +++ b/calico-vpp-agent/watchers/uplink_manager_watcher.go @@ -0,0 +1,64 @@ +// Copyright (C) 2023 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watchers + +import ( + "encoding/json" + "os" + + "github.com/pkg/errors" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" +) + +type UplinkManagerWatcher struct { + log *log.Entry + usr2SignalChannel chan os.Signal +} + +func NewUplinkManagerWatcher(usr2SignalChannel chan os.Signal, log *log.Entry) *UplinkManagerWatcher { + return &UplinkManagerWatcher{ + usr2SignalChannel: usr2SignalChannel, + log: log, + } +} + +func (w *UplinkManagerWatcher) WatchUplinks(t *tomb.Tomb) error { + for t.Alive() { + <-w.usr2SignalChannel + /* vpp-manager pokes us with USR2 if status changes (dynamically added interface) */ + w.log.Info("Vpp manager state changed") + dat, err := os.ReadFile(config.VppManagerInfoFile) + if err != nil { + w.log.Error(err) + } else { + err2 := json.Unmarshal(dat, common.VppManagerInfo) + if err2 != nil { + w.log.Error(errors.Errorf("cannot unmarshal vpp manager info file %s", err2)) + } else if common.VppManagerInfo.Status == config.Ready { + w.log.Info("local vppmanager state updated") + common.SendEvent(common.CalicoVppEvent{ + Type: common.UplinksUpdated, + }) + } else { + w.log.Error(errors.Errorf("vpp manager file status not ready after dynamically added interface")) + } + } + } + return nil +} diff --git a/calico-vpp-agent/watchers/uplink_route_watcher.go b/calico-vpp-agent/watchers/uplink_route_watcher.go index 27e4656d..4ed0795a 100644 --- a/calico-vpp-agent/watchers/uplink_route_watcher.go +++ b/calico-vpp-agent/watchers/uplink_route_watcher.go @@ -55,6 +55,7 @@ func NewRouteWatcher(log *log.Entry) *RouteWatcher { common.IpamConfChanged, common.NetAddedOrUpdated, common.NetDeleted, + common.UplinksUpdated, ) return routeWatcher } @@ -174,11 +175,7 @@ func (r *RouteWatcher) getNetworkRoute(network string, physicalNet string) (rout return routes, nil } -func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { - r.netlinkFailed = make(chan struct{}, 1) - r.addrUpdate = make(chan struct{}, 10) - - go r.watchAddresses(t) +func (r *RouteWatcher) AddRoutesForServices() { for _, serviceCIDR := range *config.ServiceCIDRs { // Add a route for the service prefix through VPP. This is required even if kube-proxy is // running on the host to ensure correct source address selection if the host has multiple interfaces @@ -208,6 +205,14 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } } } +} + +func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { + r.netlinkFailed = make(chan struct{}, 1) + r.addrUpdate = make(chan struct{}, 10) + + go r.watchAddresses(t) + r.AddRoutesForServices() for { r.closeLock.Lock() netlinkUpdates := make(chan netlink.RouteUpdate, 10) @@ -239,6 +244,8 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { return nil case event := <-r.eventChan: switch event.Type { + case common.UplinksUpdated: + r.AddRoutesForServices() case common.NetDeleted: netDef := event.Old.(*NetworkDefinition) key := netDef.Range diff --git a/config/config.go b/config/config.go index a73b61c2..5fd27ddd 100644 --- a/config/config.go +++ b/config/config.go @@ -136,7 +136,7 @@ func RunHook(hookScript *string, hookName string, params *VppManagerParams, log if *hookScript == "" { return } - template, err := TemplateScriptReplace(*hookScript, params, nil) + template, err := TemplateScriptReplace(*hookScript, params, false) if err != nil { log.Warnf("Running hook %s errored with %s", hookName, err) return @@ -233,6 +233,11 @@ type UplinkInterfaceSpec struct { SwIfIndex uint32 `json:"-"` } +type AttachedUplinkInterfaceSpec struct { + *UplinkInterfaceSpec + LinuxConf *LinuxInterfaceState +} + func (u *UplinkInterfaceSpec) GetIsMain() bool { if u.IsMain == nil { return false @@ -539,7 +544,7 @@ const ( ) type VppManagerParams struct { - UplinksSpecs []UplinkInterfaceSpec + AttachedUplinksSpecs []*AttachedUplinkInterfaceSpec /* Capabilities */ LoadedDrivers map[string]bool KernelVersion *KernelVersion @@ -663,13 +668,13 @@ func getCpusetCpu() (string, error) { return regexp.MustCompile("[,-]").Split(cpusetCpu, 2)[0], nil } -func TemplateScriptReplace(input string, params *VppManagerParams, conf []*LinuxInterfaceState) (template string, err error) { +func TemplateScriptReplace(input string, params *VppManagerParams, withConf bool) (template string, err error) { template = input - if conf != nil { + if len(params.AttachedUplinksSpecs) != 0 && withConf { /* We might template scripts before reading interface conf */ - template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", conf[0].PciId) - for i, ifcConf := range conf { - template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(i)+"__", ifcConf.PciId) + template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", params.AttachedUplinksSpecs[0].LinuxConf.PciId) + for i, attachedInterface := range params.AttachedUplinksSpecs { + template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(i)+"__", attachedInterface.LinuxConf.PciId) } } vppcpu, err := getCpusetCpu() @@ -677,8 +682,8 @@ func TemplateScriptReplace(input string, params *VppManagerParams, conf []*Linux return "", err } template = strings.ReplaceAll(template, "__CPUSET_CPUS_FIRST__", vppcpu) - template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.UplinksSpecs[0].InterfaceName) - for i, ifc := range params.UplinksSpecs { + template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.AttachedUplinksSpecs[0].InterfaceName) + for i, ifc := range params.AttachedUplinksSpecs { template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF_"+fmt.Sprintf("%d", i)+"__", ifc.InterfaceName) } for key, value := range params.NodeAnnotations { diff --git a/vpp-manager/main.go b/vpp-manager/main.go index 87352f82..0db4efce 100644 --- a/vpp-manager/main.go +++ b/vpp-manager/main.go @@ -35,11 +35,18 @@ const ( maxCoreFiles = 2 ) +type dynamicInterface struct { + params *config.VppManagerParams + driver uplink.UplinkDriver +} + var ( runningCond *sync.Cond vppProcess *os.Process vppDeadChan chan bool signals chan os.Signal + + dynamicInterfaceAdd chan dynamicInterface /* Was VPP terminated by us ? */ internalKill bool /* Increasing index for timeout */ @@ -148,6 +155,7 @@ func main() { log = logrus.New() vppDeadChan = make(chan bool, 1) + dynamicInterfaceAdd = make(chan dynamicInterface, 1) VPPgotSigCHLD = make(map[int]bool) VPPgotTimeout = make(map[int]bool) @@ -175,7 +183,7 @@ func main() { log.Errorf("Error raising memlock limit, VPP may fail to start: %v", err) } - confs, err := startup.GetInterfaceConfig(params) + err = startup.GetInterfaceConfig(params) if err != nil { log.Fatalf("Error getting initial interface configuration: %s", err) } @@ -183,21 +191,21 @@ func main() { runningCond = sync.NewCond(&sync.Mutex{}) go handleSignals() - startup.PrintVppManagerConfig(params, confs) + startup.PrintVppManagerConfig(params) - runner := NewVPPRunner(params, confs) + runner := NewVPPRunner() makeNewVPPIndex() - if len(params.UplinksSpecs) == 1 && params.UplinksSpecs[0].VppDriver == "" { - for _, driver := range uplink.SupportedUplinkDrivers(params, confs[0], ¶ms.UplinksSpecs[0]) { + if len(params.AttachedUplinksSpecs) == 1 && params.AttachedUplinksSpecs[0].VppDriver == "" { + for _, driver := range uplink.SupportedUplinkDrivers(params, 0) { err := utils.CleanupCoreFiles(config.GetCalicoVppInitialConfig().CorePattern, maxCoreFiles) if err != nil { log.Errorf("CleanupCoreFiles errored %s", err) } internalKill = false - err = runner.Run([]uplink.UplinkDriver{driver}) + err = runner.Run([]uplink.UplinkDriver{driver}, params) if err != nil { config.RunHook(config.HookScriptVppErrored, "VPP_ERRORED", params, log) log.Errorf("VPP(%s) run failed with %s", driver.GetName(), err) @@ -216,12 +224,11 @@ func main() { } var drivers []uplink.UplinkDriver - for idx := 0; idx < len(params.UplinksSpecs); idx++ { - drivers = append(drivers, uplink.NewUplinkDriver(params.UplinksSpecs[idx].VppDriver, - params, confs[idx], ¶ms.UplinksSpecs[idx])) + for idx := 0; idx < len(params.AttachedUplinksSpecs); idx++ { + drivers = append(drivers, uplink.NewUplinkDriver(params, idx)) } - err = runner.Run(drivers) + err = runner.Run(drivers, params) if err != nil { config.RunHook(config.HookScriptVppErrored, "VPP_ERRORED", params, log) log.Errorf("VPP run failed with %v", err) diff --git a/vpp-manager/startup/interface_config.go b/vpp-manager/startup/interface_config.go index c0c51373..52561f30 100644 --- a/vpp-manager/startup/interface_config.go +++ b/vpp-manager/startup/interface_config.go @@ -30,13 +30,23 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) -func GetInterfaceConfig(params *config.VppManagerParams) (conf []*config.LinuxInterfaceState, err error) { +func GetInterfaceConfigFromLinux(ifSpec *config.AttachedUplinkInterfaceSpec) (err error) { + configuration, err := loadInterfaceConfigFromLinux(*ifSpec.UplinkInterfaceSpec) + if err != nil { + return err + } + ifSpec.LinuxConf = configuration + return nil +} + +func GetInterfaceConfig(params *config.VppManagerParams) (err error) { errs := []error{} - conf = []*config.LinuxInterfaceState{} - for _, ifSpec := range params.UplinksSpecs { - configuration, err := loadInterfaceConfigFromLinux(ifSpec) + configToSave := []*config.LinuxInterfaceState{} + for idx, ifSpec := range params.AttachedUplinksSpecs { + configuration, err := loadInterfaceConfigFromLinux(*ifSpec.UplinkInterfaceSpec) errs = append(errs, err) - conf = append(conf, configuration) + params.AttachedUplinksSpecs[idx].LinuxConf = configuration + configToSave = append(configToSave, configuration) } allLoaded := true for i := range errs { @@ -46,38 +56,38 @@ func GetInterfaceConfig(params *config.VppManagerParams) (conf []*config.LinuxIn } } if allLoaded { - err = saveConfig(params, conf) + err = saveConfig(params, configToSave) if err != nil { log.Warnf("Could not save interface config: %v", err) } } else { // Loading config failed, try loading from save file - confFile, err2 := loadInterfaceConfigFromFile(params) + confFile, err2 := loadInterfaceConfigFromFile() if err2 != nil { - return nil, err2 + return err2 } // If loaded from file replace non loaded interface configs - for i := range conf { - if conf[i] == nil { + for i := range params.AttachedUplinksSpecs { + if params.AttachedUplinksSpecs[i].LinuxConf == nil { for j := range confFile { - if confFile[j].InterfaceName == params.UplinksSpecs[i].InterfaceName { - conf[i] = confFile[j] + if confFile[j].InterfaceName == params.AttachedUplinksSpecs[i].InterfaceName { + params.AttachedUplinksSpecs[i].LinuxConf = confFile[j] } } } } - for i := range conf { - if conf[i] == nil { - return nil, fmt.Errorf("interface configs not found") + for i := range params.AttachedUplinksSpecs { + if params.AttachedUplinksSpecs[i].LinuxConf == nil { + return fmt.Errorf("interface configs not found") } } log.Infof("Loaded config. Interfaces marked as down since loading config from linux failed.") // This ensures we don't try to set the interface down in runVpp() - for _, config := range conf { - config.IsUp = false + for _, attachedUplink := range params.AttachedUplinksSpecs { + attachedUplink.LinuxConf.IsUp = false } } - return conf, nil + return nil } func loadInterfaceConfigFromLinux(ifSpec config.UplinkInterfaceSpec) (*config.LinuxInterfaceState, error) { @@ -179,7 +189,7 @@ func saveConfig(params *config.VppManagerParams, conf []*config.LinuxInterfaceSt return nil } -func loadInterfaceConfigFromFile(params *config.VppManagerParams) ([]*config.LinuxInterfaceState, error) { +func loadInterfaceConfigFromFile() ([]*config.LinuxInterfaceState, error) { conf := []*config.LinuxInterfaceState{} if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" { return nil, fmt.Errorf("interface config save file not configured") diff --git a/vpp-manager/startup/startup.go b/vpp-manager/startup/startup.go index 8a896729..d0690d36 100644 --- a/vpp-manager/startup/startup.go +++ b/vpp-manager/startup/startup.go @@ -32,22 +32,26 @@ func NewVppManagerParams() *config.VppManagerParams { /* uplink configuration: This is being deprecated */ if mainInterface := *config.InterfaceVar; mainInterface != "" { log.Warn("Use of CALICOVPP_INTERFACE, CALICOVPP_NATIVE_DRIVER and CALICOVPP_SWAP_DRIVER is deprecated, please use CALICOVPP_INTERFACES instead") - params.UplinksSpecs = []config.UplinkInterfaceSpec{{ - InterfaceName: mainInterface, - VppDriver: strings.ToLower(*config.NativeDriver), - NewDriverName: *config.SwapDriver, - }} + params.AttachedUplinksSpecs = []*config.AttachedUplinkInterfaceSpec{ + { + UplinkInterfaceSpec: &config.UplinkInterfaceSpec{ + InterfaceName: mainInterface, + VppDriver: strings.ToLower(*config.NativeDriver), + NewDriverName: *config.SwapDriver, + }, + }, + } } /* uplinks configuration */ for index, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces { _ = uplink.Validate(nil, index == 0) - params.UplinksSpecs = append(params.UplinksSpecs, uplink) + params.AttachedUplinksSpecs = append(params.AttachedUplinksSpecs, &config.AttachedUplinkInterfaceSpec{UplinkInterfaceSpec: &config.GetCalicoVppInterfaces().UplinkInterfaces[index]}) } - if len(params.UplinksSpecs) == 0 { + if len(params.AttachedUplinksSpecs) == 0 { log.Panicf("No interface specified. Specify an interface through the environment variable") } - params.UplinksSpecs[0].IsMain = &config.True + params.AttachedUplinksSpecs[0].IsMain = &config.True /* Drivers */ params.LoadedDrivers = make(map[string]bool) @@ -87,13 +91,13 @@ func NewVppManagerParams() *config.VppManagerParams { } -func PrintVppManagerConfig(params *config.VppManagerParams, confs []*config.LinuxInterfaceState) { +func PrintVppManagerConfig(params *config.VppManagerParams) { log.Infof("-- Environment --") log.Infof("Hugepages %d", params.AvailableHugePages) log.Infof("KernelVersion %s", params.KernelVersion) log.Infof("Drivers %v", params.LoadedDrivers) log.Infof("initial iommu status %s", params.InitialVfioEnableUnsafeNoIommuMode) - for _, ifSpec := range params.UplinksSpecs { + for _, ifSpec := range params.AttachedUplinksSpecs { log.Infof("-- Interface Spec --") log.Infof("Interface Name: %s", ifSpec.InterfaceName) log.Infof("Native Driver: %s", ifSpec.VppDriver) @@ -102,21 +106,21 @@ func PrintVppManagerConfig(params *config.VppManagerParams, confs []*config.Linu log.Infof("Tap MTU: %d", ifSpec.Mtu) } - for _, conf := range confs { + for _, attachedUplink := range params.AttachedUplinksSpecs { log.Infof("-- Interface config --") - log.Infof("Node IP4: %s", conf.NodeIP4) - log.Infof("Node IP6: %s", conf.NodeIP6) - log.Infof("PciId: %s", conf.PciId) - log.Infof("Driver: %s", conf.Driver) - log.Infof("Linux IF was up ? %t", conf.IsUp) - log.Infof("Promisc was on ? %t", conf.PromiscOn) - log.Infof("DoSwapDriver: %t", conf.DoSwapDriver) - log.Infof("Mac: %s", conf.HardwareAddr.String()) - log.Infof("Addresses: [%s]", conf.AddressString()) - log.Infof("Routes: [%s]", conf.RouteString()) - log.Infof("PHY original #Queues rx:%d tx:%d", conf.NumRxQueues, conf.NumTxQueues) - log.Infof("MTU %d", conf.Mtu) - log.Infof("isTunTap %t", conf.IsTunTap) - log.Infof("isVeth %t", conf.IsVeth) + log.Infof("Node IP4: %s", attachedUplink.LinuxConf.NodeIP4) + log.Infof("Node IP6: %s", attachedUplink.LinuxConf.NodeIP6) + log.Infof("PciId: %s", attachedUplink.LinuxConf.PciId) + log.Infof("Driver: %s", attachedUplink.LinuxConf.Driver) + log.Infof("Linux IF was up ? %t", attachedUplink.LinuxConf.IsUp) + log.Infof("Promisc was on ? %t", attachedUplink.LinuxConf.PromiscOn) + log.Infof("DoSwapDriver: %t", attachedUplink.LinuxConf.DoSwapDriver) + log.Infof("Mac: %s", attachedUplink.LinuxConf.HardwareAddr.String()) + log.Infof("Addresses: [%s]", attachedUplink.LinuxConf.AddressString()) + log.Infof("Routes: [%s]", attachedUplink.LinuxConf.RouteString()) + log.Infof("PHY original #Queues rx:%d tx:%d", attachedUplink.LinuxConf.NumRxQueues, attachedUplink.LinuxConf.NumTxQueues) + log.Infof("MTU %d", attachedUplink.LinuxConf.Mtu) + log.Infof("isTunTap %t", attachedUplink.LinuxConf.IsTunTap) + log.Infof("isVeth %t", attachedUplink.LinuxConf.IsVeth) } } diff --git a/vpp-manager/uplink/af_packet.go b/vpp-manager/uplink/af_packet.go index 4985e97c..612fdbf0 100644 --- a/vpp-manager/uplink/af_packet.go +++ b/vpp-manager/uplink/af_packet.go @@ -41,40 +41,40 @@ func (d *AFPacketDriver) IsSupported(warn bool) bool { func (d *AFPacketDriver) GetDefaultRxMode() types.RxMode { return types.InterruptRxMode } func (d *AFPacketDriver) PreconfigureLinux() error { - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.attachedInterface.InterfaceName) if err != nil { - return errors.Wrapf(err, "Error finding link %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Error finding link %s", d.attachedInterface.InterfaceName) } err = netlink.SetPromiscOn(link) if err != nil { - return errors.Wrapf(err, "Error set link %s promisc on", d.spec.InterfaceName) + return errors.Wrapf(err, "Error set link %s promisc on", d.attachedInterface.InterfaceName) } return nil } func (d *AFPacketDriver) RestoreLinux(allInterfacesPhysical bool) { if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) + err := d.moveInterfaceFromNS(d.attachedInterface.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // Interface should pop back in root ns once vpp exits - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } - if !d.conf.PromiscOn { + if !d.attachedInterface.LinuxConf.PromiscOn { log.Infof("Setting promisc off") err = netlink.SetPromiscOff(link) if err != nil { - log.Errorf("Error setting link %s promisc off %v", d.spec.InterfaceName, err) + log.Errorf("Error setting link %s promisc off %v", d.attachedInterface.InterfaceName, err) } } @@ -96,7 +96,7 @@ func (d *AFPacketDriver) fetchBooleanAnnotation(annotation string, defaultValue } func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.attachedInterface.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } @@ -117,19 +117,18 @@ func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int } log.Infof("Created AF_PACKET interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.attachedInterface.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.attachedInterface.InterfaceName) if err != nil { return err } return nil } -func NewAFPacketDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *AFPacketDriver { +func NewAFPacketDriver(params *config.VppManagerParams, idx int) *AFPacketDriver { d := &AFPacketDriver{} d.name = NATIVE_DRIVER_AF_PACKET - d.conf = conf + d.attachedInterface = params.AttachedUplinksSpecs[idx] d.params = params - d.spec = spec return d } diff --git a/vpp-manager/uplink/af_xdp.go b/vpp-manager/uplink/af_xdp.go index 045dfec5..546f2030 100644 --- a/vpp-manager/uplink/af_xdp.go +++ b/vpp-manager/uplink/af_xdp.go @@ -52,9 +52,9 @@ func (d *AFXDPDriver) IsSupported(warn bool) bool { } return false } - if d.conf.Mtu > maxAfXDPMTU { + if d.attachedInterface.LinuxConf.Mtu > maxAfXDPMTU { if warn { - log.Warnf("MTU %d too large for AF_XDP (max 3072)", d.conf.Mtu) + log.Warnf("MTU %d too large for AF_XDP (max 3072)", d.attachedInterface.LinuxConf.Mtu) } return false } @@ -62,67 +62,67 @@ func (d *AFXDPDriver) IsSupported(warn bool) bool { } func (d *AFXDPDriver) PreconfigureLinux() error { - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.attachedInterface.InterfaceName) if err != nil { - return errors.Wrapf(err, "Error finding link %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Error finding link %s", d.attachedInterface.InterfaceName) } err = netlink.SetPromiscOn(link) if err != nil { - return errors.Wrapf(err, "Error setting link %s promisc on", d.spec.InterfaceName) + return errors.Wrapf(err, "Error setting link %s promisc on", d.attachedInterface.InterfaceName) } - err = utils.SetInterfaceRxQueues(d.spec.InterfaceName, d.spec.NumRxQueues) + err = utils.SetInterfaceRxQueues(d.attachedInterface.InterfaceName, d.attachedInterface.NumRxQueues) if err != nil { - log.Errorf("Error setting link %s NumQueues to %d, using %d queues: %v", d.spec.InterfaceName, d.spec.NumRxQueues, d.conf.NumRxQueues, err) + log.Errorf("Error setting link %s NumQueues to %d, using %d queues: %v", d.attachedInterface.InterfaceName, d.attachedInterface.NumRxQueues, d.attachedInterface.LinuxConf.NumRxQueues, err) /* Try with linux NumRxQueues on error, otherwise af_xdp wont start */ - d.spec.NumRxQueues = d.conf.NumRxQueues + d.attachedInterface.NumRxQueues = d.attachedInterface.LinuxConf.NumRxQueues } - if d.conf.Mtu > maxAfXDPMTU { + if d.attachedInterface.LinuxConf.Mtu > maxAfXDPMTU { log.Infof("Reducing interface MTU to %d for AF_XDP", maxAfXDPMTU) err = netlink.LinkSetMTU(link, maxAfXDPMTU) if err != nil { return errors.Wrapf(err, "Error reducing MTU to %d", maxAfXDPMTU) } - d.conf.Mtu = maxAfXDPMTU + d.attachedInterface.LinuxConf.Mtu = maxAfXDPMTU } - if d.spec.Mtu > maxAfXDPMTU { + if d.attachedInterface.Mtu > maxAfXDPMTU { log.Infof("Reducing user specified MTU to %d", maxAfXDPMTU) - d.spec.Mtu = maxAfXDPMTU + d.attachedInterface.Mtu = maxAfXDPMTU } return nil } func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) { if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) + err := d.moveInterfaceFromNS(d.attachedInterface.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // Interface should pop back in root ns once vpp exits - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } /* Restore XDP specific settings */ log.Infof("Removing AF XDP conf") - if !d.conf.PromiscOn { + if !d.attachedInterface.LinuxConf.PromiscOn { log.Infof("Setting promisc off") err = netlink.SetPromiscOff(link) if err != nil { - log.Errorf("Error setting link %s promisc off %v", d.spec.InterfaceName, err) + log.Errorf("Error setting link %s promisc off %v", d.attachedInterface.InterfaceName, err) } } - if d.conf.NumRxQueues != d.spec.NumRxQueues { - log.Infof("Setting back %d queues", d.conf.NumRxQueues) - err = utils.SetInterfaceRxQueues(d.spec.InterfaceName, d.conf.NumRxQueues) + if d.attachedInterface.LinuxConf.NumRxQueues != d.attachedInterface.NumRxQueues { + log.Infof("Setting back %d queues", d.attachedInterface.LinuxConf.NumRxQueues) + err = utils.SetInterfaceRxQueues(d.attachedInterface.InterfaceName, d.attachedInterface.LinuxConf.NumRxQueues) if err != nil { - log.Errorf("Error setting link %s NumQueues to %d %v", d.spec.InterfaceName, d.conf.NumRxQueues, err) + log.Errorf("Error setting link %s NumQueues to %d %v", d.attachedInterface.InterfaceName, d.attachedInterface.LinuxConf.NumRxQueues, err) } } @@ -131,7 +131,7 @@ func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) { } func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.attachedInterface.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } @@ -145,23 +145,22 @@ func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, u } log.Infof("Created AF_XDP interface %d", intf.SwIfIndex) - err = vpp.SetInterfaceMacAddress(intf.SwIfIndex, d.conf.HardwareAddr) + err = vpp.SetInterfaceMacAddress(intf.SwIfIndex, d.attachedInterface.LinuxConf.HardwareAddr) if err != nil { return errors.Wrap(err, "could not set af_xdp interface mac address in vpp") } - d.spec.SwIfIndex = intf.SwIfIndex - err = d.TagMainInterface(vpp, intf.SwIfIndex, d.spec.InterfaceName) + d.attachedInterface.SwIfIndex = intf.SwIfIndex + err = d.TagMainInterface(vpp, intf.SwIfIndex, d.attachedInterface.InterfaceName) if err != nil { return err } return nil } -func NewAFXDPDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *AFXDPDriver { +func NewAFXDPDriver(params *config.VppManagerParams, idx int) *AFXDPDriver { d := &AFXDPDriver{} d.name = NATIVE_DRIVER_AF_XDP - d.conf = conf + d.attachedInterface = params.AttachedUplinksSpecs[idx] d.params = params - d.spec = spec return d } diff --git a/vpp-manager/uplink/avf.go b/vpp-manager/uplink/avf.go index 0dd2aa7a..fa79c676 100644 --- a/vpp-manager/uplink/avf.go +++ b/vpp-manager/uplink/avf.go @@ -42,9 +42,9 @@ func (d *AVFDriver) IsSupported(warn bool) (supported bool) { } supported = supported && ret - ret = d.conf.Driver == config.DRIVER_I40E || d.conf.Driver == config.DRIVER_ICE + ret = d.attachedInterface.LinuxConf.Driver == config.DRIVER_I40E || d.attachedInterface.LinuxConf.Driver == config.DRIVER_ICE if !ret && warn { - log.Warnf("Interface driver is <%s>, not %s", d.conf.Driver, config.DRIVER_I40E) + log.Warnf("Interface driver is <%s>, not %s", d.attachedInterface.LinuxConf.Driver, config.DRIVER_I40E) } supported = supported && ret @@ -52,9 +52,9 @@ func (d *AVFDriver) IsSupported(warn bool) (supported bool) { } func (d *AVFDriver) PreconfigureLinux() (err error) { - pciId, err := utils.GetInterfacePciId(d.spec.InterfaceName) + pciId, err := utils.GetInterfacePciId(d.attachedInterface.InterfaceName) if err != nil { - return errors.Wrapf(err, "cannot get interface %s pciID", d.spec.InterfaceName) + return errors.Wrapf(err, "cannot get interface %s pciID", d.attachedInterface.InterfaceName) } numVFs, err := utils.GetInterfaceNumVFs(pciId) @@ -66,34 +66,34 @@ func (d *AVFDriver) PreconfigureLinux() (err error) { /* This is a PF */ d.pfPCI = pciId if numVFs == 0 { - log.Infof("Creating a VF for %s", d.spec.InterfaceName) + log.Infof("Creating a VF for %s", d.attachedInterface.InterfaceName) err := utils.CreateInterfaceVF(pciId) if err != nil { - return errors.Wrapf(err, "Couldnt create VF for %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt create VF for %s", d.attachedInterface.InterfaceName) } /* Create a mac for the new VF */ - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.attachedInterface.InterfaceName) if err != nil { - return errors.Wrapf(err, "Couldnt find Interface %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt find Interface %s", d.attachedInterface.InterfaceName) } - hardwareAddr := utils.CycleHardwareAddr(d.conf.HardwareAddr, 7) + hardwareAddr := utils.CycleHardwareAddr(d.attachedInterface.LinuxConf.HardwareAddr, 7) err = netlink.LinkSetVfHardwareAddr(link, 0 /* vf */, hardwareAddr) if err != nil { - return errors.Wrapf(err, "Couldnt set VF 0 hwaddr %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt set VF 0 hwaddr %s", d.attachedInterface.InterfaceName) } } vfPCI, err := utils.GetInterfaceVFPciId(pciId) if err != nil { - return errors.Wrapf(err, "Couldnt get VF pciID for %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt get VF pciID for %s", d.attachedInterface.InterfaceName) } d.vfPCI = vfPCI } if d.pfPCI != "" { - err := utils.SetVFSpoofTrust(d.spec.InterfaceName, 0 /* vf */, false /* spoof */, true /* trust */) + err := utils.SetVFSpoofTrust(d.attachedInterface.InterfaceName, 0 /* vf */, false /* spoof */, true /* trust */) if err != nil { - return errors.Wrapf(err, "Couldnt set VF spoof off trust on %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt set VF spoof off trust on %s", d.attachedInterface.InterfaceName) } } @@ -116,20 +116,20 @@ func (d *AVFDriver) PreconfigureLinux() (err error) { func (d *AVFDriver) RestoreLinux(allInterfacesPhysical bool) { if !allInterfacesPhysical { if d.pfPCI != "" { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) + err := d.moveInterfaceFromNS(d.attachedInterface.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } } } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } @@ -141,7 +141,7 @@ func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, upl if d.pfPCI != "" { /* We were passed a PF, move it to vpp's NS so it doesn't conflict with vpptap0 */ - err := d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err := d.moveInterfaceToNS(d.attachedInterface.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } @@ -162,19 +162,18 @@ func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, upl return errors.Wrapf(err, "Error setting AVF promisc on") } - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.attachedInterface.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.attachedInterface.InterfaceName) if err != nil { return err } return nil } -func NewAVFDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *AVFDriver { +func NewAVFDriver(params *config.VppManagerParams, idx int) *AVFDriver { d := &AVFDriver{} d.name = NATIVE_DRIVER_AVF - d.conf = conf + d.attachedInterface = params.AttachedUplinksSpecs[idx] d.params = params - d.spec = spec return d } diff --git a/vpp-manager/uplink/common.go b/vpp-manager/uplink/common.go index a02df8f4..fb66fa4e 100644 --- a/vpp-manager/uplink/common.go +++ b/vpp-manager/uplink/common.go @@ -42,10 +42,9 @@ const ( ) type UplinkDriverData struct { - conf *config.LinuxInterfaceState - params *config.VppManagerParams - name string - spec *config.UplinkInterfaceSpec + params *config.VppManagerParams + name string + attachedInterface *config.AttachedUplinkInterfaceSpec } type UplinkDriver interface { @@ -113,12 +112,12 @@ func (d *UplinkDriverData) moveInterfaceToNS(ifName string, pid int) error { } func (d *UplinkDriverData) removeLinuxIfConf(setIfDown bool) { - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.attachedInterface.UplinkInterfaceSpec.InterfaceName) if err != nil { - log.Errorf("Error finding link %s: %s", d.spec.InterfaceName, err) + log.Errorf("Error finding link %s: %s", d.attachedInterface.UplinkInterfaceSpec.InterfaceName, err) } else { // Remove routes to not have them conflict with vpptap0 - for _, route := range d.conf.Routes { + for _, route := range d.attachedInterface.LinuxConf.Routes { log.Infof("deleting Route %s", route.String()) err := netlink.RouteDel(&route) if err != nil { @@ -126,30 +125,30 @@ func (d *UplinkDriverData) removeLinuxIfConf(setIfDown bool) { } } // Remove addresses to not have them conflict with vpptap0 - for _, addr := range d.conf.Addresses { + for _, addr := range d.attachedInterface.LinuxConf.Addresses { err := netlink.AddrDel(link, &addr) if err != nil { log.Errorf("Error removing address %s from tap interface : %+v", addr, err) } } - if d.conf.IsUp && setIfDown { + if d.attachedInterface.LinuxConf.IsUp && setIfDown { err = netlink.LinkSetDown(link) if err != nil { // In case it still succeeded err2 := netlink.LinkSetUp(link) - log.Errorf("Error setting link %s down: %s (err2 %s)", d.spec.InterfaceName, err, err2) + log.Errorf("Error setting link %s down: %s (err2 %s)", d.attachedInterface.UplinkInterfaceSpec.InterfaceName, err, err2) } } } } func (d *UplinkDriverData) restoreLinuxIfConf(link netlink.Link) { - err := netlink.LinkSetMTU(link, d.conf.Mtu) + err := netlink.LinkSetMTU(link, d.attachedInterface.LinuxConf.Mtu) if err != nil { - log.Errorf("Cannot restore mtu to %d: %v", d.conf.Mtu, err) + log.Errorf("Cannot restore mtu to %d: %v", d.attachedInterface.LinuxConf.Mtu, err) } - for _, addr := range d.conf.Addresses { + for _, addr := range d.attachedInterface.LinuxConf.Addresses { log.Infof("restoring address %s", addr.String()) err := netlink.AddrAdd(link, &addr) if err != nil { @@ -157,7 +156,7 @@ func (d *UplinkDriverData) restoreLinuxIfConf(link netlink.Link) { // Keep going for the rest of the config } } - for _, route := range d.conf.Routes { + for _, route := range d.attachedInterface.LinuxConf.Routes { log.Infof("restoring route %s", route.String()) route.LinkIndex = link.Attrs().Index err := netlink.RouteAdd(&route) @@ -177,60 +176,61 @@ func (d *UplinkDriverData) UpdateVppConfigFile(template string) string { func (d *UplinkDriverData) getGenericVppInterface() types.GenericVppInterface { return types.GenericVppInterface{ - NumRxQueues: d.spec.NumRxQueues, - RxQueueSize: d.spec.RxQueueSize, - TxQueueSize: d.spec.TxQueueSize, - NumTxQueues: d.spec.NumTxQueues, - HardwareAddr: d.conf.HardwareAddr, - HostInterfaceName: d.spec.InterfaceName, + NumRxQueues: d.attachedInterface.UplinkInterfaceSpec.NumRxQueues, + RxQueueSize: d.attachedInterface.UplinkInterfaceSpec.RxQueueSize, + TxQueueSize: d.attachedInterface.UplinkInterfaceSpec.TxQueueSize, + NumTxQueues: d.attachedInterface.UplinkInterfaceSpec.NumTxQueues, + HardwareAddr: d.attachedInterface.LinuxConf.HardwareAddr, + HostInterfaceName: d.attachedInterface.UplinkInterfaceSpec.InterfaceName, } } -func SupportedUplinkDrivers(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) []UplinkDriver { +func SupportedUplinkDrivers(params *config.VppManagerParams, idx int) []UplinkDriver { lst := make([]UplinkDriver, 0) - if d := NewVirtioDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewVirtioDriver(params, idx); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewAVFDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewAVFDriver(params, idx); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewRDMADriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewRDMADriver(params, idx); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewVmxnet3Driver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewVmxnet3Driver(params, idx); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewAFXDPDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewAFXDPDriver(params, idx); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewAFPacketDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewAFPacketDriver(params, idx); d.IsSupported(false /* warn */) { lst = append(lst, d) } return lst } -func NewUplinkDriver(name string, params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) (d UplinkDriver) { +func NewUplinkDriver(params *config.VppManagerParams, idx int) (d UplinkDriver) { + name := params.AttachedUplinksSpecs[idx].VppDriver switch name { case NATIVE_DRIVER_RDMA: - d = NewRDMADriver(params, conf, spec) + d = NewRDMADriver(params, idx) case NATIVE_DRIVER_VMXNET3: - d = NewVmxnet3Driver(params, conf, spec) + d = NewVmxnet3Driver(params, idx) case NATIVE_DRIVER_AF_PACKET: - d = NewAFPacketDriver(params, conf, spec) + d = NewAFPacketDriver(params, idx) case NATIVE_DRIVER_AF_XDP: - d = NewAFXDPDriver(params, conf, spec) + d = NewAFXDPDriver(params, idx) case NATIVE_DRIVER_VIRTIO: - d = NewVirtioDriver(params, conf, spec) + d = NewVirtioDriver(params, idx) case NATIVE_DRIVER_AVF: - d = NewAVFDriver(params, conf, spec) + d = NewAVFDriver(params, idx) case NATIVE_DRIVER_DPDK: - d = NewDPDKDriver(params, conf, spec) + d = NewDPDKDriver(params, idx) case NATIVE_DRIVER_NONE: fallthrough default: log.Warnf("Using default driver") - d = NewDefaultDriver(params, conf, spec) + d = NewDefaultDriver(params, idx) } d.IsSupported(true /* warn */) return d diff --git a/vpp-manager/uplink/default.go b/vpp-manager/uplink/default.go index 61e8be99..5d1f70a9 100644 --- a/vpp-manager/uplink/default.go +++ b/vpp-manager/uplink/default.go @@ -42,13 +42,13 @@ func (d *DefaultDriver) IsSupported(warn bool) bool { func (d *DefaultDriver) PreconfigureLinux() (err error) { d.removeLinuxIfConf(true /* down */) - if d.conf.DoSwapDriver { - if d.conf.PciId == "" { + if d.attachedInterface.LinuxConf.DoSwapDriver { + if d.attachedInterface.LinuxConf.PciId == "" { log.Warnf("PCI ID not found, not swapping drivers") } else { - err = utils.SwapDriver(d.conf.PciId, d.spec.NewDriverName, true) + err = utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, d.attachedInterface.NewDriverName, true) if err != nil { - log.Warnf("Failed to swap driver to %s: %v", d.spec.NewDriverName, err) + log.Warnf("Failed to swap driver to %s: %v", d.attachedInterface.NewDriverName, err) } } } @@ -56,20 +56,20 @@ func (d *DefaultDriver) PreconfigureLinux() (err error) { } func (d *DefaultDriver) RestoreLinux(allInterfacesPhysical bool) { - if d.conf.PciId != "" && d.conf.Driver != "" { - err := utils.SwapDriver(d.conf.PciId, d.conf.Driver, false) + if d.attachedInterface.LinuxConf.PciId != "" && d.attachedInterface.LinuxConf.Driver != "" { + err := utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, d.attachedInterface.LinuxConf.Driver, false) if err != nil { - log.Warnf("Error swapping back driver to %s for %s: %v", d.conf.Driver, d.conf.PciId, err) + log.Warnf("Error swapping back driver to %s for %s: %v", d.attachedInterface.LinuxConf.Driver, d.attachedInterface.LinuxConf.PciId, err) } } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } @@ -79,29 +79,28 @@ func (d *DefaultDriver) RestoreLinux(allInterfacesPhysical bool) { func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // If interface is still in the host, move it to vpp netns to allow creation of the tap - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.attachedInterface.InterfaceName, vppPid) if err != nil { - log.Infof("Did NOT move interface %s to VPP netns: %v", d.spec.InterfaceName, err) + log.Infof("Did NOT move interface %s to VPP netns: %v", d.attachedInterface.InterfaceName, err) } else { - log.Infof("Moved interface %s to VPP netns", d.spec.InterfaceName) + log.Infof("Moved interface %s to VPP netns", d.attachedInterface.InterfaceName) } // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.GetIsMain() { + if !d.attachedInterface.GetIsMain() { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } - swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) + swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.attachedInterface.InterfaceName) if err != nil { - return fmt.Errorf("error trying to find interface with tag main-%s", d.spec.InterfaceName) + return fmt.Errorf("error trying to find interface with tag main-%s", d.attachedInterface.InterfaceName) } - d.spec.SwIfIndex = swIfIndex + d.attachedInterface.SwIfIndex = swIfIndex return nil } -func NewDefaultDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *DefaultDriver { +func NewDefaultDriver(params *config.VppManagerParams, idx int) *DefaultDriver { d := &DefaultDriver{} d.name = NATIVE_DRIVER_NONE - d.conf = conf + d.attachedInterface = params.AttachedUplinksSpecs[idx] d.params = params - d.spec = spec return d } diff --git a/vpp-manager/uplink/dpdk.go b/vpp-manager/uplink/dpdk.go index 6e0103a9..96de885d 100644 --- a/vpp-manager/uplink/dpdk.go +++ b/vpp-manager/uplink/dpdk.go @@ -38,7 +38,7 @@ type DPDKDriver struct { func (d *DPDKDriver) IsSupported(warn bool) (supported bool) { var ret bool supported = true - ret = d.conf.PciId != "" + ret = d.attachedInterface.LinuxConf.PciId != "" if !ret && warn { log.Warnf("did not find pci device id for interface") } @@ -47,18 +47,18 @@ func (d *DPDKDriver) IsSupported(warn bool) (supported bool) { } func (d *DPDKDriver) getFinalDriver() string { - if d.conf.DoSwapDriver { - return d.spec.NewDriverName + if d.attachedInterface.LinuxConf.DoSwapDriver { + return d.attachedInterface.NewDriverName } - return d.conf.Driver + return d.attachedInterface.LinuxConf.Driver } func (d *DPDKDriver) PreconfigureLinux() (err error) { d.removeLinuxIfConf(true /* down */) - if d.conf.DoSwapDriver { - err = utils.SwapDriver(d.conf.PciId, d.spec.NewDriverName, true) + if d.attachedInterface.LinuxConf.DoSwapDriver { + err = utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, d.attachedInterface.NewDriverName, true) if err != nil { - log.Warnf("Failed to swap driver to %s: %v", d.spec.NewDriverName, err) + log.Warnf("Failed to swap driver to %s: %v", d.attachedInterface.NewDriverName, err) } } if d.getFinalDriver() == config.DRIVER_VFIO_PCI && @@ -84,15 +84,15 @@ func (d *DPDKDriver) UpdateVppConfigFile(template string) string { if d.params.AvailableHugePages > 0 { template = fmt.Sprintf( "%s\ndpdk {\ndev %s { num-rx-queues %d num-tx-queues %d num-rx-desc %d num-tx-desc %d tag %s } \n}\n", - template, d.conf.PciId, d.spec.NumRxQueues, d.spec.NumTxQueues, - d.spec.RxQueueSize, d.spec.TxQueueSize, "main-"+d.spec.InterfaceName, + template, d.attachedInterface.LinuxConf.PciId, d.attachedInterface.NumRxQueues, d.attachedInterface.NumTxQueues, + d.attachedInterface.RxQueueSize, d.attachedInterface.TxQueueSize, "main-"+d.attachedInterface.InterfaceName, ) } else { template = fmt.Sprintf( "%s\ndpdk {\niova-mode va\nno-hugetlb\ndev %s { num-rx-queues %d num-tx-queues %d num-rx-desc %d num-tx-desc %d tag %s } \n}\n", - template, d.conf.PciId, d.spec.NumRxQueues, d.spec.NumTxQueues, - d.spec.RxQueueSize, d.spec.TxQueueSize, "main-"+d.spec.InterfaceName, + template, d.attachedInterface.LinuxConf.PciId, d.attachedInterface.NumRxQueues, d.attachedInterface.NumTxQueues, + d.attachedInterface.RxQueueSize, d.attachedInterface.TxQueueSize, "main-"+d.attachedInterface.InterfaceName, ) // If no hugepages, also edit `buffers {}` @@ -114,29 +114,29 @@ write: } func (d *DPDKDriver) restoreInterfaceName() error { - newName, err := utils.GetInterfaceNameFromPci(d.conf.PciId) + newName, err := utils.GetInterfaceNameFromPci(d.attachedInterface.LinuxConf.PciId) if err != nil { - return errors.Wrapf(err, "Error getting new if name for %s: %v", newName, d.conf.PciId) + return errors.Wrapf(err, "Error getting new if name for %s: %v", newName, d.attachedInterface.LinuxConf.PciId) } - if newName == d.spec.InterfaceName { + if newName == d.attachedInterface.InterfaceName { return nil } link, err := netlink.LinkByName(newName) if err != nil { return errors.Wrapf(err, "Error getting new link %s: %v", newName, link) } - err = netlink.LinkSetName(link, d.spec.InterfaceName) + err = netlink.LinkSetName(link, d.attachedInterface.InterfaceName) if err != nil { - return errors.Wrapf(err, "Error setting new if name for %s: %v", d.spec.InterfaceName, link) + return errors.Wrapf(err, "Error setting new if name for %s: %v", d.attachedInterface.InterfaceName, link) } return nil } func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { - if d.conf.PciId != "" && d.conf.Driver != "" { - err := utils.SwapDriver(d.conf.PciId, d.conf.Driver, false) + if d.attachedInterface.LinuxConf.PciId != "" && d.attachedInterface.LinuxConf.Driver != "" { + err := utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, d.attachedInterface.LinuxConf.Driver, false) if err != nil { - log.Warnf("Error swapping back driver to %s for %s: %v", d.conf.Driver, d.conf.PciId, err) + log.Warnf("Error swapping back driver to %s for %s: %v", d.attachedInterface.LinuxConf.Driver, d.attachedInterface.LinuxConf.PciId, err) } } @@ -150,14 +150,14 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { time.Sleep(500 * time.Millisecond) } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } @@ -177,16 +177,16 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // Nothing to do VPP autocreates on startup // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.GetIsMain() { + if !d.attachedInterface.GetIsMain() { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } - swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) + swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.attachedInterface.InterfaceName) if err != nil || swIfIndex == ^uint32(0) { - return fmt.Errorf("error trying to find interface with tag main-%s", d.spec.InterfaceName) + return fmt.Errorf("error trying to find interface with tag main-%s", d.attachedInterface.InterfaceName) } - log.Debugf("Found interface with swIfIndex %d for %s", swIfIndex, d.spec.InterfaceName) - d.spec.SwIfIndex = swIfIndex - err = vpp.SetInterfaceMacAddress(swIfIndex, d.conf.HardwareAddr) + log.Debugf("Found interface with swIfIndex %d for %s", swIfIndex, d.attachedInterface.InterfaceName) + d.attachedInterface.SwIfIndex = swIfIndex + err = vpp.SetInterfaceMacAddress(swIfIndex, d.attachedInterface.LinuxConf.HardwareAddr) if err != nil && gerrors.Is(err, types.VppErrorUnimplemented) { log.Warn("Setting dpdk interface mac address in vpp unsupported") } else if err != nil { @@ -195,11 +195,10 @@ func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, up return nil } -func NewDPDKDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *DPDKDriver { +func NewDPDKDriver(params *config.VppManagerParams, idx int) *DPDKDriver { d := &DPDKDriver{} d.name = NATIVE_DRIVER_DPDK - d.conf = conf + d.attachedInterface = params.AttachedUplinksSpecs[idx] d.params = params - d.spec = spec return d } diff --git a/vpp-manager/uplink/rdma.go b/vpp-manager/uplink/rdma.go index 0ec581d5..328b06f8 100644 --- a/vpp-manager/uplink/rdma.go +++ b/vpp-manager/uplink/rdma.go @@ -33,9 +33,9 @@ func (d *RDMADriver) IsSupported(warn bool) (supported bool) { var ret bool supported = true - ret = d.conf.Driver == config.DRIVER_MLX5_CORE + ret = d.attachedInterface.LinuxConf.Driver == config.DRIVER_MLX5_CORE if !ret && warn { - log.Warnf("Interface driver is <%s>, not %s", d.conf.Driver, config.DRIVER_MLX5_CORE) + log.Warnf("Interface driver is <%s>, not %s", d.attachedInterface.LinuxConf.Driver, config.DRIVER_MLX5_CORE) } supported = supported && ret @@ -49,19 +49,19 @@ func (d *RDMADriver) PreconfigureLinux() (err error) { func (d *RDMADriver) RestoreLinux(allInterfacesPhysical bool) { if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) + err := d.moveInterfaceFromNS(d.attachedInterface.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } @@ -79,26 +79,25 @@ func (d *RDMADriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, up return errors.Wrapf(err, "Error creating RDMA interface") } - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.attachedInterface.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } log.Infof("Created RDMA interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.attachedInterface.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.attachedInterface.InterfaceName) if err != nil { return err } return nil } -func NewRDMADriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *RDMADriver { +func NewRDMADriver(params *config.VppManagerParams, idx int) *RDMADriver { d := &RDMADriver{} d.name = NATIVE_DRIVER_RDMA - d.conf = conf + d.attachedInterface = params.AttachedUplinksSpecs[idx] d.params = params - d.spec = spec return d } diff --git a/vpp-manager/uplink/virtio.go b/vpp-manager/uplink/virtio.go index 78497268..323afb34 100644 --- a/vpp-manager/uplink/virtio.go +++ b/vpp-manager/uplink/virtio.go @@ -45,9 +45,9 @@ func (d *VirtioDriver) IsSupported(warn bool) (supported bool) { } supported = supported && ret - ret = d.conf.Driver == config.DRIVER_VIRTIO_PCI + ret = d.attachedInterface.LinuxConf.Driver == config.DRIVER_VIRTIO_PCI if !ret && warn { - log.Warnf("Interface driver is <%s>, not %s", d.conf.Driver, config.DRIVER_VIRTIO_PCI) + log.Warnf("Interface driver is <%s>, not %s", d.attachedInterface.LinuxConf.Driver, config.DRIVER_VIRTIO_PCI) } supported = supported && ret @@ -55,11 +55,11 @@ func (d *VirtioDriver) IsSupported(warn bool) (supported bool) { } func (d *VirtioDriver) PreconfigureLinux() (err error) { - newDriverName := d.spec.NewDriverName - doSwapDriver := d.conf.DoSwapDriver + newDriverName := d.attachedInterface.NewDriverName + doSwapDriver := d.attachedInterface.LinuxConf.DoSwapDriver if newDriverName == "" { newDriverName = config.DRIVER_VFIO_PCI - doSwapDriver = config.DRIVER_VFIO_PCI != d.conf.Driver + doSwapDriver = config.DRIVER_VFIO_PCI != d.attachedInterface.LinuxConf.Driver } if d.params.InitialVfioEnableUnsafeNoIommuMode == config.VFIO_UNSAFE_NO_IOMMU_MODE_NO { @@ -70,7 +70,7 @@ func (d *VirtioDriver) PreconfigureLinux() (err error) { } d.removeLinuxIfConf(true /* down */) if doSwapDriver { - err = utils.SwapDriver(d.conf.PciId, newDriverName, true) + err = utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, newDriverName, true) if err != nil { log.Warnf("Failed to swap driver to %s: %v", newDriverName, err) } @@ -85,26 +85,26 @@ func (d *VirtioDriver) RestoreLinux(allInterfacesPhysical bool) { log.Warnf("Virtio restore error %v", err) } } - if d.conf.PciId != "" && d.conf.Driver != "" { - err := utils.SwapDriver(d.conf.PciId, d.conf.Driver, false) + if d.attachedInterface.LinuxConf.PciId != "" && d.attachedInterface.LinuxConf.Driver != "" { + err := utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, d.attachedInterface.LinuxConf.Driver, false) if err != nil { - log.Warnf("Error swapping back driver to %s for %s: %v", d.conf.Driver, d.conf.PciId, err) + log.Warnf("Error swapping back driver to %s for %s: %v", d.attachedInterface.LinuxConf.Driver, d.attachedInterface.LinuxConf.PciId, err) } } if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) + err := d.moveInterfaceFromNS(d.attachedInterface.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } @@ -115,7 +115,7 @@ func (d *VirtioDriver) RestoreLinux(allInterfacesPhysical bool) { func (d *VirtioDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { intf := types.VirtioInterface{ GenericVppInterface: d.getGenericVppInterface(), - PciId: d.conf.PciId, + PciId: d.attachedInterface.LinuxConf.PciId, } swIfIndex, err := vpp.CreateVirtio(&intf) if err != nil { @@ -123,19 +123,18 @@ func (d *VirtioDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, } log.Infof("Created VIRTIO interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.attachedInterface.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.attachedInterface.InterfaceName) if err != nil { return err } return nil } -func NewVirtioDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *VirtioDriver { +func NewVirtioDriver(params *config.VppManagerParams, idx int) *VirtioDriver { d := &VirtioDriver{} d.name = NATIVE_DRIVER_VIRTIO - d.conf = conf d.params = params - d.spec = spec + d.attachedInterface = params.AttachedUplinksSpecs[idx] return d } diff --git a/vpp-manager/uplink/vmxnet3.go b/vpp-manager/uplink/vmxnet3.go index 33b1f433..cce1e2f3 100644 --- a/vpp-manager/uplink/vmxnet3.go +++ b/vpp-manager/uplink/vmxnet3.go @@ -33,9 +33,9 @@ func (d *Vmxnet3Driver) IsSupported(warn bool) (supported bool) { var ret bool supported = true - ret = d.conf.Driver == config.DRIVER_VMXNET3 + ret = d.attachedInterface.LinuxConf.Driver == config.DRIVER_VMXNET3 if !ret && warn { - log.Warnf("Interface driver is <%s>, not %s", d.conf.Driver, config.DRIVER_VMXNET3) + log.Warnf("Interface driver is <%s>, not %s", d.attachedInterface.LinuxConf.Driver, config.DRIVER_VMXNET3) } supported = supported && ret @@ -50,35 +50,35 @@ func (d *Vmxnet3Driver) PreconfigureLinux() (err error) { } } d.removeLinuxIfConf(true /* down */) - driverName, err := utils.GetDriverNameFromPci(d.conf.PciId) + driverName, err := utils.GetDriverNameFromPci(d.attachedInterface.LinuxConf.PciId) if err != nil { - return errors.Wrapf(err, "Couldnt get VF driver Name for %s", d.conf.PciId) + return errors.Wrapf(err, "Couldnt get VF driver Name for %s", d.attachedInterface.LinuxConf.PciId) } if driverName != config.DRIVER_VFIO_PCI { - err := utils.SwapDriver(d.conf.PciId, config.DRIVER_VFIO_PCI, true) + err := utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, config.DRIVER_VFIO_PCI, true) if err != nil { - return errors.Wrapf(err, "Couldnt swap %s to vfio_pci", d.conf.PciId) + return errors.Wrapf(err, "Couldnt swap %s to vfio_pci", d.attachedInterface.LinuxConf.PciId) } } return nil } func (d *Vmxnet3Driver) RestoreLinux(allInterfacesPhysical bool) { - if d.conf.PciId != "" && d.conf.Driver != "" { - err := utils.SwapDriver(d.conf.PciId, d.conf.Driver, true) + if d.attachedInterface.LinuxConf.PciId != "" && d.attachedInterface.LinuxConf.Driver != "" { + err := utils.SwapDriver(d.attachedInterface.LinuxConf.PciId, d.attachedInterface.LinuxConf.Driver, true) if err != nil { - log.Warnf("Error swapping back driver to %s for %s: %v", d.conf.Driver, d.conf.PciId, err) + log.Warnf("Error swapping back driver to %s for %s: %v", d.attachedInterface.LinuxConf.Driver, d.attachedInterface.LinuxConf.PciId, err) } } - if !d.conf.IsUp { + if !d.attachedInterface.LinuxConf.IsUp { return } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.attachedInterface.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.attachedInterface.InterfaceName, err) return } @@ -97,7 +97,7 @@ func (d *Vmxnet3Driver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, intf := types.Vmxnet3Interface{ GenericVppInterface: d.getGenericVppInterface(), EnableGso: *config.GetCalicoVppDebug().GSOEnabled, - PciId: d.conf.PciId, + PciId: d.attachedInterface.LinuxConf.PciId, } swIfIndex, err := vpp.CreateVmxnet3(&intf) if err != nil { @@ -106,19 +106,18 @@ func (d *Vmxnet3Driver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, log.Infof("Created Vmxnet3 interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.attachedInterface.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.attachedInterface.InterfaceName) if err != nil { return err } return nil } -func NewVmxnet3Driver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *Vmxnet3Driver { +func NewVmxnet3Driver(params *config.VppManagerParams, idx int) *Vmxnet3Driver { d := &Vmxnet3Driver{} d.name = NATIVE_DRIVER_VMXNET3 - d.conf = conf + d.attachedInterface = params.AttachedUplinksSpecs[idx] d.params = params - d.spec = spec return d } diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index 62f131ca..85190726 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -46,21 +46,15 @@ import ( const DefaultPhysicalNetworkName = "" type VppRunner struct { - params *config.VppManagerParams - conf []*config.LinuxInterfaceState - vpp *vpplink.VppLink - uplinkDriver []uplink.UplinkDriver + vpp *vpplink.VppLink } -func NewVPPRunner(params *config.VppManagerParams, confs []*config.LinuxInterfaceState) *VppRunner { - return &VppRunner{ - params: params, - conf: confs, - } +func NewVPPRunner() *VppRunner { + return &VppRunner{} } -func (v *VppRunner) GenerateVppConfigExecFile() error { - template, err := config.TemplateScriptReplace(*config.ConfigExecTemplate, v.params, v.conf) +func (v *VppRunner) GenerateVppConfigExecFile(params *config.VppManagerParams) error { + template, err := config.TemplateScriptReplace(*config.ConfigExecTemplate, params, true) if err != nil { return err } @@ -72,8 +66,8 @@ func (v *VppRunner) GenerateVppConfigExecFile() error { return err } -func (v *VppRunner) GenerateVppConfigFile(drivers []uplink.UplinkDriver) error { - template, err := config.TemplateScriptReplace(*config.ConfigTemplate, v.params, v.conf) +func (v *VppRunner) GenerateVppConfigFile(drivers []uplink.UplinkDriver, params *config.VppManagerParams) error { + template, err := config.TemplateScriptReplace(*config.ConfigTemplate, params, true) if err != nil { return err } @@ -88,34 +82,49 @@ func (v *VppRunner) GenerateVppConfigFile(drivers []uplink.UplinkDriver) error { return err } -func (v *VppRunner) Run(drivers []uplink.UplinkDriver) error { - v.uplinkDriver = drivers - for idx := range v.conf { - log.Infof("Running with uplink %s", drivers[idx].GetName()) +func (v *VppRunner) preconfigureLinux(drivers []uplink.UplinkDriver, params *config.VppManagerParams) (err error) { + for idx := range params.AttachedUplinksSpecs { + err = drivers[idx].PreconfigureLinux() + if err != nil { + return errors.Wrapf(err, "Error pre-configuring Linux main IF: %s", drivers[idx]) + } } - err := v.GenerateVppConfigExecFile() + return nil +} + +func (v *VppRunner) preconfigureVPP(drivers []uplink.UplinkDriver, params *config.VppManagerParams) error { + err := v.GenerateVppConfigExecFile(params) if err != nil { return errors.Wrap(err, "Error generating VPP config Exec") } - err = v.GenerateVppConfigFile(drivers) + err = v.GenerateVppConfigFile(drivers, params) if err != nil { return errors.Wrap(err, "Error generating VPP config") } - for idx := range v.conf { - err = v.uplinkDriver[idx].PreconfigureLinux() - if err != nil { - return errors.Wrapf(err, "Error pre-configuring Linux main IF: %s", v.uplinkDriver[idx]) - } + err = v.preconfigureLinux(drivers, params) + if err != nil { + return errors.Wrap(err, "Error pre-configuring Linux") } - config.RunHook(config.HookScriptBeforeVppRun, "BEFORE_VPP_RUN", v.params, log) - err = v.runVpp() + return nil +} + +func (v *VppRunner) Run(drivers []uplink.UplinkDriver, params *config.VppManagerParams) error { + for idx, attachedInterface := range params.AttachedUplinksSpecs { + log.Infof("Running %s with uplink %s", attachedInterface.InterfaceName, drivers[idx].GetName()) + } + err := v.preconfigureVPP(drivers, params) + if err != nil { + return err + } + config.RunHook(config.HookScriptBeforeVppRun, "BEFORE_VPP_RUN", params, log) + err = v.runVpp(drivers, params) if err != nil { return errors.Wrapf(err, "Error running VPP") } - config.RunHook(config.HookScriptVppDoneOk, "VPP_DONE_OK", v.params, log) + config.RunHook(config.HookScriptVppDoneOk, "VPP_DONE_OK", params, log) return nil } @@ -359,7 +368,7 @@ func (v *VppRunner) allocateStaticVRFs() error { } // Configure specific VRFs for a given tap to the host to handle broadcast / multicast traffic sent by the host -func (v *VppRunner) setupTapVRF(ifSpec *config.UplinkInterfaceSpec, ifState *config.LinuxInterfaceState, tapSwIfIndex uint32) (vrfs []uint32, err error) { +func (v *VppRunner) setupVppHostVRF(ifSpec *config.UplinkInterfaceSpec, ifState *config.LinuxInterfaceState, tapSwIfIndex uint32) (vrfs []uint32, err error) { for _, ipFamily := range vpplink.IpFamilies { vrfId, err := v.vpp.AllocateVRF(ipFamily.IsIp6, fmt.Sprintf("host-tap-%s-%s", ifSpec.InterfaceName, ipFamily.Str)) if err != nil { @@ -418,95 +427,82 @@ func (v *VppRunner) setupTapVRF(ifSpec *config.UplinkInterfaceSpec, ifState *con return vrfs, nil } -// configureVppUplinkInterface configures one uplink interface in VPP -// and creates the corresponding tap in Linux -func (v *VppRunner) configureVppUplinkInterface( - uplinkDriver uplink.UplinkDriver, - ifState *config.LinuxInterfaceState, - ifSpec config.UplinkInterfaceSpec, -) (err error) { - // Configure the physical network if we see it for the first time - if _, ok := config.Info.PhysicalNets[ifSpec.PhysicalNetworkName]; !ok { - err = v.AllocatePhysicalNetworkVRFs(ifSpec.PhysicalNetworkName) - if err != nil { - return err - } - } - +func (v *VppRunner) configureVppUplink(swIfIndex uint32, uplinkMtu int, mode types.RxMode, physicalNetworkName string) (err error) { // Always enable GSO feature on data interface, only a tiny negative effect on perf if GSO is not // enabled on the taps or already done before an encap if *config.GetCalicoVppDebug().GSOEnabled { - err = v.vpp.EnableGSOFeature(ifSpec.SwIfIndex) + err = v.vpp.EnableGSOFeature(swIfIndex) if err != nil { return errors.Wrap(err, "Error enabling GSO on uplink interface") } } - uplinkMtu := vpplink.DefaultIntTo(ifSpec.Mtu, ifState.Mtu) - err = v.vpp.SetInterfaceMtu(ifSpec.SwIfIndex, uplinkMtu) + err = v.vpp.SetInterfaceMtu(swIfIndex, uplinkMtu) if err != nil { return errors.Wrapf(err, "Error setting %d MTU on uplink interface", uplinkMtu) } - err = v.vpp.SetInterfaceRxMode(ifSpec.SwIfIndex, types.AllQueues, ifSpec.GetRxModeWithDefault(uplinkDriver.GetDefaultRxMode())) + err = v.vpp.SetInterfaceRxMode(swIfIndex, types.AllQueues, mode) if err != nil { log.Warnf("%v", err) } - err = v.vpp.EnableInterfaceIP6(ifSpec.SwIfIndex) + err = v.vpp.EnableInterfaceIP6(swIfIndex) if err != nil { return errors.Wrap(err, "Error enabling ipv6 on uplink interface") } - err = v.vpp.DisableIP6RouterAdvertisements(ifSpec.SwIfIndex) + err = v.vpp.DisableIP6RouterAdvertisements(swIfIndex) if err != nil { return errors.Wrap(err, "Error disabling ipv6 RA on uplink interface") } - err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex) + err = v.vpp.CnatEnableFeatures(swIfIndex) if err != nil { return errors.Wrap(err, "Error configuring NAT on uplink interface") } - if ifSpec.PhysicalNetworkName != "" { + if physicalNetworkName != "" { for _, ipFamily := range vpplink.IpFamilies { - err = v.vpp.SetInterfaceVRF(ifSpec.SwIfIndex, config.Info.PhysicalNets[ifSpec.PhysicalNetworkName].VrfId, ipFamily.IsIp6) + err = v.vpp.SetInterfaceVRF(swIfIndex, config.Info.PhysicalNets[physicalNetworkName].VrfId, ipFamily.IsIp6) if err != nil { - return errors.Wrapf(err, "error setting interface in vrf %d", config.Info.PhysicalNets[ifSpec.PhysicalNetworkName]) + return errors.Wrapf(err, "error setting interface in vrf %d", config.Info.PhysicalNets[physicalNetworkName]) } } - value := config.Info.PhysicalNets[ifSpec.PhysicalNetworkName] - config.Info.PhysicalNets[ifSpec.PhysicalNetworkName] = config.PhysicalNetwork{ + value := config.Info.PhysicalNets[physicalNetworkName] + config.Info.PhysicalNets[physicalNetworkName] = config.PhysicalNetwork{ VrfId: value.VrfId, PodVrfId: value.PodVrfId, } } + return +} - for _, addr := range ifState.Addresses { +func (v *VppRunner) addAddressesAndRoutesToVppUplink(swIfIndex uint32, addresses []netlink.Addr, routes []netlink.Route, isMain bool) (err error) { + for _, addr := range addresses { if addr.IPNet.IP.IsLinkLocalUnicast() && !common.IsFullyQualified(addr.IPNet) && common.IsV6Cidr(addr.IPNet) { log.Infof("Not adding address %s to uplink interface (vpp requires /128 link-local)", addr.String()) continue } else { log.Infof("Adding address %s to uplink interface", addr.String()) } - err = v.vpp.AddInterfaceAddress(ifSpec.SwIfIndex, addr.IPNet) + err = v.vpp.AddInterfaceAddress(swIfIndex, addr.IPNet) if err != nil { log.Errorf("Error adding address to uplink interface: %v", err) } } - for _, route := range ifState.Routes { + for _, route := range routes { err = v.vpp.RouteAdd(&types.Route{ Dst: route.Dst, Paths: []types.RoutePath{{ Gw: route.Gw, - SwIfIndex: ifSpec.SwIfIndex, + SwIfIndex: swIfIndex, }}, }) if err != nil { log.Errorf("cannot add route in vpp: %v", err) } } - gws, err := config.GetCalicoVppInitialConfig().GetDefaultGWs() if err != nil { return err @@ -516,73 +512,48 @@ func (v *VppRunner) configureVppUplinkInterface( err = v.vpp.RouteAdd(&types.Route{ Paths: []types.RoutePath{{ Gw: defaultGW, - SwIfIndex: ifSpec.SwIfIndex, + SwIfIndex: swIfIndex, }}, }) if err != nil { log.Errorf("cannot add default route via %s in vpp: %v", defaultGW, err) } } - - if ifSpec.GetIsMain() { + if isMain { if config.GetCalicoVppInitialConfig().ExtraAddrCount > 0 { - err = v.addExtraAddresses(ifState.Addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, ifSpec.SwIfIndex) + err = v.addExtraAddresses(addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, swIfIndex) if err != nil { log.Errorf("Cannot configure requested extra addresses: %v", err) } } } + return +} - log.Infof("Creating Linux side interface") - vpptap0Flags := types.TapFlagNone - if *config.GetCalicoVppDebug().GSOEnabled { - vpptap0Flags = vpptap0Flags | types.TapFlagGSO | types.TapGROCoalesce - } - - tapSwIfIndex, err := v.vpp.CreateTapV2(&types.TapV2{ - GenericVppInterface: types.GenericVppInterface{ - HostInterfaceName: ifSpec.InterfaceName, - RxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize, - TxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize, - HardwareAddr: utils.VppSideMac, - }, - HostNamespace: "pid:1", // create tap in root netns - Tag: "host-" + ifSpec.InterfaceName, - Flags: vpptap0Flags, - HostMtu: uplinkMtu, - HostMacAddress: ifState.HardwareAddr, - }) - if err != nil { - return errors.Wrap(err, "Error creating tap") - } - - vrfs, err := v.setupTapVRF(&ifSpec, ifState, tapSwIfIndex) - if err != nil { - return errors.Wrap(err, "error configuring VRF for tap") - } +func (v *VppRunner) configureVppHost(swIfIndex uint32, ifState *config.LinuxInterfaceState, vrfs []uint32) (err error) { // Always set this tap on worker 0 - err = v.vpp.SetInterfaceRxPlacement(tapSwIfIndex, 0 /*queue*/, 0 /*worker*/, false /*main*/) + err = v.vpp.SetInterfaceRxPlacement(swIfIndex, 0 /*queue*/, 0 /*worker*/, false /*main*/) if err != nil { return errors.Wrap(err, "Error setting tap rx placement") } - err = v.vpp.SetInterfaceMtu(uint32(tapSwIfIndex), vpplink.MAX_MTU) + err = v.vpp.SetInterfaceMtu(uint32(swIfIndex), vpplink.MAX_MTU) if err != nil { return errors.Wrapf(err, "Error setting %d MTU on tap interface", vpplink.MAX_MTU) } if ifState.Hasv6 { - err = v.vpp.DisableIP6RouterAdvertisements(tapSwIfIndex) + err = v.vpp.DisableIP6RouterAdvertisements(swIfIndex) if err != nil { return errors.Wrap(err, "Error disabling ip6 RA on vpptap0") } } - err = v.configurePunt(tapSwIfIndex, *ifState) + err = v.configurePunt(swIfIndex, *ifState) if err != nil { return errors.Wrap(err, "Error adding redirect to tap") } - err = v.vpp.EnableArpProxy(tapSwIfIndex, vrfs[0 /* ip4 */]) + err = v.vpp.EnableArpProxy(swIfIndex, vrfs[0 /* ip4 */]) if err != nil { return errors.Wrap(err, "Error enabling ARP proxy") } @@ -590,7 +561,7 @@ func (v *VppRunner) configureVppUplinkInterface( for _, addr := range ifState.Addresses { if addr.IPNet.IP.To4() == nil { log.Infof("Adding ND proxy for address %s", addr.IPNet.IP) - err = v.vpp.EnableIP6NdProxy(tapSwIfIndex, addr.IPNet.IP) + err = v.vpp.EnableIP6NdProxy(swIfIndex, addr.IPNet.IP) if err != nil { log.Errorf("Error configuring nd proxy for address %s: %v", addr.IPNet.IP.String(), err) } @@ -598,36 +569,104 @@ func (v *VppRunner) configureVppUplinkInterface( } if *config.GetCalicoVppDebug().GSOEnabled { - err = v.vpp.EnableGSOFeature(tapSwIfIndex) + err = v.vpp.EnableGSOFeature(swIfIndex) if err != nil { return errors.Wrap(err, "Error enabling GSO on vpptap0") } } - err = v.vpp.SetInterfaceRxMode(tapSwIfIndex, types.AllQueues, config.GetCalicoVppInterfaces().VppHostTapSpec.GetRxModeWithDefault(types.AdaptativeRxMode)) + err = v.vpp.SetInterfaceRxMode(swIfIndex, types.AllQueues, config.GetCalicoVppInterfaces().VppHostTapSpec.GetRxModeWithDefault(types.AdaptativeRxMode)) if err != nil { log.Errorf("Error SetInterfaceRxMode on vpptap0 %v", err) } - err = v.vpp.CnatEnableFeatures(tapSwIfIndex) + err = v.vpp.CnatEnableFeatures(swIfIndex) if err != nil { return errors.Wrap(err, "Error configuring NAT on vpptap0") } - err = v.vpp.RegisterPodInterface(tapSwIfIndex) + err = v.vpp.RegisterPodInterface(swIfIndex) if err != nil { return errors.Wrap(err, "error configuring vpptap0 as pod intf") } - err = v.vpp.RegisterHostInterface(tapSwIfIndex) + err = v.vpp.RegisterHostInterface(swIfIndex) if err != nil { return errors.Wrap(err, "error configuring vpptap0 as host intf") } + return +} + +func (v *VppRunner) createVppHost(interfaceName string, uplinkMtu int, hwAddr net.HardwareAddr) (tapSwIfIndex uint32, err error) { + vpptap0Flags := types.TapFlagNone + if *config.GetCalicoVppDebug().GSOEnabled { + vpptap0Flags = vpptap0Flags | types.TapFlagGSO | types.TapGROCoalesce + } + + tapSwIfIndex, err = v.vpp.CreateTapV2(&types.TapV2{ + GenericVppInterface: types.GenericVppInterface{ + HostInterfaceName: interfaceName, + RxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize, + TxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize, + HardwareAddr: utils.VppSideMac, + }, + HostNamespace: "pid:1", // create tap in root netns + Tag: "host-" + interfaceName, + Flags: vpptap0Flags, + HostMtu: uplinkMtu, + HostMacAddress: hwAddr, + }) + if err != nil { + return tapSwIfIndex, errors.Wrap(err, "Error creating tap") + } + + return +} + +// configureInterface configures one uplink interface in VPP +// and creates the corresponding tap in Linux +func (v *VppRunner) configureInterface(attachedInterface *config.AttachedUplinkInterfaceSpec, rxMode types.RxMode) (err error) { + ifState := attachedInterface.LinuxConf + // Configure the physical network if we see it for the first time + if _, ok := config.Info.PhysicalNets[attachedInterface.PhysicalNetworkName]; !ok { + err = v.allocatePhysicalNetworkVRFs(attachedInterface.PhysicalNetworkName) + if err != nil { + return err + } + } + + uplinkMtu := vpplink.DefaultIntTo(attachedInterface.Mtu, ifState.Mtu) + + err = v.configureVppUplink(attachedInterface.SwIfIndex, uplinkMtu, rxMode, attachedInterface.PhysicalNetworkName) + if err != nil { + return errors.Wrap(err, "Error configuring vpp uplink") + } + err = v.addAddressesAndRoutesToVppUplink(attachedInterface.SwIfIndex, ifState.Addresses, ifState.Routes, attachedInterface.GetIsMain()) + if err != nil { + return errors.Wrap(err, "Error adding addresses and routes to vpp uplink") + } + + log.Infof("Creating Linux side interface") + + tapSwIfIndex, err := v.createVppHost(attachedInterface.InterfaceName, uplinkMtu, ifState.HardwareAddr) + if err != nil { + return errors.Wrap(err, "error creating vpp host (tap0)") + } + + vrfs, err := v.setupVppHostVRF(attachedInterface.UplinkInterfaceSpec, ifState, tapSwIfIndex) + if err != nil { + return errors.Wrap(err, "error configuring VRF for tap") + } + + err = v.configureVppHost(tapSwIfIndex, ifState, vrfs) + if err != nil { + return errors.Wrap(err, "error configuring vpp host interface (tap0)") + } // Linux side tap setup - link, err := netlink.LinkByName(ifSpec.InterfaceName) + link, err := netlink.LinkByName(attachedInterface.InterfaceName) if err != nil { - return errors.Wrapf(err, "cannot find interface named %s", ifSpec.InterfaceName) + return errors.Wrapf(err, "cannot find interface named %s", attachedInterface.InterfaceName) } fakeNextHopIP4, fakeNextHopIP6, err := v.configureLinuxTap(link, *ifState) @@ -643,12 +682,12 @@ func (v *VppRunner) configureVppUplinkInterface( if config.Info.UplinkStatuses != nil { config.Info.UplinkStatuses[link.Attrs().Name] = config.UplinkStatus{ TapSwIfIndex: tapSwIfIndex, - SwIfIndex: ifSpec.SwIfIndex, + SwIfIndex: attachedInterface.SwIfIndex, Mtu: uplinkMtu, - PhysicalNetworkName: ifSpec.PhysicalNetworkName, + PhysicalNetworkName: attachedInterface.PhysicalNetworkName, LinkIndex: link.Attrs().Index, Name: link.Attrs().Name, - IsMain: ifSpec.GetIsMain(), + IsMain: attachedInterface.GetIsMain(), FakeNextHopIP4: fakeNextHopIP4, FakeNextHopIP6: fakeNextHopIP6, } @@ -667,6 +706,11 @@ func (v *VppRunner) doVppGlobalConfiguration() (err error) { return errors.Wrap(err, "Error configuring cnat source policy") } + err = v.configureGlobalPunt() + if err != nil { + return errors.Wrap(err, "Error adding redirect to tap") + } + return nil } @@ -733,7 +777,7 @@ func (v *VppRunner) updateCalicoNode(ifState *config.LinuxInterfaceState) (err e return errors.Wrap(err, "Error updating node") } -func (v *VppRunner) pingCalicoVpp() error { +func (v *VppRunner) pingCalicoVpp(sig syscall.Signal, sigstr string) error { dat, err := os.ReadFile(config.CalicoVppPidFile) if err != nil { if errors.Is(err, os.ErrNotExist) { @@ -746,24 +790,24 @@ func (v *VppRunner) pingCalicoVpp() error { if err != nil { return errors.Wrapf(err, "Error parsing %s", dat) } - err = syscall.Kill(int(pid), syscall.SIGUSR1) + err = syscall.Kill(int(pid), sig) if err != nil { - return errors.Wrapf(err, "Error kill -SIGUSR1 %d", int(pid)) + return errors.Wrapf(err, "Error kill -%s %d", sigstr, int(pid)) } - log.Infof("Did kill -SIGUSR1 %d", int(pid)) + log.Infof("Did kill -%s %d", sigstr, int(pid)) return nil } -func (v *VppRunner) allInterfacesPhysical() bool { - for _, ifConf := range v.conf { - if ifConf.IsTunTap || ifConf.IsVeth { +func (v *VppRunner) allInterfacesPhysical(params *config.VppManagerParams) bool { + for _, attachedInterface := range params.AttachedUplinksSpecs { + if attachedInterface.LinuxConf.IsTunTap || attachedInterface.LinuxConf.IsVeth { return false } } return true } -func (v *VppRunner) AllocatePhysicalNetworkVRFs(phyNet string) (err error) { +func (v *VppRunner) allocatePhysicalNetworkVRFs(phyNet string) (err error) { // for ip4 mainId, err := v.vpp.AllocateVRF(false, fmt.Sprintf("physical-net-%s-ip4", phyNet)) if err != nil { @@ -792,9 +836,8 @@ func (v *VppRunner) AllocatePhysicalNetworkVRFs(phyNet string) (err error) { return nil } -// Returns VPP exit code -func (v *VppRunner) runVpp() (err error) { - if !v.allInterfacesPhysical() { // use separate net namespace because linux deletes these interfaces when ns is deleted +func (v *VppRunner) startVPP(params *config.VppManagerParams) (err error) { + if !v.allInterfacesPhysical(params) { // use separate net namespace because linux deletes these interfaces when ns is deleted if ns.IsNSorErr(utils.GetnetnsPath(config.VppNetnsName)) != nil { _, err = utils.NewNS(config.VppNetnsName) if err != nil { @@ -836,70 +879,86 @@ func (v *VppRunner) runVpp() (err error) { } vppProcess = vppCmd.Process } + return nil +} - /** - * From this point it is very important that every exit - * path calls restoreConfiguration after vpp exits */ - defer v.restoreConfiguration(v.allInterfacesPhysical()) - - log.Infof("VPP started [PID %d]", vppProcess.Pid) - runningCond.Broadcast() +func (v *VppRunner) createInterface(uplinkDriver uplink.UplinkDriver, attachedInterface *config.AttachedUplinkInterfaceSpec) error { + err := uplinkDriver.CreateMainVppInterface(v.vpp, vppProcess.Pid, attachedInterface.UplinkInterfaceSpec) + if err != nil { + return errors.Wrapf(err, "Error creating uplink interface %s", attachedInterface.InterfaceName) + } + // Data interface configuration + err = v.vpp.Retry(2*time.Second, 10, v.vpp.InterfaceAdminUp, attachedInterface.SwIfIndex) + if err != nil { + return errors.Wrap(err, "Error setting uplink interface up") + } + return nil +} - // If needed, wait some time that vpp boots up - time.Sleep(time.Duration(config.GetCalicoVppInitialConfig().VppStartupSleepSeconds) * time.Second) +func (v *VppRunner) createAndConfigureInterface(uplinkDriver uplink.UplinkDriver, attachedInterface *config.AttachedUplinkInterfaceSpec) (err error) { + err = v.createInterface(uplinkDriver, attachedInterface) + if err != nil { + return errors.Wrap(err, "Error configuring VPP interface") + } + rxMode := attachedInterface.GetRxModeWithDefault(uplinkDriver.GetDefaultRxMode()) + err = v.configureInterface(attachedInterface, rxMode) + if err != nil { + return errors.Wrap(err, "Error configuring VPP interface") + } + return nil +} +func (v *VppRunner) connectAndConfigureVpp() error { vpp, err := utils.CreateVppLink() v.vpp = vpp if err != nil { - terminateVpp("Error connecting to VPP: %v", err) - v.vpp.Close() - <-vppDeadChan - return fmt.Errorf("cannot connect to VPP after 10 tries") + return errors.Wrapf(err, "Error connecting to VPP after 10 tries") } - err = v.doVppGlobalConfiguration() if err != nil { - terminateVpp("Error configuring VPP: %v", err) - v.vpp.Close() - <-vppDeadChan return errors.Wrap(err, "Error configuring VPP") } - // add main network that has the default VRF config.Info.PhysicalNets[DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex} + return nil +} - err = v.configureGlobalPunt() +// Returns VPP exit code +func (v *VppRunner) runVpp(drivers []uplink.UplinkDriver, params *config.VppManagerParams) (err error) { + err = v.startVPP(params) if err != nil { - return errors.Wrap(err, "Error adding redirect to tap") + return err } - for idx := 0; idx < len(v.params.UplinksSpecs); idx++ { - err := v.uplinkDriver[idx].CreateMainVppInterface(vpp, vppProcess.Pid, &v.params.UplinksSpecs[idx]) - if err != nil { - terminateVpp("Error creating uplink interface %s: %v", v.params.UplinksSpecs[idx].InterfaceName, err) - v.vpp.Close() - <-vppDeadChan - return errors.Wrap(err, "Error creating uplink interface") - } + /** + * From this point it is very important that every exit + * path calls restoreConfiguration after vpp exits */ + defer v.restoreConfiguration(drivers, v.allInterfacesPhysical(params), *params) - // Data interface configuration - err = v.vpp.Retry(2*time.Second, 10, v.vpp.InterfaceAdminUp, v.params.UplinksSpecs[idx].SwIfIndex) - if err != nil { - terminateVpp("Error setting uplink interface up: %v", err) - v.vpp.Close() - <-vppDeadChan - return errors.Wrap(err, "Error setting uplink interface up") - } + log.Infof("VPP started [PID %d]", vppProcess.Pid) + runningCond.Broadcast() + + // If needed, wait some time that vpp boots up + time.Sleep(time.Duration(config.GetCalicoVppInitialConfig().VppStartupSleepSeconds) * time.Second) - err = v.configureVppUplinkInterface(v.uplinkDriver[idx], v.conf[idx], v.params.UplinksSpecs[idx]) + err = v.connectAndConfigureVpp() + if err != nil { + terminateVpp(err.Error()) + v.vpp.Close() + <-vppDeadChan + return fmt.Errorf(err.Error()) + } + for idx := range params.AttachedUplinksSpecs { + err := v.createAndConfigureInterface(drivers[idx], params.AttachedUplinksSpecs[idx]) if err != nil { - terminateVpp("Error configuring VPP: %v", err) + terminateVpp(err.Error()) + v.vpp.Close() <-vppDeadChan - return errors.Wrap(err, "Error configuring VPP") + return errors.Wrap(err, err.Error()) } } // Update the Calico node with the IP address actually configured on VPP - err = v.updateCalicoNode(v.conf[0]) + err = v.updateCalicoNode(params.AttachedUplinksSpecs[0].LinuxConf) if err != nil { terminateVpp("Error updating Calico node (SIGINT %d): %v", vppProcess.Pid, err) <-vppDeadChan @@ -914,29 +973,86 @@ func (v *VppRunner) runVpp() (err error) { var t tomb.Tomb // close vpp as we do not program - v.vpp.Close() - config.RunHook(config.HookScriptVppRunning, "VPP_RUNNING", v.params, log) + //v.vpp.Close() + config.RunHook(config.HookScriptVppRunning, "VPP_RUNNING", params, log) - <-vppDeadChan - log.Infof("VPP Exited: status %v", err) + log.Info("waiting first time") + select { + case <-vppDeadChan: + v.vpp.Close() + log.Infof("VPP Exited: status %v", err) - err = t.Killf("Vpp exited, stopping watchers") + err = t.Killf("Vpp exited, stopping watchers") + if err != nil { + log.Errorf("Error Killf vpp: %v", err) + } + case newInfo := <-dynamicInterfaceAdd: + err = v.addDynamicInterface(newInfo) + if err != nil { + log.Error(err) + } + } + return nil +} + +func (v *VppRunner) addDynamicInterface(newInfo dynamicInterface) (err error) { + driver := newInfo.driver + drivers := []uplink.UplinkDriver{driver} + params := newInfo.params + log.Infof("%+v", params) + log.Infof("%+v", drivers) + err = v.preconfigureLinux(drivers, params) + if err != nil { + return errors.Wrap(err, "Error pre-configuring Linux") + } + /** + * From this point it is very important that every exit + * path calls restoreConfiguration after vpp exits */ + defer v.restoreConfiguration(drivers, v.allInterfacesPhysical(params), *params) + err = v.createAndConfigureInterface(drivers[0], params.AttachedUplinksSpecs[0]) if err != nil { - log.Errorf("Error Killf vpp: %v", err) + terminateVpp(err.Error()) + v.vpp.Close() + <-vppDeadChan + return errors.Wrap(err, err.Error()) + } + err = utils.WriteInfoFile() + if err != nil { + log.Errorf("Error writing vpp manager file: %v", err) + } + err = v.pingCalicoVpp(syscall.SIGUSR2, "SIGUSR2") + if err != nil { + log.Errorf("Error pinging calico-vpp: %v", err) + } + var t tomb.Tomb + log.Info("waiting n time") + select { + case <-vppDeadChan: + v.vpp.Close() + log.Infof("VPP Exited: status %v", err) + + err = t.Killf("Vpp exited, stopping watchers") + if err != nil { + log.Errorf("Error Killf vpp: %v", err) + } + case newInfo := <-dynamicInterfaceAdd: + err = v.addDynamicInterface(newInfo) + if err != nil { + log.Error(err) + } } return nil } -func (v *VppRunner) restoreConfiguration(allInterfacesPhysical bool) { - log.Infof("Restoring configuration") +func (v *VppRunner) restoreConfiguration(drivers []uplink.UplinkDriver, allInterfacesPhysical bool, params config.VppManagerParams) { err := utils.ClearVppManagerFiles() if err != nil { log.Errorf("Error clearing vpp manager files: %v", err) } - for idx := range v.params.UplinksSpecs { - v.uplinkDriver[idx].RestoreLinux(allInterfacesPhysical) + for idx := range params.AttachedUplinksSpecs { + drivers[idx].RestoreLinux(allInterfacesPhysical) } - err = v.pingCalicoVpp() + err = v.pingCalicoVpp(syscall.SIGUSR1, "SIGUSR1") if err != nil { log.Errorf("Error pinging calico-vpp: %v", err) }