simplify punt fake gateway

This patch simplifies the way we handle the punt routes
when routing for-me traffic VPP cannot handle to the host.
Previously, we were using different fake gateways for ip4 & ip6.
We now use the ip4 indirection in both cases.
We also generate different hardware addresses using the uplink order
in the configuration.

Finally, we also make the isMain parameter public in the config.

Signed-off-by: Nathan Skrzypczak <[email protected]>

config/config.go

@@ -58,6 +58,11 @@ const (
 	VppSigKillTimeout = 2
 	DefaultEncapSize  = 60 // Used to lower the MTU of the routes to the cluster
 
+	DefaultPhysicalNetworkName = ""
+
+	// BaseVppSideHardwareAddress is the base hardware address of VPP side of the HostPunt
+	// tap interface. It is used to generate hardware addresses for each uplink interface.
+	BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00"
 )
 
 var (
@@ -130,6 +135,10 @@ var (
 	}
 
 	Info = &VppManagerInfo{}
+
+	// VppHostPuntFakeGatewayAddress is the fake gateway we use with a static neighbor
+	// in the punt table to route punted packets to the host
+	VppHostPuntFakeGatewayAddress = net.ParseIP("169.254.0.1")
 )
 
 func RunHook(hookScript *string, hookName string, params *VppManagerParams, log *logrus.Logger) {
@@ -222,7 +231,7 @@ func (i *InterfaceSpec) Validate(maxIfSpec *InterfaceSpec) error {
 
 type UplinkInterfaceSpec struct {
 	InterfaceSpec
-	IsMain              *bool             `json:"-"`
+	IsMain              bool              `json:"isMain"`
 	PhysicalNetworkName string            `json:"physicalNetworkName"`
 	InterfaceName       string            `json:"interfaceName"`
 	VppDriver           string            `json:"vppDriver"`
@@ -231,17 +240,26 @@ type UplinkInterfaceSpec struct {
 	// Mtu is the User specified MTU for uplink & the tap
 	Mtu       int    `json:"mtu"`
 	SwIfIndex uint32 `json:"-"`
+
+	// uplinkInterfaceIndex is the index of the uplinkInterface in the list
+	uplinkInterfaceIndex int `json:"-"`
 }
 
-func (u *UplinkInterfaceSpec) GetIsMain() bool {
-	if u.IsMain == nil {
-		return false
+func (u *UplinkInterfaceSpec) GetVppSideHardwareAddress() net.HardwareAddr {
+	mac, _ := net.ParseMAC(BaseVppSideHardwareAddress)
+	mac[len(mac)-1] = byte(u.uplinkInterfaceIndex)
+	if u.uplinkInterfaceIndex > 255 {
+		panic("too many uplinkinteraces")
 	}
-	return *u.IsMain
+	return mac
+}
+
+func (u *UplinkInterfaceSpec) SetUplinkInterfaceIndex(uplinkInterfaceIndex int) {
+	u.uplinkInterfaceIndex = uplinkInterfaceIndex
 }
 
-func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec, isMain bool) (err error) {
-	if !isMain && u.VppDriver == "" {
+func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec) (err error) {
+	if !u.IsMain && u.VppDriver == "" {
 		return errors.Errorf("vpp driver should be specified for secondary uplink interfaces")
 	}
 	return u.InterfaceSpec.Validate(maxIfSpec)

vpp-manager/startup/startup.go

@@ -40,14 +40,30 @@ func NewVppManagerParams() *config.VppManagerParams {
 	}
 
 	/* uplinks configuration */
-	for index, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces {
-		_ = uplink.Validate(nil, index == 0)
+	isMainCount := 0
+	for _, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces {
 		params.UplinksSpecs = append(params.UplinksSpecs, uplink)
+		if uplink.IsMain {
+			isMainCount++
+		}
 	}
 	if len(params.UplinksSpecs) == 0 {
 		log.Panicf("No interface specified. Specify an interface through the environment variable")
 	}
-	params.UplinksSpecs[0].IsMain = &config.True
+	if isMainCount == 0 {
+		// By default the first interface is main
+		params.UplinksSpecs[0].IsMain = true
+	} else if isMainCount > 1 {
+		log.Panicf("Too many interfaces tagged Main")
+	}
+
+	for index, uplink := range params.UplinksSpecs {
+		uplink.SetUplinkInterfaceIndex(index)
+		err := uplink.Validate(nil)
+		if err != nil {
+			log.Panicf("error validating uplink %s %s", uplink.String(), err)
+		}
+	}
 
 	/* Drivers */
 	params.LoadedDrivers = make(map[string]bool)

vpp-manager/uplink/default.go

@@ -86,7 +86,7 @@ func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int,
 		log.Infof("Moved interface %s to VPP netns", d.spec.InterfaceName)
 	}
 	// refusing to run on secondary interfaces as we have no way to figure out the sw_if_index
-	if !d.spec.GetIsMain() {
+	if !d.spec.IsMain {
 		return fmt.Errorf("%s driver not supported for secondary interfaces", d.name)
 	}
 	swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName)

vpp-manager/uplink/dpdk.go

@@ -177,7 +177,7 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) {
 func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	// Nothing to do VPP autocreates on startup
 	// refusing to run on secondary interfaces as we have no way to figure out the sw_if_index
-	if !d.spec.GetIsMain() {
+	if !d.spec.IsMain {
 		return fmt.Errorf("%s driver not supported for secondary interfaces", d.name)
 	}
 	swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName)

vpp-manager/vpp_runner.go

@@ -43,8 +43,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
 
-const DefaultPhysicalNetworkName = ""
-
 type VppRunner struct {
 	params       *config.VppManagerParams
 	conf         []*config.LinuxInterfaceState
@@ -141,28 +139,26 @@ func (v *VppRunner) configureGlobalPunt() (err error) {
 }
 
 func (v *VppRunner) configurePunt(tapSwIfIndex uint32, ifState config.LinuxInterfaceState) (err error) {
-	for _, neigh := range []net.IP{utils.FakeVppNextHopIP4, utils.FakeVppNextHopIP6} {
-		err = v.vpp.AddNeighbor(&types.Neighbor{
-			SwIfIndex:    tapSwIfIndex,
-			IP:           neigh,
-			HardwareAddr: ifState.HardwareAddr,
+	err = v.vpp.AddNeighbor(&types.Neighbor{
+		SwIfIndex:    tapSwIfIndex,
+		IP:           config.VppHostPuntFakeGatewayAddress,
+		HardwareAddr: ifState.HardwareAddr,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "Error adding neighbor %s to tap", config.VppHostPuntFakeGatewayAddress)
+	}
+	/* In the punt table (where all punted traffics ends), route to the tap */
+	for _, address := range ifState.Addresses {
+		err = v.vpp.RouteAdd(&types.Route{
+			Dst:   address.IPNet,
+			Table: common.PuntTableId,
+			Paths: []types.RoutePath{{
+				Gw:        config.VppHostPuntFakeGatewayAddress,
+				SwIfIndex: tapSwIfIndex,
+			}},
 		})
 		if err != nil {
-			return errors.Wrapf(err, "Error adding neighbor %s to tap", neigh)
-		}
-		/* In the punt table (where all punted traffics ends), route to the tap */
-		for _, address := range ifState.Addresses {
-			err = v.vpp.RouteAdd(&types.Route{
-				Dst:   address.IPNet,
-				Table: common.PuntTableId,
-				Paths: []types.RoutePath{{
-					Gw:        neigh,
-					SwIfIndex: tapSwIfIndex,
-				}},
-			})
-			if err != nil {
-				return errors.Wrapf(err, "error adding vpp side routes for interface")
-			}
+			return errors.Wrapf(err, "error adding vpp side routes for interface")
 		}
 	}
 
@@ -524,7 +520,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 		}
 	}
 
-	if ifSpec.GetIsMain() {
+	if ifSpec.IsMain {
 		if config.GetCalicoVppInitialConfig().ExtraAddrCount > 0 {
 			err = v.addExtraAddresses(ifState.Addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, ifSpec.SwIfIndex)
 			if err != nil {
@@ -544,7 +540,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 			HostInterfaceName: ifSpec.InterfaceName,
 			RxQueueSize:       config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize,
 			TxQueueSize:       config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize,
-			HardwareAddr:      utils.VppSideMac,
+			HardwareAddr:      ifSpec.GetVppSideHardwareAddress(),
 		},
 		HostNamespace:  "pid:1", // create tap in root netns
 		Tag:            "host-" + ifSpec.InterfaceName,
@@ -648,7 +644,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 			PhysicalNetworkName: ifSpec.PhysicalNetworkName,
 			LinkIndex:           link.Attrs().Index,
 			Name:                link.Attrs().Name,
-			IsMain:              ifSpec.GetIsMain(),
+			IsMain:              ifSpec.IsMain,
 			FakeNextHopIP4:      fakeNextHopIP4,
 			FakeNextHopIP6:      fakeNextHopIP6,
 		}
@@ -866,7 +862,7 @@ func (v *VppRunner) runVpp() (err error) {
 	}
 
 	// add main network that has the default VRF
-	config.Info.PhysicalNets[DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex}
+	config.Info.PhysicalNets[config.DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex}
 
 	err = v.configureGlobalPunt()
 	if err != nil {