Skip to content

Commit

Permalink
add redirect to host rules for dns issue in kind
Browse files Browse the repository at this point in the history
A dns issue showed in kind cluster, due to specific iptables rules added by docker.
This commit addresses that by adding a feature allowing to redirect some traffic to host,
to get it pass through iptables.
This is applied to coredns pods.
We use some new user config to specify rulesfor this feature.
A tag "host" is added to cnat snat poliy feature and we use it for our vpptap0 to
disable snat on traffic outgoing through it.
  • Loading branch information
hedibouattour authored and sknat committed Aug 22, 2023
1 parent d055915 commit b969608
Show file tree
Hide file tree
Showing 21 changed files with 3,938 additions and 15 deletions.
82 changes: 82 additions & 0 deletions calico-vpp-agent/cni/cni_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common"
"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers"
"github.com/projectcalico/vpp-dataplane/v3/config"
"github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils"
"github.com/projectcalico/vpp-dataplane/v3/vpplink"
"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
)
Expand All @@ -61,6 +62,8 @@ type Server struct {

availableBuffers uint64

RedirectToHostClassifyTableIndex uint32

networkDefinitions sync.Map
cniMultinetEventChan chan common.CalicoVppEvent
nodeBGPSpec *common.LocalNodeSpec
Expand Down Expand Up @@ -245,6 +248,12 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro
ErrorMessage: err.Error(),
}, nil
}
if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" {
err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex)
if err != nil {
return nil, err
}
}

s.podInterfaceMap[podSpec.Key()] = *podSpec
cniServerStateFile := fmt.Sprintf("%s%d", config.CniServerStateFile, storage.CniServerStateFileVersion)
Expand Down Expand Up @@ -315,6 +324,34 @@ func (s *Server) rescanState() {
default:
s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err)
}
if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" {
err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex)
if err != nil {
s.log.Error(err)
}
}
}
}

func (s *Server) DelRedirectToHostOnInterface(swIfIndex uint32) error {
err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableId, types.InvalidTableId, false /*isAdd*/)
if err != nil {
return errors.Wrapf(err, "Error deleting classify input table from interface")
} else {
s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex)
return nil
}
}

func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32) error {
s.log.Infof("Setting classify input acl table %d on interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex)
err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableId, types.InvalidTableId, true)
if err != nil {
s.log.Warnf("Error setting classify input table: %s, retrying...", err)
return errors.Errorf("could not set input acl table %d for interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex)
} else {
s.log.Infof("set input acl table %d for interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex)
return nil
}
}

Expand Down Expand Up @@ -442,6 +479,47 @@ forloop:
return nil
}

func (s *Server) getMainTap0Info() (tapSwIfIndex uint32, address net.IP) {
for _, i := range common.VppManagerInfo.UplinkStatuses {
if i.IsMain {
tapSwIfIndex = i.TapSwIfIndex
break
}
}
address = utils.FakeVppNextHopIP4
return
}

func (s *Server) createRedirectToHostRules() (uint32, error) {
var maxNumEntries uint32
if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 {
maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules))
} else {
maxNumEntries = 1
}
index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{
Mask: types.DstThreeTupleMask,
NextTableIndex: types.InvalidID,
MaxNumEntries: maxNumEntries,
MissNextIndex: ^uint32(0),
})
if err != nil {
return types.InvalidID, err
}
tap0swifindex, tap0nexthop := s.getMainTap0Info()
for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules {
err = s.vpp.AddSessionRedirect(&types.SessionRedirect{
FiveTuple: types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.Ip), rule.Port),
TableIndex: index,
}, &types.RoutePath{Gw: tap0nexthop, SwIfIndex: tap0swifindex})
if err != nil {
return types.InvalidID, err
}
}

return index, nil
}

func (s *Server) ServeCNI(t *tomb.Tomb) error {
err := syscall.Unlink(config.CNIServerSocket)
if err != nil && !gerrors.Is(err, os.ErrNotExist) {
Expand All @@ -453,6 +531,10 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error {
return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket)
}

s.RedirectToHostClassifyTableIndex, err = s.createRedirectToHostRules()
if err != nil {
return err
}
cniproto.RegisterCniDataplaneServer(s.grpcServer, s)

if *config.GetCalicoVppFeatureGates().MultinetEnabled {
Expand Down
6 changes: 6 additions & 0 deletions calico-vpp-agent/cni/network_vpp.go
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,12 @@ err:

// CleanUpVPPNamespace deletes the devices in the network namespace.
func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) {
if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" {
err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex)
if err != nil {
s.log.Error(err)
}
}
err := ns.IsNSorErr(podSpec.NetnsName)
if err != nil {
s.log.Infof("pod(del) netns '%s' doesn't exist, skipping", podSpec.NetnsName)
Expand Down
9 changes: 9 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,13 @@ func (u *UplinkInterfaceSpec) String() string {
return string(b)
}

type RedirectToHostRulesConfigType struct {
Port uint16 `json:"port,omitempty"`
Ip string `json:"ip,omitempty"`
/* "tcp", "udp",... */
Proto types.IPProto `json:"proto,omitempty"`
}

type CalicoVppDebugConfigType struct {
PoliciesEnabled *bool `json:"policiesEnabled,omitempty"`
ServicesEnabled *bool `json:"servicesEnabled,omitempty"`
Expand Down Expand Up @@ -401,6 +408,8 @@ type CalicoVppInitialConfigConfigType struct { //out of agent and vppmanager
IfConfigSavePath string `json:"ifConfigSavePath"`
/* Comma separated list of IPs to be configured in VPP as default GW */
DefaultGWs string `json:"defaultGWs"`
/* List of rules for redirecting traffic to host */
RedirectToHostRules []RedirectToHostRulesConfigType `json:"redirectToHostRules"`
}

func (self *CalicoVppInitialConfigConfigType) Validate() (err error) { return nil }
Expand Down
7 changes: 7 additions & 0 deletions docs/developper_guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ export CALICO_ENCAPSULATION_V4=IPIP
export CALICO_ENCAPSULATION_V6=None
export CALICO_NAT_OUTGOING=Enabled
````
To add a redirection to host rule:
````bash
# --------------- redirect ----------------
export CALICOVPP_REDIRECT_PROTO="\"udp\""
export CALICOVPP_REDIRECT_PORT=53
export CALICOVPP_REDIRECT_IP="\"172.18.0.1\""
````
To run with hugepages on:

````bash
Expand Down
6 changes: 6 additions & 0 deletions vpp-manager/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ import (
"github.com/projectcalico/vpp-dataplane/v3/vpplink"
)

var (
FakeVppNextHopIP4 = net.ParseIP("169.254.0.1")
FakeVppNextHopIP6 = net.ParseIP("fc00:ffff:ffff:ffff:ca11:c000:fd10:fffe")
VppSideMac, _ = net.ParseMAC("02:ca:11:c0:fd:10")
)

func IsDriverLoaded(driver string) (bool, error) {
_, err := os.Stat("/sys/bus/pci/drivers/" + driver)
if err == nil {
Expand Down
15 changes: 7 additions & 8 deletions vpp-manager/vpp_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,6 @@ import (

const DefaultPhysicalNetworkName = ""

var (
fakeVppNextHopIP4 = net.ParseIP("169.254.0.1")
fakeVppNextHopIP6 = net.ParseIP("fc00:ffff:ffff:ffff:ca11:c000:fd10:fffe")
vppSideMac, _ = net.ParseMAC("02:ca:11:c0:fd:10")
)

type VppRunner struct {
params *config.VppManagerParams
conf []*config.LinuxInterfaceState
Expand Down Expand Up @@ -147,7 +141,7 @@ func (v *VppRunner) configureGlobalPunt() (err error) {
}

func (v *VppRunner) configurePunt(tapSwIfIndex uint32, ifState config.LinuxInterfaceState) (err error) {
for _, neigh := range []net.IP{fakeVppNextHopIP4, fakeVppNextHopIP6} {
for _, neigh := range []net.IP{utils.FakeVppNextHopIP4, utils.FakeVppNextHopIP6} {
err = v.vpp.AddNeighbor(&types.Neighbor{
SwIfIndex: tapSwIfIndex,
IP: neigh,
Expand Down Expand Up @@ -550,7 +544,7 @@ func (v *VppRunner) configureVppUplinkInterface(
HostInterfaceName: ifSpec.InterfaceName,
RxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize,
TxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize,
HardwareAddr: vppSideMac,
HardwareAddr: utils.VppSideMac,
},
HostNamespace: "pid:1", // create tap in root netns
Tag: "host-" + ifSpec.InterfaceName,
Expand Down Expand Up @@ -625,6 +619,11 @@ func (v *VppRunner) configureVppUplinkInterface(
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
}

err = v.vpp.RegisterHostInterface(tapSwIfIndex)
if err != nil {
return errors.Wrap(err, "error configuring vpptap0 as host intf")
}

// Linux side tap setup
link, err := netlink.LinkByName(ifSpec.InterfaceName)
if err != nil {
Expand Down
138 changes: 138 additions & 0 deletions vpplink/classify.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
// Copyright (C) 2023 Cisco Systems Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vpplink

import (
"fmt"

"github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/classify"
"github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/interface_types"
"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
)

func (v *VppLink) addDelClassifyTable(table *types.ClassifyTable, action types.ClassifyAction) (uint32, error) {
client := classify.NewServiceClient(v.GetConnection())

isAdd, delChain := false, false
currentDataFlags := uint8(0)
tableIndex := types.InvalidTableId
switch action {
case types.AddAbsolute:
isAdd = true
case types.AddRelative:
isAdd = true
currentDataFlags = uint8(1)
case types.Del:
tableIndex = table.TableIndex
case types.DelChain:
tableIndex = table.TableIndex
delChain = true
}

mask := ExtendToVector(table.Mask)

matchNVectors := table.MatchNVectors
if matchNVectors == 0 {
matchNVectors = uint32(len(mask)) / types.VectorSize
}

nBuckets := table.NBuckets
if nBuckets == 0 {
// We provide as many buckets as the max number of entries we expect in the table
nBuckets = table.MaxNumEntries
}

memorySize := table.MemorySize
if memorySize == 0 {
/* memory needed for the table:
* - each entry has a size of (32-bytes + mask vectors)
* - up to 2 entries per page for collision resolution
* - double for margin
*/
memorySize = table.MaxNumEntries * (32 + matchNVectors*types.VectorSize) * 2 * 2
}

response, err := client.ClassifyAddDelTable(v.GetContext(), &classify.ClassifyAddDelTable{
IsAdd: isAdd,
DelChain: delChain,
TableIndex: tableIndex,
Nbuckets: nBuckets,
MemorySize: memorySize,
SkipNVectors: table.SkipNVectors,
MatchNVectors: matchNVectors,
NextTableIndex: table.NextTableIndex,
MissNextIndex: table.MissNextIndex,
CurrentDataFlag: currentDataFlags,
CurrentDataOffset: table.CurrentDataOffset,
MaskLen: uint32(len(mask)),
Mask: mask,
})

if err != nil {
return types.InvalidID, fmt.Errorf("Failed to %s the classify table: %w", map[bool]string{true: "add", false: "del"}[isAdd], err)
}

return response.NewTableIndex, nil
}

func (v *VppLink) AddClassifyTable(table *types.ClassifyTable) (uint32, error) {
return v.addDelClassifyTable(table, types.AddRelative)
}

func (v *VppLink) DelClassifyTable(tableIndex uint32) error {
_, err := v.addDelClassifyTable(&types.ClassifyTable{TableIndex: tableIndex}, types.Del)
return err
}

func ExtendToVector(match []byte) []byte {
n := len(match) % types.VectorSize
if n != 0 {
match = match[:len(match)+types.VectorSize-n]
}
return match
}

func (v *VppLink) SetClassifyInputInterfaceTables(swIfIndex uint32, ip4TableIndex uint32, ip6TableIndex uint32, l2TableIndex uint32, isAdd bool) error {
client := classify.NewServiceClient(v.GetConnection())

_, err := client.InputACLSetInterface(v.GetContext(), &classify.InputACLSetInterface{
IsAdd: isAdd,
SwIfIndex: interface_types.InterfaceIndex(swIfIndex),
IP4TableIndex: ip4TableIndex,
IP6TableIndex: ip6TableIndex,
L2TableIndex: l2TableIndex,
})
if err != nil {
return fmt.Errorf("failed to set input acl tables for this interface: %w", err)
}
return nil
}

func (v *VppLink) SetClassifyOutputInterfaceTables(swIfIndex uint32, ip4TableIndex uint32, ip6TableIndex uint32, l2TableIndex uint32, isAdd bool) error {
client := classify.NewServiceClient(v.GetConnection())

_, err := client.OutputACLSetInterface(v.GetContext(), &classify.OutputACLSetInterface{
IsAdd: isAdd,
SwIfIndex: interface_types.InterfaceIndex(swIfIndex),
IP4TableIndex: ip4TableIndex,
IP6TableIndex: ip6TableIndex,
L2TableIndex: l2TableIndex,
})
if err != nil {
return fmt.Errorf("failed to set input acl tables for this interface: %w", err)
}
return nil
}
8 changes: 8 additions & 0 deletions vpplink/cnat.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,14 @@ func (v *VppLink) RemovePodInterface(swIfIndex uint32) (err error) {
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, false /* isAdd */, cnat.CNAT_POLICY_POD)
}

func (v *VppLink) RegisterHostInterface(swIfIndex uint32) (err error) {
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, true /* isAdd */, cnat.CNAT_POLICY_HOST)
}

func (v *VppLink) RemoveHostInterface(swIfIndex uint32) (err error) {
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, false /* isAdd */, cnat.CNAT_POLICY_HOST)
}

func (v *VppLink) EnableDisableCnatSNAT(swIfIndex uint32, isIp6 bool, isEnable bool) (err error) {
if isEnable {
return v.enableCnatSNAT(swIfIndex, isIp6)
Expand Down
Loading

0 comments on commit b969608

Please sign in to comment.