Skip to content

Commit

Permalink
add redirect to host rules for dns issue in kind
Browse files Browse the repository at this point in the history
A dns issue showed in kind cluster, due to specific iptables rules added by docker.
This commit addresses that by adding a feature allowing to redirect some traffic to host,
to get it pass through iptables.
This is applied to coredns pods.
We use some new user config to specify rulesfor this feature.
A tag "host" is added to cnat snat poliy feature and we use it for our vpptap0 to
disable snat on traffic outgoing through it.
  • Loading branch information
hedibouattour committed Aug 16, 2023
1 parent 6b35da9 commit 1850f59
Show file tree
Hide file tree
Showing 21 changed files with 3,957 additions and 6 deletions.
82 changes: 82 additions & 0 deletions calico-vpp-agent/cni/cni_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"fmt"
"net"
"os"
"strings"
"sync"
"syscall"

Expand Down Expand Up @@ -61,6 +62,8 @@ type Server struct {

availableBuffers uint64

RedirectToHostClassifyTableIndex uint32

networkDefinitions sync.Map
cniMultinetEventChan chan common.CalicoVppEvent
nodeBGPSpec *common.LocalNodeSpec
Expand Down Expand Up @@ -245,6 +248,12 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro
ErrorMessage: err.Error(),
}, nil
}
if s.PodNeedsRedirectToHost(podSpec) {
err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex, 30)
if err != nil {
return nil, err
}
}

s.podInterfaceMap[podSpec.Key()] = *podSpec
cniServerStateFile := fmt.Sprintf("%s%d", config.CniServerStateFile, storage.CniServerStateFileVersion)
Expand Down Expand Up @@ -315,9 +324,57 @@ func (s *Server) rescanState() {
default:
s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err)
}
if s.PodNeedsRedirectToHost(&podSpecCopy) {
err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex, 30)
if err != nil {
s.log.Error(err)
}
}
}
}

func (s *Server) PodNeedsRedirectToHost(podSpec *storage.LocalPodSpec) bool {
return strings.HasPrefix(podSpec.WorkloadID, "kube-system/coredns")
}

func (s *Server) DelRedirectToHostOnInterface(swIfIndex uint32) error {
err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableId, types.InvalidTableId, false /*isAdd*/)
if err != nil {
return errors.Wrapf(err, "Error deleting classify input table from interface")
} else {
s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex)
return nil
}
}

func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32, timeout int) error {
s.log.Infof("Setting classify input acl table %d on interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex)
done := make(chan bool)
go func() {
times := 0
for {
err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableId, types.InvalidTableId, true)
if err != nil {
s.log.Warnf("Error setting classify input table: %s, retrying...", err)
} else {
s.log.Infof("set input acl table %d for interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex)
done <- true
return
}
times += 1
if times >= timeout {
done <- false
return
}
}
}()
result := <-done
if !result {
return errors.Errorf("could not set input acl table %d for interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex)
}
return nil
}

func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) {
partialPodSpec := NewLocalPodSpecFromDel(request)
// Only try to delete the device if a namespace was passed in.
Expand Down Expand Up @@ -442,6 +499,27 @@ forloop:
return nil
}

func (s *Server) createRedirectToHostRules() (uint32, error) {
var missNextIndex int32 = -1
index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{
Mask: types.DstThreeTupleMask,
NextTableIndex: types.InvalidID,
MaxNumEntries: 200000,
MissNextIndex: uint32(missNextIndex),
})
if err != nil {
return types.InvalidID, err
}
err = s.vpp.AddSessionRedirect(&types.SessionRedirect{
FiveTuple: types.NewDst3Tuple(config.GetRedirectToHostRules().Proto, net.ParseIP(config.GetSnatExclusionPrefixes().Prefix), config.GetRedirectToHostRules().Port),
TableIndex: index,
}, &types.RoutePath{Gw: net.ParseIP(config.GetRedirectToHostRules().Ip), SwIfIndex: 2})
if err != nil {
return types.InvalidID, err
}
return index, nil
}

func (s *Server) ServeCNI(t *tomb.Tomb) error {
err := syscall.Unlink(config.CNIServerSocket)
if err != nil && !gerrors.Is(err, os.ErrNotExist) {
Expand All @@ -453,6 +531,10 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error {
return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket)
}

s.RedirectToHostClassifyTableIndex, err = s.createRedirectToHostRules()
if err != nil {
return err
}
cniproto.RegisterCniDataplaneServer(s.grpcServer, s)

if *config.GetCalicoVppFeatureGates().MultinetEnabled {
Expand Down
6 changes: 6 additions & 0 deletions calico-vpp-agent/cni/network_vpp.go
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,12 @@ err:

// CleanUpVPPNamespace deletes the devices in the network namespace.
func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) {
if s.PodNeedsRedirectToHost(podSpec) {
err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex)
if err != nil {
s.log.Error(err)
}
}
err := ns.IsNSorErr(podSpec.NetnsName)
if err != nil {
s.log.Infof("pod(del) netns '%s' doesn't exist, skipping", podSpec.NetnsName)
Expand Down
17 changes: 17 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ var (
ServiceCIDRs = PrefixListEnvVar("SERVICE_PREFIX")
IPSecIkev2Psk = StringEnvVar("CALICOVPP_IPSEC_IKEV2_PSK", "")
CalicoVppDebug = JsonEnvVar("CALICOVPP_DEBUG", &CalicoVppDebugConfigType{})
CalicoVppSnatExclusionPrefixes = JsonEnvVar("CALICOVPP_SNAT_EXCLUSION_PREFIXES", &SnatExclusionPrefixesConfigType{})
CalicoVppRedirectToHostRules = JsonEnvVar("CALICOVPP_REDIRECT_TO_HOST_RULES", &RedirectToHostRulesConfigType{})
CalicoVppInterfaces = JsonEnvVar("CALICOVPP_INTERFACES", &CalicoVppInterfacesConfigType{})
CalicoVppFeatureGates = JsonEnvVar("CALICOVPP_FEATURE_GATES", &CalicoVppFeatureGatesConfigType{})
CalicoVppIpsec = JsonEnvVar("CALICOVPP_IPSEC", &CalicoVppIpsecConfigType{})
Expand Down Expand Up @@ -158,6 +160,10 @@ func GetCalicoVppFeatureGates() *CalicoVppFeatureGatesConfigType { return *Cal
func GetCalicoVppIpsec() *CalicoVppIpsecConfigType { return *CalicoVppIpsec }
func GetCalicoVppSrv6() *CalicoVppSrv6ConfigType { return *CalicoVppSrv6 }
func GetCalicoVppInitialConfig() *CalicoVppInitialConfigConfigType { return *CalicoVppInitialConfig }
func GetRedirectToHostRules() *RedirectToHostRulesConfigType { return *CalicoVppRedirectToHostRules }
func GetSnatExclusionPrefixes() *SnatExclusionPrefixesConfigType {
return *CalicoVppSnatExclusionPrefixes
}

type InterfaceSpec struct {
NumRxQueues int `json:"rx"`
Expand Down Expand Up @@ -252,6 +258,17 @@ func (u *UplinkInterfaceSpec) String() string {
return string(b)
}

type RedirectToHostRulesConfigType struct {
Port uint16 `json:"port,omitempty"`
Ip string `json:"ip,omitempty"`
/* "tcp", "udp",... */
Proto types.IPProto `json:"proto,omitempty"`
}

type SnatExclusionPrefixesConfigType struct {
Prefix string `json:"prefix,omitempty"`
}

type CalicoVppDebugConfigType struct {
PoliciesEnabled *bool `json:"policiesEnabled,omitempty"`
ServicesEnabled *bool `json:"servicesEnabled,omitempty"`
Expand Down
12 changes: 12 additions & 0 deletions docs/developper_guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,18 @@ export CALICO_ENCAPSULATION_V4=IPIP
export CALICO_ENCAPSULATION_V6=None
export CALICO_NAT_OUTGOING=Enabled
````
To exclude a prefix from SNAT :
````bash
# --------------- snat ----------------
export CALICOVPP_SNAT_PREFIX="\"172.18.0.1\""
````
To add redirection to host rules:
````bash
# --------------- redirect ----------------
export CALICOVPP_REDIRECT_PROTO="\"udp\""
export CALICOVPP_REDIRECT_PORT=53
export CALICOVPP_REDIRECT_IP="\"169.254.0.1\""
````
To run with hugepages on:

````bash
Expand Down
5 changes: 5 additions & 0 deletions vpp-manager/vpp_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,11 @@ func (v *VppRunner) configureVppUplinkInterface(
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
}

err = v.vpp.RegisterHostInterface(tapSwIfIndex)
if err != nil {
return errors.Wrap(err, "error configuring vpptap0 as host intf")
}

// Linux side tap setup
link, err := netlink.LinkByName(ifSpec.InterfaceName)
if err != nil {
Expand Down
140 changes: 140 additions & 0 deletions vpplink/classify.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// Copyright (C) 2023 Cisco Systems Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vpplink

import (
"fmt"

"github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/classify"
"github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/interface_types"
"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
)

type lo uint8

func (v *VppLink) addDelClassifyTable(table *types.ClassifyTable, action types.ClassifyAction) (uint32, error) {
client := classify.NewServiceClient(v.GetConnection())

isAdd, delChain := false, false
currentDataFlags := uint8(0)
tableIndex := types.InvalidTableId
switch action {
case types.AddAbsolute:
isAdd = true
case types.AddRelative:
isAdd = true
currentDataFlags = uint8(1)
case types.Del:
tableIndex = table.TableIndex
case types.DelChain:
tableIndex = table.TableIndex
delChain = true
}

mask := ExtendToVector(table.Mask)

matchNVectors := table.MatchNVectors
if matchNVectors == 0 {
matchNVectors = uint32(len(mask)) / types.VectorSize
}

nBuckets := table.NBuckets
if nBuckets == 0 {
// We provide as many buckets as the max number of entries we expect in the table
nBuckets = table.MaxNumEntries
}

memorySize := table.MemorySize
if memorySize == 0 {
/* memory needed for the table:
* - each entry has a size of (32-bytes + mask vectors)
* - up to 2 entries per page for collision resolution
* - double for margin
*/
memorySize = table.MaxNumEntries * (32 + matchNVectors*types.VectorSize) * 2 * 2
}

response, err := client.ClassifyAddDelTable(v.GetContext(), &classify.ClassifyAddDelTable{
IsAdd: isAdd,
DelChain: delChain,
TableIndex: tableIndex,
Nbuckets: nBuckets,
MemorySize: memorySize,
SkipNVectors: table.SkipNVectors,
MatchNVectors: matchNVectors,
NextTableIndex: table.NextTableIndex,
MissNextIndex: table.MissNextIndex, //ip4-input
CurrentDataFlag: currentDataFlags,
CurrentDataOffset: table.CurrentDataOffset,
MaskLen: uint32(len(mask)),
Mask: mask,
})

if err != nil {
return types.InvalidID, fmt.Errorf("Failed to %s the classify table: %w", map[bool]string{true: "add", false: "del"}[isAdd], err)
}

return response.NewTableIndex, nil
}

func (v *VppLink) AddClassifyTable(table *types.ClassifyTable) (uint32, error) {
return v.addDelClassifyTable(table, types.AddRelative)
}

func (v *VppLink) DelClassifyTable(tableIndex uint32) error {
_, err := v.addDelClassifyTable(&types.ClassifyTable{TableIndex: tableIndex}, types.Del)
return err
}

func ExtendToVector(match []byte) []byte {
n := len(match) % types.VectorSize
if n != 0 {
match = match[:len(match)+types.VectorSize-n]
}
return match
}

func (v *VppLink) SetClassifyInputInterfaceTables(swIfIndex uint32, ip4TableIndex uint32, ip6TableIndex uint32, l2TableIndex uint32, isAdd bool) error {
client := classify.NewServiceClient(v.GetConnection())

_, err := client.InputACLSetInterface(v.GetContext(), &classify.InputACLSetInterface{
IsAdd: isAdd,
SwIfIndex: interface_types.InterfaceIndex(swIfIndex),
IP4TableIndex: ip4TableIndex,
IP6TableIndex: ip6TableIndex,
L2TableIndex: l2TableIndex,
})
if err != nil {
return fmt.Errorf("failed to set input acl tables for this interface: %w", err)
}
return nil
}

func (v *VppLink) SetClassifyOutputInterfaceTables(swIfIndex uint32, ip4TableIndex uint32, ip6TableIndex uint32, l2TableIndex uint32, isAdd bool) error {
client := classify.NewServiceClient(v.GetConnection())

_, err := client.OutputACLSetInterface(v.GetContext(), &classify.OutputACLSetInterface{
IsAdd: isAdd,
SwIfIndex: interface_types.InterfaceIndex(swIfIndex),
IP4TableIndex: ip4TableIndex,
IP6TableIndex: ip6TableIndex,
L2TableIndex: l2TableIndex,
})
if err != nil {
return fmt.Errorf("failed to set input acl tables for this interface: %w", err)
}
return nil
}
8 changes: 8 additions & 0 deletions vpplink/cnat.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,14 @@ func (v *VppLink) RemovePodInterface(swIfIndex uint32) (err error) {
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, false /* isAdd */, cnat.CNAT_POLICY_POD)
}

func (v *VppLink) RegisterHostInterface(swIfIndex uint32) (err error) {
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, true /* isAdd */, cnat.CNAT_POLICY_HOST)
}

func (v *VppLink) RemoveHostInterface(swIfIndex uint32) (err error) {
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, false /* isAdd */, cnat.CNAT_POLICY_HOST)
}

func (v *VppLink) EnableDisableCnatSNAT(swIfIndex uint32, isIp6 bool, isEnable bool) (err error) {
if isEnable {
return v.enableCnatSNAT(swIfIndex, isIp6)
Expand Down
Loading

0 comments on commit 1850f59

Please sign in to comment.