From 5b0442445922b3da35b66d9ec1843e1e73faeb4c Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Sun, 25 Jun 2023 17:00:22 +0200 Subject: [PATCH 1/5] Add a flowtable to speed up forwarding --- pkg/nftables/nftables.tpl | 14 ++++++++++++++ pkg/nftables/rendering.go | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pkg/nftables/nftables.tpl b/pkg/nftables/nftables.tpl index 5581c822..e8c37792 100644 --- a/pkg/nftables/nftables.tpl +++ b/pkg/nftables/nftables.tpl @@ -35,9 +35,23 @@ table inet firewall { counter drop_total { } counter drop_ratelimit { } + # create a flowtable for all interfaces + flowtable f { + hook ingress priority 1; devices = { + {{- range .VrfIDs }} + "vlan{{ . }}", + "vrf{{ . }}, + {{- end }} + }; + } + chain forward { type filter hook forward priority 1; policy drop; + # offload established connections + ip protocol { tcp, udp } flow offload @f + ip6 nexthdr { tcp, udp } flow offload @f + # network traffic accounting for external traffic ip saddr != @internal_prefixes oifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name external_in comment "count external traffic incomming" ip daddr != @internal_prefixes iifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name external_out comment "count external traffic outgoing" diff --git a/pkg/nftables/rendering.go b/pkg/nftables/rendering.go index 8f75caba..6667ad13 100644 --- a/pkg/nftables/rendering.go +++ b/pkg/nftables/rendering.go @@ -19,6 +19,7 @@ type firewallRenderingData struct { Sets []dns.RenderIPSet InternalPrefixes string PrivateVrfID uint + VrfIDs []int64 } func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) { @@ -48,6 +49,16 @@ func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) { if f.cache.IsInitialized() { sets = f.cache.GetSetsForRendering(f.clusterwideNetworkPolicies.GetFQDNs()) } + + var vrfIDs []int64 + for _, nw := range f.networkMap { + nw := nw + if nw.Vrf == nil { + continue + } + vrfIDs = append(vrfIDs, *nw.Vrf) + } + return &firewallRenderingData{ PrivateVrfID: uint(*f.primaryPrivateNet.Vrf), InternalPrefixes: strings.Join(f.firewall.Spec.InternalPrefixes, ", "), @@ -58,6 +69,7 @@ func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) { RateLimitRules: rateLimitRules(f), SnatRules: snatRules, Sets: sets, + VrfIDs: vrfIDs, }, nil } From 4752ecbbb10ed9ce3f9b668a7b70e5d70f3b9740 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Sun, 25 Jun 2023 17:18:47 +0200 Subject: [PATCH 2/5] Add test --- pkg/nftables/nftables.tpl | 8 ++++++-- pkg/nftables/rendering.go | 8 +++++--- pkg/nftables/rendering_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/pkg/nftables/nftables.tpl b/pkg/nftables/nftables.tpl index e8c37792..b6067703 100644 --- a/pkg/nftables/nftables.tpl +++ b/pkg/nftables/nftables.tpl @@ -35,22 +35,26 @@ table inet firewall { counter drop_total { } counter drop_ratelimit { } + {{- if gt (len .VrfIDs) 0 }} # create a flowtable for all interfaces flowtable f { - hook ingress priority 1; devices = { + hook ingress priority 1; devices = { {{- range .VrfIDs }} "vlan{{ . }}", - "vrf{{ . }}, + "vrf{{ . }}", {{- end }} }; } + {{- end }} chain forward { type filter hook forward priority 1; policy drop; + {{- if gt (len .VrfIDs) 0 }} # offload established connections ip protocol { tcp, udp } flow offload @f ip6 nexthdr { tcp, udp } flow offload @f + {{- end }} # network traffic accounting for external traffic ip saddr != @internal_prefixes oifname {"vlan{{ .PrivateVrfID }}", "vrf{{ .PrivateVrfID }}"} counter name external_in comment "count external traffic incomming" diff --git a/pkg/nftables/rendering.go b/pkg/nftables/rendering.go index 6667ad13..be8dda65 100644 --- a/pkg/nftables/rendering.go +++ b/pkg/nftables/rendering.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "sort" "strings" "text/template" @@ -19,7 +20,7 @@ type firewallRenderingData struct { Sets []dns.RenderIPSet InternalPrefixes string PrivateVrfID uint - VrfIDs []int64 + VrfIDs []int } func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) { @@ -50,14 +51,15 @@ func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) { sets = f.cache.GetSetsForRendering(f.clusterwideNetworkPolicies.GetFQDNs()) } - var vrfIDs []int64 + var vrfIDs []int for _, nw := range f.networkMap { nw := nw if nw.Vrf == nil { continue } - vrfIDs = append(vrfIDs, *nw.Vrf) + vrfIDs = append(vrfIDs, int(*nw.Vrf)) } + sort.Ints(vrfIDs) return &firewallRenderingData{ PrivateVrfID: uint(*f.primaryPrivateNet.Vrf), diff --git a/pkg/nftables/rendering_test.go b/pkg/nftables/rendering_test.go index 8c73ffb4..6bdc5715 100644 --- a/pkg/nftables/rendering_test.go +++ b/pkg/nftables/rendering_test.go @@ -84,6 +84,33 @@ func TestFirewallRenderingData_renderString(t *testing.T) { }, wantErr: false, }, + { + name: "flowtable", + data: &firewallRenderingData{ + ForwardingRules: forwardingRules{ + Egress: []string{"egress rule"}, + Ingress: []string{"ingress rule"}, + }, + InternalPrefixes: "1.2.3.4", + RateLimitRules: []string{"meta iifname \"eth0\" limit rate over 10 mbytes/second counter name drop_ratelimit drop"}, + SnatRules: []string{}, + PrivateVrfID: uint(42), + Sets: []dns.RenderIPSet{ + { + SetName: "test", + IPs: []string{"10.0.0.1", "10.0.0.2"}, + Version: dns.IPv4, + }, + { + SetName: "test2", + IPs: []string{"2001:db8:85a3::8a2e:370:7334", "2001:db8:85a3::8a2e:370:7335"}, + Version: dns.IPv6, + }, + }, + VrfIDs: []int{60, 90, 104009}, + }, + wantErr: false, + }, } for _, tt := range tests { tt := tt From ff1dbd8a74530f71087217ff0b70cffe3b47aede Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Sun, 25 Jun 2023 17:19:02 +0200 Subject: [PATCH 3/5] Add test --- pkg/nftables/test_data/flowtable.nftable.v4 | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 pkg/nftables/test_data/flowtable.nftable.v4 diff --git a/pkg/nftables/test_data/flowtable.nftable.v4 b/pkg/nftables/test_data/flowtable.nftable.v4 new file mode 100644 index 00000000..2271a72b --- /dev/null +++ b/pkg/nftables/test_data/flowtable.nftable.v4 @@ -0,0 +1,88 @@ +table inet firewall { + # internal prefixes, which are not leaving the partition or the partition interconnect + set internal_prefixes { + type ipv4_addr + flags interval + auto-merge + + elements = { 1.2.3.4 } + + } + + # Prefixes in the cluster, typically 10.x.x.x + # FIXME Should be filled with nodeCidr + set cluster_prefixes { + type ipv4_addr + flags interval + auto-merge + elements = { 10.0.0.0/8 } + } + + set test { + type ipv4_addr + + elements = { 10.0.0.1, 10.0.0.2 } + + } + + set test2 { + type ipv6_addr + + elements = { 2001:db8:85a3::8a2e:370:7334, 2001:db8:85a3::8a2e:370:7335 } + + } + + # counters + counter internal_in { } + counter internal_out { } + counter external_in { } + counter external_out { } + counter drop_total { } + counter drop_ratelimit { } + # create a flowtable for all interfaces + flowtable f { + hook ingress priority 1; devices = { + "vlan60", + "vrf60", + "vlan90", + "vrf90", + "vlan104009", + "vrf104009", + }; + } + + chain forward { + type filter hook forward priority 1; policy drop; + # offload established connections + ip protocol { tcp, udp } flow offload @f + ip6 nexthdr { tcp, udp } flow offload @f + + # network traffic accounting for external traffic + ip saddr != @internal_prefixes oifname {"vlan42", "vrf42"} counter name external_in comment "count external traffic incomming" + ip daddr != @internal_prefixes iifname {"vlan42", "vrf42"} counter name external_out comment "count external traffic outgoing" + + # network traffic accounting for internal traffic + ip saddr @internal_prefixes oifname {"vlan42", "vrf42"} counter name internal_in comment "count internal traffic incomming" + ip daddr @internal_prefixes iifname {"vlan42", "vrf42"} counter name internal_out comment "count internal traffic outgoing" + + # rate limits + meta iifname "eth0" limit rate over 10 mbytes/second counter name drop_ratelimit drop + + # state dependent rules + ct state established,related counter accept comment "accept established connections" + ct state invalid counter drop comment "drop packets with invalid ct state" + + # icmp + ip protocol icmp icmp type echo-request limit rate over 10/second burst 4 packets counter drop comment "drop ping floods" + ip protocol icmp icmp type { destination-unreachable, router-solicitation, router-advertisement, time-exceeded, parameter-problem } counter log prefix "nftables-firewall-accepted: " accept comment "accept icmp" + + # dynamic ingress rules + ingress rule + + # dynamic egress rules + egress rule + + counter comment "count and log dropped packets" + limit rate 10/second counter name drop_total log prefix "nftables-firewall-dropped: " + } +} From d088fd2fee6ab7921d0b0d933db6015cb1e451cb Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Sun, 25 Jun 2023 17:48:14 +0200 Subject: [PATCH 4/5] fix devices --- pkg/nftables/nftables.tpl | 7 +------ pkg/nftables/rendering.go | 9 ++++----- pkg/nftables/rendering_test.go | 2 +- pkg/nftables/test_data/flowtable.nftable.v4 | 9 +-------- 4 files changed, 7 insertions(+), 20 deletions(-) diff --git a/pkg/nftables/nftables.tpl b/pkg/nftables/nftables.tpl index b6067703..21a744bd 100644 --- a/pkg/nftables/nftables.tpl +++ b/pkg/nftables/nftables.tpl @@ -38,12 +38,7 @@ table inet firewall { {{- if gt (len .VrfIDs) 0 }} # create a flowtable for all interfaces flowtable f { - hook ingress priority 1; devices = { - {{- range .VrfIDs }} - "vlan{{ . }}", - "vrf{{ . }}", - {{- end }} - }; + hook ingress priority 1; devices = { {{ StringsJoin .VrfIDs ", " }} }; } {{- end }} diff --git a/pkg/nftables/rendering.go b/pkg/nftables/rendering.go index be8dda65..7c824221 100644 --- a/pkg/nftables/rendering.go +++ b/pkg/nftables/rendering.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "os" - "sort" "strings" "text/template" @@ -20,7 +19,7 @@ type firewallRenderingData struct { Sets []dns.RenderIPSet InternalPrefixes string PrivateVrfID uint - VrfIDs []int + VrfIDs []string } func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) { @@ -51,15 +50,15 @@ func newFirewallRenderingData(f *Firewall) (*firewallRenderingData, error) { sets = f.cache.GetSetsForRendering(f.clusterwideNetworkPolicies.GetFQDNs()) } - var vrfIDs []int + var vrfIDs []string for _, nw := range f.networkMap { nw := nw if nw.Vrf == nil { continue } - vrfIDs = append(vrfIDs, int(*nw.Vrf)) + vrfIDs = append(vrfIDs, fmt.Sprintf("vlan%d", *nw.Vrf)) + vrfIDs = append(vrfIDs, fmt.Sprintf("vrf%d", *nw.Vrf)) } - sort.Ints(vrfIDs) return &firewallRenderingData{ PrivateVrfID: uint(*f.primaryPrivateNet.Vrf), diff --git a/pkg/nftables/rendering_test.go b/pkg/nftables/rendering_test.go index 6bdc5715..7c8f757c 100644 --- a/pkg/nftables/rendering_test.go +++ b/pkg/nftables/rendering_test.go @@ -107,7 +107,7 @@ func TestFirewallRenderingData_renderString(t *testing.T) { Version: dns.IPv6, }, }, - VrfIDs: []int{60, 90, 104009}, + VrfIDs: []string{"vlan60", "vrf60", "vlan90", "vrf90", "vlan104009", "vrf104009"}, }, wantErr: false, }, diff --git a/pkg/nftables/test_data/flowtable.nftable.v4 b/pkg/nftables/test_data/flowtable.nftable.v4 index 2271a72b..0e1bedcc 100644 --- a/pkg/nftables/test_data/flowtable.nftable.v4 +++ b/pkg/nftables/test_data/flowtable.nftable.v4 @@ -41,14 +41,7 @@ table inet firewall { counter drop_ratelimit { } # create a flowtable for all interfaces flowtable f { - hook ingress priority 1; devices = { - "vlan60", - "vrf60", - "vlan90", - "vrf90", - "vlan104009", - "vrf104009", - }; + hook ingress priority 1; devices = { vlan60, vrf60, vlan90, vrf90, vlan104009, vrf104009 }; } chain forward { From 82dc1424b3331c9728bce7c9d7542dc32e0da1b4 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Mon, 26 Jun 2023 13:25:21 +0200 Subject: [PATCH 5/5] counter --- pkg/nftables/nftables.tpl | 2 +- pkg/nftables/test_data/flowtable.nftable.v4 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/nftables/nftables.tpl b/pkg/nftables/nftables.tpl index 21a744bd..85e193ae 100644 --- a/pkg/nftables/nftables.tpl +++ b/pkg/nftables/nftables.tpl @@ -38,7 +38,7 @@ table inet firewall { {{- if gt (len .VrfIDs) 0 }} # create a flowtable for all interfaces flowtable f { - hook ingress priority 1; devices = { {{ StringsJoin .VrfIDs ", " }} }; + hook ingress priority 1; devices = { {{ StringsJoin .VrfIDs ", " }} }; counter; } {{- end }} diff --git a/pkg/nftables/test_data/flowtable.nftable.v4 b/pkg/nftables/test_data/flowtable.nftable.v4 index 0e1bedcc..adf10a23 100644 --- a/pkg/nftables/test_data/flowtable.nftable.v4 +++ b/pkg/nftables/test_data/flowtable.nftable.v4 @@ -41,7 +41,7 @@ table inet firewall { counter drop_ratelimit { } # create a flowtable for all interfaces flowtable f { - hook ingress priority 1; devices = { vlan60, vrf60, vlan90, vrf90, vlan104009, vrf104009 }; + hook ingress priority 1; devices = { vlan60, vrf60, vlan90, vrf90, vlan104009, vrf104009 }; counter; } chain forward {