Skip to content

Commit

Permalink
Merge pull request #379 from jacobweinstock/auto-discover-trusted-pro…
Browse files Browse the repository at this point in the history
…xies

Add auto discovery of trusted proxies in Kubernetes, relays and giaddr fix:

## Description

<!--- Please describe what this PR is going to change -->
They will eliminate issues from users having to determine the trusted proxies. We've seen issues where the command we provided in the Helm chart doesn't accurately get the pod CIDRs from a cluster. Incorporating this into the Helm chart was tried, but ultimately not possible without extra ordinary effort and a lot of unneeded code to maintain. This functionality can be disabled if desired. By default we run auto discovery.

This will help simplify the deployment of Smee in the Helm Chart.

Also, move go.mod to Go 1.21. This is because when developing with Go 1.21 and `go mod tidy` will update the `go.mod` file.

Dependency updates for tinkerbell/dhcp provide a fix for dhcp relays and the `giaddr`.

## Why is this needed

<!--- Link to issue you have raised -->

Fixes: #382 

## How Has This Been Tested?
<!--- Please describe in detail how you tested your changes. -->
<!--- Include details of your testing environment, and the tests you ran to -->
<!--- see how your change affects other areas of the code, etc. -->


## How are existing users impacted? What migration steps/scripts do we need?

<!--- Fixes a bug, unblocks installation, removes a component of the stack etc -->
<!--- Requires a DB migration script, etc. -->


## Checklist:

I have:

- [ ] updated the documentation and/or roadmap (if required)
- [ ] added unit or e2e tests
- [ ] provided instructions on how to upgrade
  • Loading branch information
jacobweinstock authored Dec 15, 2023
2 parents 5d3ede1 + 4bdee41 commit ccc2c65
Show file tree
Hide file tree
Showing 8 changed files with 401 additions and 28 deletions.
98 changes: 96 additions & 2 deletions cmd/smee/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,15 @@ package main

import (
"context"
"strings"

"github.com/go-logr/logr"
"github.com/tinkerbell/dhcp/backend/file"
"github.com/tinkerbell/dhcp/backend/kube"
"github.com/tinkerbell/dhcp/handler"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)
Expand All @@ -27,7 +31,7 @@ type File struct {
Enabled bool
}

func (k *Kube) Backend(ctx context.Context) (handler.BackendReader, error) {
func (k *Kube) getClient() (*rest.Config, error) {
ccfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{
ExplicitPath: k.ConfigFilePath,
Expand All @@ -47,6 +51,15 @@ func (k *Kube) Backend(ctx context.Context) (handler.BackendReader, error) {
return nil, err
}

return config, nil
}

func (k *Kube) backend(ctx context.Context) (handler.BackendReader, error) {
config, err := k.getClient()
if err != nil {
return nil, err
}

kb, err := kube.NewBackend(config)
if err != nil {
return nil, err
Expand All @@ -62,7 +75,7 @@ func (k *Kube) Backend(ctx context.Context) (handler.BackendReader, error) {
return kb, nil
}

func (s *File) Backend(ctx context.Context, logger logr.Logger) (handler.BackendReader, error) {
func (s *File) backend(ctx context.Context, logger logr.Logger) (handler.BackendReader, error) {
f, err := file.NewWatcher(logger, s.FilePath)
if err != nil {
return nil, err
Expand All @@ -72,3 +85,84 @@ func (s *File) Backend(ctx context.Context, logger logr.Logger) (handler.Backend

return f, nil
}

// discoverTrustedProxies will use the Kubernetes client to discover the CIDR Ranges for Pods in cluster.
func (k *Kube) discoverTrustedProxies(ctx context.Context, l logr.Logger, trustedProxies []string) []string {
config, err := k.getClient()
if err != nil {
l.Error(err, "failed to get Kubernetes client config")
return nil
}
c, err := corev1client.NewForConfig(config)
if err != nil {
l.Error(err, "failed to create Kubernetes client")
return nil
}

return combinedCIDRs(ctx, l, c, trustedProxies)
}

// combinedCIDRs returns the CIDR Ranges for Pods in cluster. Not all Kubernetes distributions provide a way to discover the entire podCIDR.
// Some distributions just provide the podCIDRs assigned to each node. combinedCIDRs tries all known locations where pod CIDRs might exist.
// For example, if a cluster has 3 nodes, each with a /24 podCIDR, and the cluster has a /16 podCIDR, combinedCIDRs will return 4 CIDR ranges.
func combinedCIDRs(ctx context.Context, l logr.Logger, c corev1client.CoreV1Interface, trustedProxies []string) []string {
var tp []string
tp = append(tp, trustedProxies...)
if podCIDRS, err := perNodePodCIDRs(ctx, c); err == nil {
tp = append(tp, podCIDRS...)
} else {
l.V(1).Info("failed to get per node podCIDRs", "err", err)
}

if clusterCIDR, err := clusterPodCIDR(ctx, c); err == nil {
tp = append(tp, clusterCIDR...)
} else {
l.V(1).Info("failed to get cluster wide podCIDR", "err", err)
}

return tp
}

// perNodePodCIDRs returns the CIDR Range for Pods on each node. This is the per node podCIDR as compared to the total podCIDR.
// This will get the podCIDR from each node in the cluster, not the entire cluster podCIDR. If a cluster grows after this is run,
// the new nodes will not be included until this func is run again.
// This should be used in conjunction with ClusterPodCIDR to be as complete and cross distribution compatible as possible.
func perNodePodCIDRs(ctx context.Context, c corev1client.CoreV1Interface) ([]string, error) {
ns, err := c.Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
}

var trustedProxies []string
for _, n := range ns.Items {
trustedProxies = append(trustedProxies, n.Spec.PodCIDRs...)
}

return trustedProxies, nil
}

// clusterPodCIDR returns the CIDR Range for Pods in cluster. This is the total podCIDR as compared to the per node podCIDR.
// Some Kubernetes distributions do not run a kube-controller-manager pod, so this func should be used in conjunction with PerNodePodCIDRs
// to be as complete and cross distribution compatible as possible.
func clusterPodCIDR(ctx context.Context, c corev1client.CoreV1Interface) ([]string, error) {
// https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/
pods, err := c.Pods("kube-system").List(ctx, metav1.ListOptions{
LabelSelector: "component=kube-controller-manager",
})
if err != nil {
return nil, err
}

var trustedProxies []string
for _, p := range pods.Items {
for _, c := range p.Spec.Containers {
for _, e := range c.Command {
if strings.HasPrefix(e, "--cluster-cidr") {
trustedProxies = append(trustedProxies, strings.Split(e, "=")[1])
}
}
}
}

return trustedProxies, nil
}
251 changes: 251 additions & 0 deletions cmd/smee/backend_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
package main

import (
"context"
"testing"

"github.com/go-logr/logr"
"github.com/google/go-cmp/cmp"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
)

func TestClusterPodCIDR(t *testing.T) {
tests := map[string]struct {
spec []runtime.Object
want []string
}{
"no podCIDR": {},
"podCIDR": {
spec: []runtime.Object{
&v1.PodList{
Items: []v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"component": "kube-controller-manager",
},
Namespace: "kube-system",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Command: []string{
"kube-controller-manager",
"--allocate-node-cidrs=true",
"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
"--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf",
"--bind-address=127.0.0.1",
"--client-ca-file=/etc/kubernetes/pki/ca.crt",
"--cluster-cidr=10.244.0.0/16",
"--cluster-name=kubernetes",
},
},
},
},
},
},
},
},
want: []string{"10.244.0.0/16"},
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
c := fake.NewSimpleClientset(test.spec...)
got, err := clusterPodCIDR(context.Background(), c.CoreV1())
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(got, test.want); diff != "" {
t.Fatalf("unexpected result (+want -got):\n%s", diff)
}
})
}
}

func TestPerNodePodCIDRs(t *testing.T) {
tests := map[string]struct {
spec []runtime.Object
want []string
}{
"no podCIDR": {},
"podCIDR": {
spec: []runtime.Object{
&v1.NodeList{
Items: []v1.Node{
{
ObjectMeta: metav1.ObjectMeta{
Name: "node1",
},
Spec: v1.NodeSpec{
PodCIDRs: []string{"10.42.0.0/24"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node2",
},
Spec: v1.NodeSpec{
PodCIDRs: []string{"10.42.1.0/24"},
},
},
},
},
},
want: []string{"10.42.0.0/24", "10.42.1.0/24"},
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
c := fake.NewSimpleClientset(test.spec...)
got, err := perNodePodCIDRs(context.Background(), c.CoreV1())
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(got, test.want); diff != "" {
t.Fatalf("unexpected result (+want -got):\n%s", diff)
}
})
}
}

func TestCombinedCIDRs(t *testing.T) {
tests := map[string]struct {
spec []runtime.Object
want []string
}{
"no podCIDR": {},
"podCIDR": {
spec: []runtime.Object{
&v1.NodeList{
Items: []v1.Node{
{
ObjectMeta: metav1.ObjectMeta{
Name: "node1",
},
Spec: v1.NodeSpec{
PodCIDRs: []string{"10.42.0.0/24"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node2",
},
Spec: v1.NodeSpec{
PodCIDRs: []string{"10.42.1.0/24"},
},
},
},
},
&v1.PodList{
Items: []v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"component": "kube-controller-manager",
},
Namespace: "kube-system",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Command: []string{
"kube-controller-manager",
"--allocate-node-cidrs=true",
"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
"--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf",
"--bind-address=127.0.0.1",
"--client-ca-file=/etc/kubernetes/pki/ca.crt",
"--cluster-cidr=10.244.0.0/16",
"--cluster-name=kubernetes",
},
},
},
},
},
},
},
},
want: []string{"10.42.0.0/24", "10.42.1.0/24", "10.244.0.0/16"},
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
c := fake.NewSimpleClientset(test.spec...)
got := combinedCIDRs(context.Background(), logr.Discard(), c.CoreV1(), nil)
if diff := cmp.Diff(got, test.want); diff != "" {
t.Fatalf("unexpected result (+want -got):\n%s", diff)
}
})
}
}

func TestDiscoverTrustedProxies(t *testing.T) {
t.Skip("dont think i can mock this")
tests := map[string]struct {
spec []runtime.Object
want []string
}{
"no podCIDR": {},
"podCIDR": {
spec: []runtime.Object{
&v1.NodeList{
Items: []v1.Node{
{
ObjectMeta: metav1.ObjectMeta{
Name: "node1",
},
Spec: v1.NodeSpec{
PodCIDRs: []string{"10.42.0.0/24"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node2",
},
Spec: v1.NodeSpec{
PodCIDRs: []string{"10.42.1.0/24"},
},
},
},
},
&v1.PodList{
Items: []v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"component": "kube-controller-manager",
},
Namespace: "kube-system",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Command: []string{
"kube-controller-manager",
"--allocate-node-cidrs=true",
"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
},
},
},
},
},
},
},
},
want: []string{"10.42.1.0/24", "10.42.0.0/24"},
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
k := &Kube{}
got := k.discoverTrustedProxies(context.Background(), logr.Discard(), nil)
if diff := cmp.Diff(got, test.want); diff != "" {
t.Fatalf("unexpected result (+want -got):\n%s", diff)
}
})
}
}
1 change: 1 addition & 0 deletions cmd/smee/flag.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ func ipxeHTTPScriptFlags(c *config, fs *flag.FlagSet) {
fs.StringVar(&c.ipxeHTTPScript.bindAddr, "http-addr", detectPublicIPv4(":80"), "[http] local IP:Port to listen on for iPXE HTTP script requests")
fs.StringVar(&c.ipxeHTTPScript.extraKernelArgs, "extra-kernel-args", "", "[http] extra set of kernel args (k=v k=v) that are appended to the kernel cmdline iPXE script")
fs.StringVar(&c.ipxeHTTPScript.trustedProxies, "trusted-proxies", "", "[http] comma separated list of trusted proxies in CIDR notation")
fs.BoolVar(&c.ipxeHTTPScript.disableDiscoverTrustedProxies, "disable-discover-trusted-proxies", false, "[http] disable discovery of trusted proxies from Kubernetes, only available for the Kubernetes backend")
fs.StringVar(&c.ipxeHTTPScript.hookURL, "osie-url", "", "[http] URL where OSIE (HookOS) images are located")
fs.StringVar(&c.ipxeHTTPScript.tinkServer, "tink-server", "", "[http] IP:Port for the Tink server")
fs.BoolVar(&c.ipxeHTTPScript.tinkServerUseTLS, "tink-server-tls", false, "[http] use TLS for Tink server")
Expand Down
Loading

0 comments on commit ccc2c65

Please sign in to comment.