From f1f1a5f1bd34be3597c24768ebc1a0818e76ed7b Mon Sep 17 00:00:00 2001 From: Rishabh Soni Date: Tue, 22 Oct 2024 15:34:19 +0530 Subject: [PATCH 1/4] feat(operator): fetch cluster name from providers Signed-off-by: Rishabh Soni --- .../internal/controller/resources.go | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index e848be8333..8dbb6a29dd 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -7,6 +7,8 @@ import ( "bytes" "context" "fmt" + "io" + "net/http" "strings" "time" @@ -469,6 +471,90 @@ func (clusterWatcher *ClusterWatcher) deployControllerDeployment(deployment *app return nil } +func (clusterWatcher *ClusterWatcher) getProvider() string { + + nodes, err := clusterWatcher.Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + clusterWatcher.Log.Warnf("Error listing nodes: %s\n", err.Error()) + } + + for _, node := range nodes.Items { + for key, label := range node.Labels { + if strings.Contains(key, "gke") || strings.Contains(label, "gke") { + return "gke" + } else if strings.Contains(key, "eks") || strings.Contains(label, "eks") { + return "eks" + } else if strings.Contains(key, "aks") || strings.Contains(label, "aks") { + return "aks" + } + } + } + return "default" +} + +func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle() string { + url := "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + clusterWatcher.Log.Warnf("failed to create request: %w", err) + return "" + } + + // Set the required header + req.Header.Set("Metadata-Flavor", "Google") + + // Create an HTTP client and make the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + clusterWatcher.Log.Warnf("error making request: %w", err) + return "" + } + defer resp.Body.Close() + + // Check for a successful response + if resp.StatusCode != http.StatusOK { + clusterWatcher.Log.Warnf("failed to fetch from metadata, status code: %d", resp.StatusCode) + return "" + } + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + clusterWatcher.Log.Warnf("error reading response body: %w", err) + return "" + } + + return string(body) +} + +func (clusterWatcher *ClusterWatcher) getClusterName() string { + provider := clusterWatcher.getProvider() + if provider == "gke" { + clusterWatcher.Log.Infof("Provider is GKE") + if clusterName := clusterWatcher.fetchClusterNameFromGoogle(); clusterName == "" { + clusterWatcher.Log.Warnf("Cannot fetch cluster name for GKE") + } else { + return clusterName + } + } + // } else if provider == "eks" { + // if clusterName, err := fetchClusterNameFromAWS(); err != nil { + // clusterWatcher.Log.Warnf("Cannot fetch cluster name %s, err.Error()") + // } else { + // return clusterName + // } + // } + // else if provider == "aks" { + // if clusterName, err := fetchClusterNameFromAzure(); err != nil { + // clusterWatcher.Log.Warnf("Cannot fetch cluster name %s, err.Error()") + // } else { + // return clusterName + // } + // } + return provider +} + func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { var caCert, tlsCrt, tlsKey *bytes.Buffer var kGenErr, err, installErr error @@ -564,6 +650,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { // kubearmor configmap configmap := addOwnership(deployments.GetKubearmorConfigMap(common.Namespace, deployments.KubeArmorConfigMapName)).(*corev1.ConfigMap) configmap.Data = common.ConfigMapData + configmap.Data["cluster"] = clusterWatcher.getClusterName() for { caCert, tlsCrt, tlsKey, kGenErr = common.GeneratePki(common.Namespace, deployments.KubeArmorControllerWebhookServiceName) From 209360bd6c5c1afcfa45cf2ce47fb88e4e528286 Mon Sep 17 00:00:00 2001 From: Rishabh Soni Date: Wed, 23 Oct 2024 11:32:08 +0530 Subject: [PATCH 2/4] Add support for EKS Signed-off-by: Rishabh Soni --- .../internal/controller/resources.go | 99 +++++++++++++++---- 1 file changed, 81 insertions(+), 18 deletions(-) diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index 8dbb6a29dd..261e1fc716 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "regexp" "strings" "time" @@ -472,7 +473,6 @@ func (clusterWatcher *ClusterWatcher) deployControllerDeployment(deployment *app } func (clusterWatcher *ClusterWatcher) getProvider() string { - nodes, err := clusterWatcher.Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) if err != nil { clusterWatcher.Log.Warnf("Error listing nodes: %s\n", err.Error()) @@ -492,12 +492,12 @@ func (clusterWatcher *ClusterWatcher) getProvider() string { return "default" } -func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle() string { +func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle() (string, error) { url := "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name" req, err := http.NewRequest("GET", url, nil) if err != nil { clusterWatcher.Log.Warnf("failed to create request: %w", err) - return "" + return "", err } // Set the required header @@ -508,51 +508,114 @@ func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle() string { resp, err := client.Do(req) if err != nil { clusterWatcher.Log.Warnf("error making request: %w", err) - return "" + return "", err } defer resp.Body.Close() // Check for a successful response if resp.StatusCode != http.StatusOK { clusterWatcher.Log.Warnf("failed to fetch from metadata, status code: %d", resp.StatusCode) - return "" + return "", err } // Read the response body body, err := io.ReadAll(resp.Body) if err != nil { clusterWatcher.Log.Warnf("error reading response body: %w", err) - return "" + return "", err + } + + return string(body), nil +} + +func (clusterWatcher *ClusterWatcher) fetchClusterNameFromAWS() (string, error) { + var token []byte + client := &http.Client{Timeout: 2 * time.Second} + req, err := http.NewRequest("PUT", "http://169.254.169.254/latest/api/token", nil) + if err != nil { + clusterWatcher.Log.Warnf("failed to create request for fetching token: %w", err) + return "", err + } + req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "21600") + + resp, err := client.Do(req) + if err != nil { + clusterWatcher.Log.Warnf("error making request: %w", err) + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + token, err = io.ReadAll(resp.Body) + if err != nil { + clusterWatcher.Log.Warnf("failed to read token: %d", err) + return "", err + } + } + + // Fetch the EKS cluster name from user data + req, err = http.NewRequest("GET", "http://169.254.169.254/latest/user-data", nil) + client = &http.Client{Timeout: 2 * time.Second} + if err != nil { + clusterWatcher.Log.Warnf("failed to create request for fetching metadata: %w", err) + return "", err + } + req.Header.Set("X-aws-ec2-metadata-token", string(token)) + + resp, err = client.Do(req) + if err != nil { + clusterWatcher.Log.Warnf("error making request: %w", err) + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + clusterWatcher.Log.Warnf("failed to fetch from metadata, status code: %d", resp.StatusCode) + return "", err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + clusterWatcher.Log.Warnf("failed to read metadata: %d", err) + return "", err + } + + // Extract EKS cluster name + re := regexp.MustCompile(`/etc/eks/bootstrap\.sh (\S+)`) + match := re.FindStringSubmatch(string(body)) + if len(match) > 0 { + return match[1], nil } - return string(body) + return "", err } func (clusterWatcher *ClusterWatcher) getClusterName() string { provider := clusterWatcher.getProvider() if provider == "gke" { clusterWatcher.Log.Infof("Provider is GKE") - if clusterName := clusterWatcher.fetchClusterNameFromGoogle(); clusterName == "" { - clusterWatcher.Log.Warnf("Cannot fetch cluster name for GKE") + if clusterName, err := clusterWatcher.fetchClusterNameFromGoogle(); err != nil { + clusterWatcher.Log.Warnf("Cannot fetch cluster name for GKE %s", err.Error()) + } else { + return clusterName + } + } else if provider == "eks" { + clusterWatcher.Log.Infof("Provider is EKS") + if clusterName, err := clusterWatcher.fetchClusterNameFromAWS(); err != nil { + clusterWatcher.Log.Warnf("Cannot fetch cluster name for EKS %s", err.Error()) } else { return clusterName } } - // } else if provider == "eks" { - // if clusterName, err := fetchClusterNameFromAWS(); err != nil { - // clusterWatcher.Log.Warnf("Cannot fetch cluster name %s, err.Error()") - // } else { - // return clusterName - // } - // } - // else if provider == "aks" { + // } else if provider == "aks" { // if clusterName, err := fetchClusterNameFromAzure(); err != nil { // clusterWatcher.Log.Warnf("Cannot fetch cluster name %s, err.Error()") // } else { // return clusterName // } // } - return provider + + return "default" } func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { From 6a750ae8d4285feed93988c3987484e145e658a2 Mon Sep 17 00:00:00 2001 From: Rishabh Soni Date: Tue, 29 Oct 2024 10:37:14 +0530 Subject: [PATCH 3/4] Remove AKS reference Signed-off-by: Rishabh Soni --- pkg/KubeArmorOperator/internal/controller/resources.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index 261e1fc716..8a32413130 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -484,8 +484,6 @@ func (clusterWatcher *ClusterWatcher) getProvider() string { return "gke" } else if strings.Contains(key, "eks") || strings.Contains(label, "eks") { return "eks" - } else if strings.Contains(key, "aks") || strings.Contains(label, "aks") { - return "aks" } } } @@ -607,13 +605,6 @@ func (clusterWatcher *ClusterWatcher) getClusterName() string { return clusterName } } - // } else if provider == "aks" { - // if clusterName, err := fetchClusterNameFromAzure(); err != nil { - // clusterWatcher.Log.Warnf("Cannot fetch cluster name %s, err.Error()") - // } else { - // return clusterName - // } - // } return "default" } From ad84c517c78c23f9612e1043ef7def64a4cf932c Mon Sep 17 00:00:00 2001 From: Rishabh Soni Date: Thu, 14 Nov 2024 15:54:51 +0530 Subject: [PATCH 4/4] Make provider hostname and endpoint configurable Signed-off-by: Rishabh Soni --- pkg/KubeArmorOperator/cmd/operator/main.go | 5 +- .../internal/controller/cluster.go | 6 ++- .../internal/controller/resources.go | 53 ++++++++++++------- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/pkg/KubeArmorOperator/cmd/operator/main.go b/pkg/KubeArmorOperator/cmd/operator/main.go index 95f87fda88..8505573da8 100644 --- a/pkg/KubeArmorOperator/cmd/operator/main.go +++ b/pkg/KubeArmorOperator/cmd/operator/main.go @@ -30,6 +30,7 @@ var ExtClient *apiextensionsclientset.Clientset var Opv1Client *opv1client.Clientset var InitDeploy bool var LogLevel string +var ProviderHostname, ProviderEndpoint string // Cmd represents the base command when called without any subcommands var Cmd = &cobra.Command{ @@ -52,7 +53,7 @@ var Cmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, PathPrefix, DeploymentName, InitDeploy) + nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, PathPrefix, DeploymentName, ProviderHostname, ProviderEndpoint, InitDeploy) go nodeWatcher.WatchConfigCrd() nodeWatcher.WatchNodes() @@ -78,6 +79,8 @@ func init() { Cmd.PersistentFlags().StringVar(&LsmOrder, "lsm", "bpf,apparmor,selinux", "lsm preference order to use") Cmd.PersistentFlags().StringVar(&PathPrefix, "pathprefix", "/rootfs/", "path prefix for runtime search") Cmd.PersistentFlags().StringVar(&DeploymentName, "deploymentName", "kubearmor-operator", "operator deployment name") + Cmd.PersistentFlags().StringVar(&ProviderHostname, "providerHostname", "", "IMDS URL hostname for retrieving cluster name") + Cmd.PersistentFlags().StringVar(&ProviderEndpoint, "providerEndpoint", "", "IMDS URL endpoint for retrieving cluster name") // TODO:- set initDeploy to false by default once this change is added to stable Cmd.PersistentFlags().BoolVar(&InitDeploy, "initDeploy", true, "Init container deployment") Cmd.PersistentFlags().StringVar(&LogLevel, "loglevel", "info", "log level, e.g., debug, info, warn, error") diff --git a/pkg/KubeArmorOperator/internal/controller/cluster.go b/pkg/KubeArmorOperator/internal/controller/cluster.go index fd1e5af0af..11787b65fa 100644 --- a/pkg/KubeArmorOperator/internal/controller/cluster.go +++ b/pkg/KubeArmorOperator/internal/controller/cluster.go @@ -38,6 +38,7 @@ var deployment_uuid types.UID var deployment_name string = "kubearmor-operator" var PathPrefix string var initDeploy bool +var ProviderHostname, ProviderEndpoint string type ClusterWatcher struct { Nodes []Node @@ -60,7 +61,7 @@ type Node struct { Seccomp string } -func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, pathPrefix, deploy_name string, initdeploy bool) *ClusterWatcher { +func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, pathPrefix, deploy_name, providerHostname, providerEndpoint string, initdeploy bool) *ClusterWatcher { if informer == nil { informer = informers.NewSharedInformerFactory(client, 0) } @@ -77,6 +78,9 @@ func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, ext PathPrefix = pathPrefix deployment_name = deploy_name initDeploy = initdeploy + ProviderHostname = providerHostname + ProviderEndpoint = providerEndpoint + return &ClusterWatcher{ Nodes: []Node{}, Daemonsets: make(map[string]int), diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index 8a32413130..67e08612da 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -472,7 +472,7 @@ func (clusterWatcher *ClusterWatcher) deployControllerDeployment(deployment *app return nil } -func (clusterWatcher *ClusterWatcher) getProvider() string { +func (clusterWatcher *ClusterWatcher) getProvider(providerHostname, providerEndpoint string) (string, string, string) { nodes, err := clusterWatcher.Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) if err != nil { clusterWatcher.Log.Warnf("Error listing nodes: %s\n", err.Error()) @@ -481,20 +481,36 @@ func (clusterWatcher *ClusterWatcher) getProvider() string { for _, node := range nodes.Items { for key, label := range node.Labels { if strings.Contains(key, "gke") || strings.Contains(label, "gke") { - return "gke" + if providerHostname != "" && providerEndpoint == "" { + providerEndpoint = "/computeMetadata/v1/instance/attributes/cluster-name" + } else if providerHostname == "" && providerEndpoint != "" { + providerHostname = "http://metadata.google.internal" + } else if providerHostname == "" && providerEndpoint == "" { + providerHostname = "http://metadata.google.internal" + providerEndpoint = "/computeMetadata/v1/instance/attributes/cluster-name" + } + return "gke", providerHostname, providerEndpoint } else if strings.Contains(key, "eks") || strings.Contains(label, "eks") { - return "eks" + if providerHostname != "" && providerEndpoint == "" { + providerEndpoint = "/latest/user-data" + } else if providerHostname == "" && providerEndpoint != "" { + providerHostname = "http://169.254.169.254/latest" + } else if providerHostname == "" && providerEndpoint == "" { + providerHostname = "http://169.254.169.254/latest" + providerEndpoint = "/latest/user-data" + } + return "eks", providerHostname, providerEndpoint } } } - return "default" + return "default", "", "" } -func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle() (string, error) { - url := "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name" +func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle(providerHostname, providerEndpoint string) (string, error) { + url := providerHostname + providerEndpoint req, err := http.NewRequest("GET", url, nil) if err != nil { - clusterWatcher.Log.Warnf("failed to create request: %w", err) + clusterWatcher.Log.Warnf("failed to create request: %w, check provider host name and endpoint", err) return "", err } @@ -505,7 +521,7 @@ func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle() (string, erro client := &http.Client{} resp, err := client.Do(req) if err != nil { - clusterWatcher.Log.Warnf("error making request: %w", err) + clusterWatcher.Log.Warnf("error making request: %w, check provider host name and endpoint", err) return "", err } defer resp.Body.Close() @@ -526,12 +542,12 @@ func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGoogle() (string, erro return string(body), nil } -func (clusterWatcher *ClusterWatcher) fetchClusterNameFromAWS() (string, error) { +func (clusterWatcher *ClusterWatcher) fetchClusterNameFromAWS(providerHostname, providerEndpoint string) (string, error) { var token []byte client := &http.Client{Timeout: 2 * time.Second} - req, err := http.NewRequest("PUT", "http://169.254.169.254/latest/api/token", nil) + req, err := http.NewRequest("PUT", providerHostname+"/latest/api/token", nil) if err != nil { - clusterWatcher.Log.Warnf("failed to create request for fetching token: %w", err) + clusterWatcher.Log.Warnf("failed to create request for fetching token: %w, check provider host name", err) return "", err } req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "21600") @@ -552,10 +568,11 @@ func (clusterWatcher *ClusterWatcher) fetchClusterNameFromAWS() (string, error) } // Fetch the EKS cluster name from user data - req, err = http.NewRequest("GET", "http://169.254.169.254/latest/user-data", nil) + url := providerHostname + providerEndpoint + req, err = http.NewRequest("GET", url, nil) client = &http.Client{Timeout: 2 * time.Second} if err != nil { - clusterWatcher.Log.Warnf("failed to create request for fetching metadata: %w", err) + clusterWatcher.Log.Warnf("failed to create request for fetching metadata: %w, check provider host name and endpoint", err) return "", err } req.Header.Set("X-aws-ec2-metadata-token", string(token)) @@ -588,18 +605,18 @@ func (clusterWatcher *ClusterWatcher) fetchClusterNameFromAWS() (string, error) return "", err } -func (clusterWatcher *ClusterWatcher) getClusterName() string { - provider := clusterWatcher.getProvider() +func (clusterWatcher *ClusterWatcher) GetClusterName(providerHostname, providerEndpoint string) string { + provider, pHostname, pEndpoint := clusterWatcher.getProvider(ProviderHostname, providerEndpoint) if provider == "gke" { clusterWatcher.Log.Infof("Provider is GKE") - if clusterName, err := clusterWatcher.fetchClusterNameFromGoogle(); err != nil { + if clusterName, err := clusterWatcher.fetchClusterNameFromGoogle(pHostname, pEndpoint); err != nil { clusterWatcher.Log.Warnf("Cannot fetch cluster name for GKE %s", err.Error()) } else { return clusterName } } else if provider == "eks" { clusterWatcher.Log.Infof("Provider is EKS") - if clusterName, err := clusterWatcher.fetchClusterNameFromAWS(); err != nil { + if clusterName, err := clusterWatcher.fetchClusterNameFromAWS(pHostname, pEndpoint); err != nil { clusterWatcher.Log.Warnf("Cannot fetch cluster name for EKS %s", err.Error()) } else { return clusterName @@ -704,7 +721,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { // kubearmor configmap configmap := addOwnership(deployments.GetKubearmorConfigMap(common.Namespace, deployments.KubeArmorConfigMapName)).(*corev1.ConfigMap) configmap.Data = common.ConfigMapData - configmap.Data["cluster"] = clusterWatcher.getClusterName() + configmap.Data["cluster"] = clusterWatcher.GetClusterName(ProviderHostname, ProviderEndpoint) for { caCert, tlsCrt, tlsKey, kGenErr = common.GeneratePki(common.Namespace, deployments.KubeArmorControllerWebhookServiceName)