From a3612a6e7e1478d188b726aa4ad7372b64d4198d Mon Sep 17 00:00:00 2001 From: Matthew Bates Date: Tue, 22 Oct 2024 10:40:11 +0100 Subject: [PATCH] adds workload status command - wip add initial implementation to inject a debug container and run it and attach to stdout update debug container name in the pod add workload as root command mark flags as required update implementation to use logs instead of exec add spinner remove the existing container check so it can be run again remove the existing container check so it can be run again generate a new unique debug container each time the cmd is used add the spiffe workload API volume mount disables container selectors Signed-off-by: Maartje Eyskens --- cmd/cofidectl/cmd/workload/workload.go | 195 +++++++++++++++++++- internal/pkg/provider/helm/values.go | 1 + internal/pkg/trustprovider/trustprovider.go | 2 +- 3 files changed, 195 insertions(+), 3 deletions(-) diff --git a/cmd/cofidectl/cmd/workload/workload.go b/cmd/cofidectl/cmd/workload/workload.go index 647e7cf..33af468 100644 --- a/cmd/cofidectl/cmd/workload/workload.go +++ b/cmd/cofidectl/cmd/workload/workload.go @@ -4,15 +4,24 @@ package workload import ( + "bytes" "context" "fmt" + "io" "os" + "time" trust_zone_proto "github.com/cofide/cofide-api-sdk/gen/go/proto/trust_zone/v1alpha1" cmdcontext "github.com/cofide/cofidectl/cmd/cofidectl/cmd/context" "github.com/cofide/cofidectl/internal/pkg/workload" "github.com/olekukonko/tablewriter" + + "github.com/briandowns/spinner" + kubeutil "github.com/cofide/cofidectl/internal/pkg/kube" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" ) type WorkloadCommand struct { @@ -29,6 +38,13 @@ var workloadRootCmdDesc = ` This command consists of multiple sub-commands to interact with workloads. ` +type Opts struct { + workloadName string // used to pass arg[0] + podName string + namespace string + trustZone string +} + func (c *WorkloadCommand) GetRootCommand() *cobra.Command { cmd := &cobra.Command{ Use: "workload list|discover [ARGS]", @@ -40,6 +56,7 @@ func (c *WorkloadCommand) GetRootCommand() *cobra.Command { cmd.AddCommand( c.GetListCommand(), c.GetDiscoverCommand(), + c.GetStatusCommand(), ) return cmd @@ -87,22 +104,73 @@ func (w *WorkloadCommand) GetListCommand() *cobra.Command { return fmt.Errorf("no trust zones have been configured") } + return nil + }, + } + + return cmd +} + +var workloadStatusCmdDesc = ` +This command will display the status of workloads in the Cofide configuration state. +` + +func (w *WorkloadCommand) GetStatusCommand() *cobra.Command { + opts := Opts{} + cmd := &cobra.Command{ + Use: "status [NAME]", + Short: "Display workload status", + Long: workloadStatusCmdDesc, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { kubeConfig, err := cmd.Flags().GetString("kube-config") if err != nil { return fmt.Errorf("failed to retrieve the kubeconfig file location") } + ds, err := w.cmdCtx.PluginManager.GetDataSource() + if err != nil { + return err + } + + var trustZones []*trust_zone_proto.TrustZone + + if opts.trustZone != "" { + trustZone, err := ds.GetTrustZone(opts.trustZone) + if err != nil { + return err + } + + trustZones = append(trustZones, trustZone) + } else { + trustZones, err = ds.ListTrustZones() + if err != nil { + return err + } + } + + if len(trustZones) == 0 { + return fmt.Errorf("no trust zones have been configured") + } + err = renderRegisteredWorkloads(cmd.Context(), kubeConfig, trustZones) if err != nil { return err } - return nil + opts.workloadName = args[0] + return w.status(cmd.Context(), kubeConfig, opts) }, } f := cmd.Flags() - f.StringVar(&opts.trustZone, "trust-zone", "", "list the registered workloads in a specific trust zone") + f.StringVar(&opts.podName, "pod-name", "", "Pod name for the workload") + f.StringVar(&opts.namespace, "namespace", "", "Namespace for the workload") + f.StringVar(&opts.trustZone, "trust-zone", "", "Trust zone for the workload") + + cobra.CheckErr(cmd.MarkFlagRequired("pod-name")) + cobra.CheckErr(cmd.MarkFlagRequired("namespace")) + cobra.CheckErr(cmd.MarkFlagRequired("trust-zone")) return cmd } @@ -137,6 +205,49 @@ func renderRegisteredWorkloads(ctx context.Context, kubeConfig string, trustZone return nil } +const debugContainerName = "cofidectl-debug-container" +const debugContainerNamePrefix = "cofidectl-debug" +const debugContainerImage = "cofidectl-debug:latest" + +func (w *WorkloadCommand) status(ctx context.Context, kubeConfig string, opts Opts) error { + ds, err := w.cmdCtx.PluginManager.GetDataSource() + if err != nil { + return err + } + + trustZone, err := ds.GetTrustZone(opts.trustZone) + if err != nil { + return err + } + + client, err := kubeutil.NewKubeClientFromSpecifiedContext(kubeConfig, *trustZone.KubernetesContext) + if err != nil { + return err + } + + // Create a spinner to display whilst the debug container is created and executed and logs retrieved + s := spinner.New(spinner.CharSets[9], 100*time.Millisecond) + s.Start() + defer s.Stop() + s.Suffix = "Starting debug container" + + pod, container, err := createDebugContainer(ctx, client, opts.podName, opts.namespace) + if err != nil { + return fmt.Errorf("could not create ephemeral debug container: %s", err) + } + + s.Suffix = "Retrieving workload status" + + workload, err := getWorkloadStatus(ctx, client, pod, container) + if err != nil { + return fmt.Errorf("could not retrieve logs of the ephemeral debug container: %w", err) + } + + fmt.Println(workload) + + return nil +} + var workloadDiscoverCmdDesc = ` This command will discover all of the unregistered workloads. ` @@ -237,3 +348,83 @@ func renderUnregisteredWorkloads(ctx context.Context, kubeConfig string, trustZo return nil } + +func createDebugContainer(ctx context.Context, client *kubeutil.Client, podName string, namespace string) (*corev1.Pod, string, error) { + pod, err := client.Clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return nil, "", fmt.Errorf("error getting pod: %v", err) + } + + debugContainerName := fmt.Sprintf("%s-%s", debugContainerNamePrefix, rand.String(5)) + + debugContainer := corev1.EphemeralContainer{ + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: debugContainerName, + Image: debugContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + TTY: true, + Stdin: true, + VolumeMounts: []corev1.VolumeMount{ + { + ReadOnly: true, + Name: "spiffe-workload-api", + MountPath: "/spiffe-workload-api", + }}, + }, + TargetContainerName: pod.Spec.Containers[0].Name, + } + + pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, debugContainer) + + _, err = client.Clientset.CoreV1().Pods(namespace).UpdateEphemeralContainers( + ctx, + pod.Name, + pod, + metav1.UpdateOptions{}, + ) + if err != nil { + return nil, "", fmt.Errorf("error creating debug container: %v", err) + } + + // Wait for the debug container to complete + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + for { + pod, err := client.Clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return nil, "", fmt.Errorf("error getting pod status: %v", err) + } + + for _, status := range pod.Status.EphemeralContainerStatuses { + if status.Name == debugContainerName && status.State.Terminated != nil { + return pod, debugContainerName, nil + } + } + + select { + case <-waitCtx.Done(): + return nil, "", fmt.Errorf("timeout waiting for debug container to complete") + default: + continue + } + } +} + +func getWorkloadStatus(ctx context.Context, client *kubeutil.Client, pod *corev1.Pod, container string) (string, error) { + logs, err := client.Clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: container, + }).Stream(ctx) + if err != nil { + return "", err + } + defer logs.Close() + + // Read the logs + buf := new(bytes.Buffer) + _, err = io.Copy(buf, logs) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/internal/pkg/provider/helm/values.go b/internal/pkg/provider/helm/values.go index 7a4d5d1..e677efe 100644 --- a/internal/pkg/provider/helm/values.go +++ b/internal/pkg/provider/helm/values.go @@ -53,6 +53,7 @@ func (g *HelmValuesGenerator) GenerateValues() (map[string]interface{}, error) { fmt.Sprintf(`"spire-agent"."workloadAttestors"."%s"."enabled"`, agentConfig.WorkloadAttestor): agentConfig.WorkloadAttestorConfig["enabled"], fmt.Sprintf(`"spire-agent"."workloadAttestors"."%s"."skipKubeletVerification"`, agentConfig.WorkloadAttestor): agentConfig.WorkloadAttestorConfig["skipKubeletVerification"], fmt.Sprintf(`"spire-agent"."workloadAttestors"."%s"."useNewContainerLocator"`, agentConfig.WorkloadAttestor): agentConfig.WorkloadAttestorConfig["useNewContainerLocator"], + fmt.Sprintf(`"spire-agent"."workloadAttestors"."%s"."disableContainerSelectors"`, agentConfig.WorkloadAttestor): agentConfig.WorkloadAttestorConfig["disableContainerSelectors"], fmt.Sprintf(`"spire-agent"."workloadAttestors"."%s"."verboseContainerLocatorLogs"`, agentConfig.WorkloadAttestor): agentConfig.WorkloadAttestorConfig["verboseContainerLocatorLogs"], `"spire-agent"."server"."address"`: "spire-server.spire", } diff --git a/internal/pkg/trustprovider/trustprovider.go b/internal/pkg/trustprovider/trustprovider.go index d7cd1ca..6427c9e 100644 --- a/internal/pkg/trustprovider/trustprovider.go +++ b/internal/pkg/trustprovider/trustprovider.go @@ -41,7 +41,7 @@ func (tp *TrustProvider) GetValues() error { WorkloadAttestorConfig: map[string]interface{}{ "enabled": true, "skipKubeletVerification": true, - "disableContainerSelectors": false, + "disableContainerSelectors": true, "useNewContainerLocator": false, "verboseContainerLocatorLogs": false, },