From db07ae8ff1f2e8b33fd67d6748588b31522f05f1 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Fri, 11 Oct 2024 12:57:19 +0200 Subject: [PATCH] feat: add app stats output This adds a new `stats` output format for the `get app` command. It prints status information about the app replicas and also some very basic metrics. --- api/client.go | 25 +++++++++ api/util/apps.go | 43 +++++++++++++++ exec/application.go | 61 ++++------------------ get/application.go | 124 ++++++++++++++++++++++++++++++++++++++++++++ get/get.go | 3 +- go.mod | 3 +- go.sum | 6 ++- 7 files changed, 209 insertions(+), 56 deletions(-) diff --git a/api/client.go b/api/client.go index bc063e5..386cad9 100644 --- a/api/client.go +++ b/api/client.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/base64" "fmt" "os" "os/user" @@ -9,6 +10,8 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/ninech/apis" + infrastructure "github.com/ninech/apis/infrastructure/v1alpha1" + meta "github.com/ninech/apis/meta/v1alpha1" "github.com/ninech/nctl/api/log" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -161,6 +164,28 @@ func (c *Client) Token(ctx context.Context) string { return token } +func (c *Client) DeploioRuntimeClient(ctx context.Context, scheme *runtime.Scheme) (runtimeclient.Client, error) { + cfg, err := c.DeploioRuntimeConfig(ctx) + if err != nil { + return nil, err + } + return runtimeclient.New(cfg, runtimeclient.Options{Scheme: scheme}) +} + +func (c *Client) DeploioRuntimeConfig(ctx context.Context) (*rest.Config, error) { + config := rest.CopyConfig(c.Config) + deploioClusterData := &infrastructure.ClusterData{} + if err := c.Get(ctx, types.NamespacedName{Name: meta.ClusterDataDeploioName}, deploioClusterData); err != nil { + return nil, fmt.Errorf("can not gather deplo.io cluster connection details: %w", err) + } + config.Host = deploioClusterData.Status.AtProvider.APIEndpoint + var err error + if config.CAData, err = base64.StdEncoding.DecodeString(deploioClusterData.Status.AtProvider.APICACert); err != nil { + return nil, fmt.Errorf("can not decode deplo.io cluster CA certificate: %w", err) + } + return config, nil +} + func LoadingRules() (*clientcmd.ClientConfigLoadingRules, error) { loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() if _, ok := os.LookupEnv("HOME"); !ok { diff --git a/api/util/apps.go b/api/util/apps.go index 7d3f16c..f6831a4 100644 --- a/api/util/apps.go +++ b/api/util/apps.go @@ -11,7 +11,9 @@ import ( "github.com/ninech/nctl/api" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -300,3 +302,44 @@ func ValidatePEM(content string) (*string, error) { } return &content, nil } + +func ApplicationLatestAvailableRelease(ctx context.Context, client *api.Client, app types.NamespacedName) (*apps.Release, error) { + releases := &apps.ReleaseList{} + if err := client.List( + ctx, + releases, + runtimeclient.InNamespace(app.Namespace), + runtimeclient.MatchingLabels{ApplicationNameLabel: app.Name}, + ); err != nil { + return nil, err + } + + if len(releases.Items) == 0 { + return nil, fmt.Errorf("no releases found for application %s", app.Name) + } + release := latestAvailableRelease(releases) + if release == nil { + return nil, fmt.Errorf("no ready release found for application %s", app.Name) + } + + return release, nil +} + +func ApplicationReplicas(ctx context.Context, client *api.Client, app types.NamespacedName) ([]apps.ReplicaObservation, error) { + release, err := ApplicationLatestAvailableRelease(ctx, client, app) + if err != nil { + return nil, err + } + + return release.Status.AtProvider.ReplicaObservation, nil +} + +func latestAvailableRelease(releases *apps.ReleaseList) *apps.Release { + OrderReleaseList(releases, false) + for _, release := range releases.Items { + if release.Status.AtProvider.ReleaseStatus == apps.ReleaseProcessStatusAvailable { + return &release + } + } + return nil +} diff --git a/exec/application.go b/exec/application.go index 74ff41f..87cae34 100644 --- a/exec/application.go +++ b/exec/application.go @@ -5,22 +5,16 @@ import ( "fmt" "io" - b64 "encoding/base64" - dockerterm "github.com/moby/term" apps "github.com/ninech/apis/apps/v1alpha1" - infrastructure "github.com/ninech/apis/infrastructure/v1alpha1" - meta "github.com/ninech/apis/meta/v1alpha1" "github.com/ninech/nctl/api" "github.com/ninech/nctl/api/util" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubectl/pkg/scheme" "k8s.io/kubectl/pkg/util/term" - runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -63,7 +57,7 @@ func (ac applicationCmd) Help() string { # Get output from running the 'date' command in an application replica. nctl exec app myapp -- date - # Use redirection to execute a comand. + # Use redirection to execute a command. echo date | nctl exec app myapp # In certain situations it might be needed to not redirect stdin. This can be @@ -77,7 +71,7 @@ func (cmd *applicationCmd) Run(ctx context.Context, client *api.Client, exec *Cm if err != nil { return fmt.Errorf("error when searching for replica to connect: %w", err) } - config, err := deploioRestConfig(ctx, client) + config, err := client.DeploioRuntimeConfig(ctx) if err != nil { return fmt.Errorf("can not create deplo.io cluster rest config: %w", err) } @@ -98,46 +92,23 @@ func (cmd *applicationCmd) Run(ctx context.Context, client *api.Client, exec *Cm }) } -func latestAvailableRelease(releases *apps.ReleaseList) *apps.Release { - util.OrderReleaseList(releases, false) - for _, release := range releases.Items { - if release.Status.AtProvider.ReleaseStatus == apps.ReleaseProcessStatusAvailable { - return &release - } - } - return nil -} - // getReplica finds a replica of the latest available release func (cmd *applicationCmd) getReplica(ctx context.Context, client *api.Client) (string, appBuildType, error) { - releases := &apps.ReleaseList{} - if err := client.List( - ctx, - releases, - runtimeclient.InNamespace(client.Project), - runtimeclient.MatchingLabels{util.ApplicationNameLabel: cmd.Name}, - ); err != nil { + release, err := util.ApplicationLatestAvailableRelease(ctx, client, client.Name(cmd.Name)) + if err != nil { return "", "", err } - - if len(releases.Items) == 0 { - return "", "", fmt.Errorf("no releases found for application %s", cmd.Name) - } - latestAvailableRelease := latestAvailableRelease(releases) - if latestAvailableRelease == nil { - return "", "", fmt.Errorf("no ready release found for application %s", cmd.Name) - } buildType := appBuildTypeBuildpack - if latestAvailableRelease.Spec.ForProvider.DockerfileBuild { + if release.Spec.ForProvider.DockerfileBuild { buildType = appBuildTypeDockerfile } - if len(latestAvailableRelease.Status.AtProvider.ReplicaObservation) == 0 { - return "", buildType, fmt.Errorf("no replica information found for release %s", latestAvailableRelease.Name) + if len(release.Status.AtProvider.ReplicaObservation) == 0 { + return "", buildType, fmt.Errorf("no replica information found for release %s", release.Name) } - if replica := readyReplica(latestAvailableRelease.Status.AtProvider.ReplicaObservation); replica != "" { + if replica := readyReplica(release.Status.AtProvider.ReplicaObservation); replica != "" { return replica, buildType, nil } - return "", buildType, fmt.Errorf("no ready replica found for release %s", latestAvailableRelease.Name) + return "", buildType, fmt.Errorf("no ready replica found for release %s", release.Name) } func readyReplica(replicaObs []apps.ReplicaObservation) string { @@ -221,20 +192,6 @@ func executeRemoteCommand(ctx context.Context, params remoteCommandParameters) e return tty.Safe(fn) } -func deploioRestConfig(ctx context.Context, client *api.Client) (*rest.Config, error) { - config := rest.CopyConfig(client.Config) - deploioClusterData := &infrastructure.ClusterData{} - if err := client.Get(ctx, types.NamespacedName{Name: meta.ClusterDataDeploioName}, deploioClusterData); err != nil { - return nil, fmt.Errorf("can not gather deplo.io cluster connection details: %w", err) - } - config.Host = deploioClusterData.Status.AtProvider.APIEndpoint - var err error - if config.CAData, err = b64.StdEncoding.DecodeString(deploioClusterData.Status.AtProvider.APICACert); err != nil { - return nil, fmt.Errorf("can not decode deplo.io cluster CA certificate: %w", err) - } - return config, nil -} - func replicaCommand(buildType appBuildType, command []string) []string { switch buildType { case appBuildTypeBuildpack: diff --git a/get/application.go b/get/application.go index adfa1af..5b02a9d 100644 --- a/get/application.go +++ b/get/application.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "strconv" "strings" "text/tabwriter" @@ -12,6 +13,10 @@ import ( "github.com/ninech/nctl/api" "github.com/ninech/nctl/api/util" "github.com/ninech/nctl/internal/format" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" ) type applicationsCmd struct { @@ -55,11 +60,25 @@ func (cmd *applicationsCmd) Run(ctx context.Context, client *api.Client, get *Cm return printApplication(appList.Items, get, defaultOut(cmd.out), false) case yamlOut: return format.PrettyPrintObjects(appList.GetItems(), format.PrintOpts{Out: defaultOut(cmd.out)}) + case stats: + return cmd.printStats(ctx, client, appList.Items, get, defaultOut(cmd.out)) } return nil } +func (cmd *applicationsCmd) Help() string { + return "To get an overview of the app and replica usage, use the flag '-o stats':\n" + + "\tREPLICA: The name of the app replica.\n" + + "\tSTATUS: Current status of the replica.\n" + + "\tCPU: Current CPU usage in millicores (1000m is a full CPU core).\n" + + "\tCPU%: Current CPU usage relative to the app size. This can be over 100% as Deploio allows bursting.\n" + + "\tMEMORY: Current Memory usage in MiB.\n" + + "\tMEMORY%: Current Memory relative to the app size. This can be over 100% as Deploio allows bursting.\n" + + "\tRESTARTS: The amount of times the replica has been restarted.\n" + + "\tLASTEXITCODE: The exit code the last time the replica restarted. This can give an indication on why the replica is restarting." +} + func printApplication(apps []apps.Application, get *Cmd, out io.Writer, header bool) error { w := tabwriter.NewWriter(out, 0, 0, 4, ' ', 0) @@ -163,3 +182,108 @@ func printDNSDetailsTabRow(items []util.DNSDetail, get *Cmd, out io.Writer) erro return nil } + +func (cmd *applicationsCmd) printStats(ctx context.Context, c *api.Client, appList []apps.Application, get *Cmd, out io.Writer) error { + scheme := runtime.NewScheme() + if err := metricsv1beta1.AddToScheme(scheme); err != nil { + return err + } + + runtimeClient, err := c.DeploioRuntimeClient(ctx, scheme) + if err != nil { + return err + } + w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) + get.writeHeader(w, "NAME", "REPLICA", "STATUS", "CPU", "CPU%", "MEMORY", "MEMORY%", "RESTARTS", "LASTEXITCODE") + + for _, app := range appList { + replicas, err := util.ApplicationReplicas(ctx, c, api.ObjectName(&app)) + if err != nil { + format.PrintWarningf("unable to get replicas for app %s\n", c.Name(app.Name)) + continue + } + + if len(replicas) == 0 { + continue + } + + for _, replica := range replicas { + podMetrics := metricsv1beta1.PodMetrics{} + if err := runtimeClient.Get(ctx, api.NamespacedName(replica.ReplicaName, app.Namespace), &podMetrics); err != nil { + format.PrintWarningf("unable to get metrics for replica %s\n", replica.ReplicaName) + } + + appResources := apps.AppResources[app.Status.AtProvider.Size] + // We expect exactly one container, fall back to [util.NoneText] if that's + // not the case. The container might simply not have any metrics yet. + cpuUsage, cpuPercentage := util.NoneText, util.NoneText + memoryUsage, memoryPercentage := util.NoneText, util.NoneText + if len(podMetrics.Containers) == 1 { + cpu := podMetrics.Containers[0].Usage[corev1.ResourceCPU] + cpuUsage = formatQuantity(corev1.ResourceCPU, cpu) + cpuPercentage = formatPercentage(cpu.MilliValue(), appResources.Cpu().MilliValue()) + memory := podMetrics.Containers[0].Usage[corev1.ResourceMemory] + memoryUsage = formatQuantity(corev1.ResourceMemory, memory) + memoryPercentage = formatPercentage(memory.MilliValue(), appResources.Memory().MilliValue()) + } + + get.writeTabRow( + w, c.Project, app.Name, + replica.ReplicaName, + string(replica.Status), + cpuUsage, + cpuPercentage, + memoryUsage, + memoryPercentage, + formatRestartCount(replica), + formatExitCode(replica), + ) + } + } + return w.Flush() +} + +// formatQuantity formats cpu/memory into human readable form. Adapted from +// https://github.com/kubernetes/kubectl/blob/v0.31.1/pkg/metricsutil/metrics_printer.go#L209 +func formatQuantity(resourceType corev1.ResourceName, quantity resource.Quantity) string { + switch resourceType { + case corev1.ResourceCPU: + return fmt.Sprintf("%vm", quantity.MilliValue()) + case corev1.ResourceMemory: + return fmt.Sprintf("%vMiB", quantity.Value()/toMiB(1)) + default: + return fmt.Sprintf("%v", quantity.Value()) + } +} + +func formatPercentage(val, total int64) string { + if total == 0 { + return util.NoneText + } + return fmt.Sprintf("%.1f", float64(val)/float64(total)*100) + "%" +} + +func toMiB(val int64) int64 { + return val * 1024 * 1024 +} + +func formatExitCode(replica apps.ReplicaObservation) string { + lastExitCode := util.NoneText + + if replica.LastExitCode != nil { + lastExitCode = strconv.Itoa(int(*replica.LastExitCode)) + // not exactly guaranteed but 137 is usually caused by the OOM killer + if *replica.LastExitCode == 137 { + lastExitCode = lastExitCode + " (Out of memory)" + } + } + return lastExitCode +} + +func formatRestartCount(replica apps.ReplicaObservation) string { + restartCount := util.NoneText + if replica.RestartCount != nil { + restartCount = strconv.Itoa(int(*replica.RestartCount)) + } + return restartCount +} diff --git a/get/get.go b/get/get.go index 872a2ee..a2ead17 100644 --- a/get/get.go +++ b/get/get.go @@ -22,7 +22,7 @@ import ( ) type Cmd struct { - Output output `help:"Configures list output. ${enum}" short:"o" enum:"full,no-header,contexts,yaml" default:"full"` + Output output `help:"Configures list output. ${enum}" short:"o" enum:"full,no-header,contexts,yaml,stats" default:"full"` AllProjects bool `help:"apply the get over all projects." short:"A"` Clusters clustersCmd `cmd:"" group:"infrastructure.nine.ch" aliases:"cluster,vcluster" help:"Get Kubernetes Clusters."` APIServiceAccounts apiServiceAccountsCmd `cmd:"" group:"iam.nine.ch" name:"apiserviceaccounts" aliases:"asa" help:"Get API Service Accounts."` @@ -51,6 +51,7 @@ const ( noHeader output = "no-header" contexts output = "contexts" yamlOut output = "yaml" + stats output = "stats" ) type listOpt func(cmd *Cmd) diff --git a/go.mod b/go.mod index 28476e0..f768c86 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/moby/moby v27.1.1+incompatible github.com/moby/term v0.5.0 - github.com/ninech/apis v0.0.0-20240918081833-57655d99d868 + github.com/ninech/apis v0.0.0-20241014111010-322e555390a0 github.com/posener/complete v1.2.3 github.com/prometheus/common v0.55.0 github.com/stretchr/testify v1.9.0 @@ -38,6 +38,7 @@ require ( k8s.io/apimachinery v0.31.0 k8s.io/client-go v0.30.1 k8s.io/kubectl v0.29.0 + k8s.io/metrics v0.29.0 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.18.4 ) diff --git a/go.sum b/go.sum index 3e0f5f4..53c1639 100644 --- a/go.sum +++ b/go.sum @@ -592,8 +592,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/ninech/apis v0.0.0-20240918081833-57655d99d868 h1:thQBwQjpYq6AjQGtVK2JFZ+iJFKFbUFhAjFuCgSZHUo= -github.com/ninech/apis v0.0.0-20240918081833-57655d99d868/go.mod h1:hrhn1IP1wHHyWfE8xvDAl39yVcT1RDr9qJOv1FYpDAU= +github.com/ninech/apis v0.0.0-20241014111010-322e555390a0 h1:ADBcd12IyQtoDWK7DHqQHWWrwo+yrJuWnj9EvEIwc8c= +github.com/ninech/apis v0.0.0-20241014111010-322e555390a0/go.mod h1:hrhn1IP1wHHyWfE8xvDAl39yVcT1RDr9qJOv1FYpDAU= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -1230,6 +1230,8 @@ k8s.io/kube-openapi v0.0.0-20240403164606-bc84c2ddaf99 h1:w6nThEmGo9zcL+xH1Tu6pj k8s.io/kube-openapi v0.0.0-20240403164606-bc84c2ddaf99/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kubectl v0.29.0 h1:Oqi48gXjikDhrBF67AYuZRTcJV4lg2l42GmvsP7FmYI= k8s.io/kubectl v0.29.0/go.mod h1:0jMjGWIcMIQzmUaMgAzhSELv5WtHo2a8pq67DtviAJs= +k8s.io/metrics v0.29.0 h1:a6dWcNM+EEowMzMZ8trka6wZtSRIfEA/9oLjuhBksGc= +k8s.io/metrics v0.29.0/go.mod h1:UCuTT4dC/x/x6ODSk87IWIZQnuAfcwxOjb1gjWJdjMA= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=