Merge pull request #178 from ninech/app-metrics

feat: add app stats output

api/client.go

@@ -2,13 +2,16 @@ package api
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
 
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
 	"github.com/ninech/apis"
+	infrastructure "github.com/ninech/apis/infrastructure/v1alpha1"
+	meta "github.com/ninech/apis/meta/v1alpha1"
 	"github.com/ninech/nctl/api/log"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -161,6 +164,28 @@ func (c *Client) Token(ctx context.Context) string {
 	return token
 }
 
+func (c *Client) DeploioRuntimeClient(ctx context.Context, scheme *runtime.Scheme) (runtimeclient.Client, error) {
+	cfg, err := c.DeploioRuntimeConfig(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return runtimeclient.New(cfg, runtimeclient.Options{Scheme: scheme})
+}
+
+func (c *Client) DeploioRuntimeConfig(ctx context.Context) (*rest.Config, error) {
+	config := rest.CopyConfig(c.Config)
+	deploioClusterData := &infrastructure.ClusterData{}
+	if err := c.Get(ctx, types.NamespacedName{Name: meta.ClusterDataDeploioName}, deploioClusterData); err != nil {
+		return nil, fmt.Errorf("can not gather deplo.io cluster connection details: %w", err)
+	}
+	config.Host = deploioClusterData.Status.AtProvider.APIEndpoint
+	var err error
+	if config.CAData, err = base64.StdEncoding.DecodeString(deploioClusterData.Status.AtProvider.APICACert); err != nil {
+		return nil, fmt.Errorf("can not decode deplo.io cluster CA certificate: %w", err)
+	}
+	return config, nil
+}
+
 func LoadingRules() (*clientcmd.ClientConfigLoadingRules, error) {
 	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
 	if _, ok := os.LookupEnv("HOME"); !ok {

api/util/apps.go

@@ -11,7 +11,9 @@ import (
 	"github.com/ninech/nctl/api"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -300,3 +302,44 @@ func ValidatePEM(content string) (*string, error) {
 	}
 	return &content, nil
 }
+
+func ApplicationLatestAvailableRelease(ctx context.Context, client *api.Client, app types.NamespacedName) (*apps.Release, error) {
+	releases := &apps.ReleaseList{}
+	if err := client.List(
+		ctx,
+		releases,
+		runtimeclient.InNamespace(app.Namespace),
+		runtimeclient.MatchingLabels{ApplicationNameLabel: app.Name},
+	); err != nil {
+		return nil, err
+	}
+
+	if len(releases.Items) == 0 {
+		return nil, fmt.Errorf("no releases found for application %s", app.Name)
+	}
+	release := latestAvailableRelease(releases)
+	if release == nil {
+		return nil, fmt.Errorf("no ready release found for application %s", app.Name)
+	}
+
+	return release, nil
+}
+
+func ApplicationReplicas(ctx context.Context, client *api.Client, app types.NamespacedName) ([]apps.ReplicaObservation, error) {
+	release, err := ApplicationLatestAvailableRelease(ctx, client, app)
+	if err != nil {
+		return nil, err
+	}
+
+	return release.Status.AtProvider.ReplicaObservation, nil
+}
+
+func latestAvailableRelease(releases *apps.ReleaseList) *apps.Release {
+	OrderReleaseList(releases, false)
+	for _, release := range releases.Items {
+		if release.Status.AtProvider.ReleaseStatus == apps.ReleaseProcessStatusAvailable {
+			return &release
+		}
+	}
+	return nil
+}

exec/application.go

@@ -5,22 +5,16 @@ import (
 	"fmt"
 	"io"
 
-	b64 "encoding/base64"
-
 	dockerterm "github.com/moby/term"
 	apps "github.com/ninech/apis/apps/v1alpha1"
-	infrastructure "github.com/ninech/apis/infrastructure/v1alpha1"
-	meta "github.com/ninech/apis/meta/v1alpha1"
 	"github.com/ninech/nctl/api"
 	"github.com/ninech/nctl/api/util"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/kubectl/pkg/scheme"
 	"k8s.io/kubectl/pkg/util/term"
-	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -63,7 +57,7 @@ func (ac applicationCmd) Help() string {
   # Get output from running the 'date' command in an application replica.
   nctl exec app myapp -- date
 
-  # Use redirection to execute a comand.
+  # Use redirection to execute a command.
   echo date | nctl exec app myapp
 
   # In certain situations it might be needed to not redirect stdin. This can be
@@ -77,7 +71,7 @@ func (cmd *applicationCmd) Run(ctx context.Context, client *api.Client, exec *Cm
 	if err != nil {
 		return fmt.Errorf("error when searching for replica to connect: %w", err)
 	}
-	config, err := deploioRestConfig(ctx, client)
+	config, err := client.DeploioRuntimeConfig(ctx)
 	if err != nil {
 		return fmt.Errorf("can not create deplo.io cluster rest config: %w", err)
 	}
@@ -98,46 +92,23 @@ func (cmd *applicationCmd) Run(ctx context.Context, client *api.Client, exec *Cm
 		})
 }
 
-func latestAvailableRelease(releases *apps.ReleaseList) *apps.Release {
-	util.OrderReleaseList(releases, false)
-	for _, release := range releases.Items {
-		if release.Status.AtProvider.ReleaseStatus == apps.ReleaseProcessStatusAvailable {
-			return &release
-		}
-	}
-	return nil
-}
-
 // getReplica finds a replica of the latest available release
 func (cmd *applicationCmd) getReplica(ctx context.Context, client *api.Client) (string, appBuildType, error) {
-	releases := &apps.ReleaseList{}
-	if err := client.List(
-		ctx,
-		releases,
-		runtimeclient.InNamespace(client.Project),
-		runtimeclient.MatchingLabels{util.ApplicationNameLabel: cmd.Name},
-	); err != nil {
+	release, err := util.ApplicationLatestAvailableRelease(ctx, client, client.Name(cmd.Name))
+	if err != nil {
 		return "", "", err
 	}
-
-	if len(releases.Items) == 0 {
-		return "", "", fmt.Errorf("no releases found for application %s", cmd.Name)
-	}
-	latestAvailableRelease := latestAvailableRelease(releases)
-	if latestAvailableRelease == nil {
-		return "", "", fmt.Errorf("no ready release found for application %s", cmd.Name)
-	}
 	buildType := appBuildTypeBuildpack
-	if latestAvailableRelease.Spec.ForProvider.DockerfileBuild {
+	if release.Spec.ForProvider.DockerfileBuild {
 		buildType = appBuildTypeDockerfile
 	}
-	if len(latestAvailableRelease.Status.AtProvider.ReplicaObservation) == 0 {
-		return "", buildType, fmt.Errorf("no replica information found for release %s", latestAvailableRelease.Name)
+	if len(release.Status.AtProvider.ReplicaObservation) == 0 {
+		return "", buildType, fmt.Errorf("no replica information found for release %s", release.Name)
 	}
-	if replica := readyReplica(latestAvailableRelease.Status.AtProvider.ReplicaObservation); replica != "" {
+	if replica := readyReplica(release.Status.AtProvider.ReplicaObservation); replica != "" {
 		return replica, buildType, nil
 	}
-	return "", buildType, fmt.Errorf("no ready replica found for release %s", latestAvailableRelease.Name)
+	return "", buildType, fmt.Errorf("no ready replica found for release %s", release.Name)
 }
 
 func readyReplica(replicaObs []apps.ReplicaObservation) string {
@@ -221,20 +192,6 @@ func executeRemoteCommand(ctx context.Context, params remoteCommandParameters) e
 	return tty.Safe(fn)
 }
 
-func deploioRestConfig(ctx context.Context, client *api.Client) (*rest.Config, error) {
-	config := rest.CopyConfig(client.Config)
-	deploioClusterData := &infrastructure.ClusterData{}
-	if err := client.Get(ctx, types.NamespacedName{Name: meta.ClusterDataDeploioName}, deploioClusterData); err != nil {
-		return nil, fmt.Errorf("can not gather deplo.io cluster connection details: %w", err)
-	}
-	config.Host = deploioClusterData.Status.AtProvider.APIEndpoint
-	var err error
-	if config.CAData, err = b64.StdEncoding.DecodeString(deploioClusterData.Status.AtProvider.APICACert); err != nil {
-		return nil, fmt.Errorf("can not decode deplo.io cluster CA certificate: %w", err)
-	}
-	return config, nil
-}
-
 func replicaCommand(buildType appBuildType, command []string) []string {
 	switch buildType {
 	case appBuildTypeBuildpack:

get/application.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 	"text/tabwriter"
 
@@ -12,6 +13,10 @@ import (
 	"github.com/ninech/nctl/api"
 	"github.com/ninech/nctl/api/util"
 	"github.com/ninech/nctl/internal/format"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/runtime"
+	metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1"
 )
 
 type applicationsCmd struct {
@@ -55,11 +60,25 @@ func (cmd *applicationsCmd) Run(ctx context.Context, client *api.Client, get *Cm
 		return printApplication(appList.Items, get, defaultOut(cmd.out), false)
 	case yamlOut:
 		return format.PrettyPrintObjects(appList.GetItems(), format.PrintOpts{Out: defaultOut(cmd.out)})
+	case stats:
+		return cmd.printStats(ctx, client, appList.Items, get, defaultOut(cmd.out))
 	}
 
 	return nil
 }
 
+func (cmd *applicationsCmd) Help() string {
+	return "To get an overview of the app and replica usage, use the flag '-o stats':\n" +
+		"\tREPLICA: The name of the app replica.\n" +
+		"\tSTATUS: Current status of the replica.\n" +
+		"\tCPU: Current CPU usage in millicores (1000m is a full CPU core).\n" +
+		"\tCPU%: Current CPU usage relative to the app size. This can be over 100% as Deploio allows bursting.\n" +
+		"\tMEMORY: Current Memory usage in MiB.\n" +
+		"\tMEMORY%: Current Memory relative to the app size. This can be over 100% as Deploio allows bursting.\n" +
+		"\tRESTARTS: The amount of times the replica has been restarted.\n" +
+		"\tLASTEXITCODE: The exit code the last time the replica restarted. This can give an indication on why the replica is restarting."
+}
+
 func printApplication(apps []apps.Application, get *Cmd, out io.Writer, header bool) error {
 	w := tabwriter.NewWriter(out, 0, 0, 4, ' ', 0)
 
@@ -163,3 +182,108 @@ func printDNSDetailsTabRow(items []util.DNSDetail, get *Cmd, out io.Writer) erro
 
 	return nil
 }
+
+func (cmd *applicationsCmd) printStats(ctx context.Context, c *api.Client, appList []apps.Application, get *Cmd, out io.Writer) error {
+	scheme := runtime.NewScheme()
+	if err := metricsv1beta1.AddToScheme(scheme); err != nil {
+		return err
+	}
+
+	runtimeClient, err := c.DeploioRuntimeClient(ctx, scheme)
+	if err != nil {
+		return err
+	}
+	w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0)
+	get.writeHeader(w, "NAME", "REPLICA", "STATUS", "CPU", "CPU%", "MEMORY", "MEMORY%", "RESTARTS", "LASTEXITCODE")
+
+	for _, app := range appList {
+		replicas, err := util.ApplicationReplicas(ctx, c, api.ObjectName(&app))
+		if err != nil {
+			format.PrintWarningf("unable to get replicas for app %s\n", c.Name(app.Name))
+			continue
+		}
+
+		if len(replicas) == 0 {
+			continue
+		}
+
+		for _, replica := range replicas {
+			podMetrics := metricsv1beta1.PodMetrics{}
+			if err := runtimeClient.Get(ctx, api.NamespacedName(replica.ReplicaName, app.Namespace), &podMetrics); err != nil {
+				format.PrintWarningf("unable to get metrics for replica %s\n", replica.ReplicaName)
+			}
+
+			appResources := apps.AppResources[app.Status.AtProvider.Size]
+			// We expect exactly one container, fall back to [util.NoneText] if that's
+			// not the case. The container might simply not have any metrics yet.
+			cpuUsage, cpuPercentage := util.NoneText, util.NoneText
+			memoryUsage, memoryPercentage := util.NoneText, util.NoneText
+			if len(podMetrics.Containers) == 1 {
+				cpu := podMetrics.Containers[0].Usage[corev1.ResourceCPU]
+				cpuUsage = formatQuantity(corev1.ResourceCPU, cpu)
+				cpuPercentage = formatPercentage(cpu.MilliValue(), appResources.Cpu().MilliValue())
+				memory := podMetrics.Containers[0].Usage[corev1.ResourceMemory]
+				memoryUsage = formatQuantity(corev1.ResourceMemory, memory)
+				memoryPercentage = formatPercentage(memory.MilliValue(), appResources.Memory().MilliValue())
+			}
+
+			get.writeTabRow(
+				w, c.Project, app.Name,
+				replica.ReplicaName,
+				string(replica.Status),
+				cpuUsage,
+				cpuPercentage,
+				memoryUsage,
+				memoryPercentage,
+				formatRestartCount(replica),
+				formatExitCode(replica),
+			)
+		}
+	}
+	return w.Flush()
+}
+
+// formatQuantity formats cpu/memory into human readable form. Adapted from
+// https://github.com/kubernetes/kubectl/blob/v0.31.1/pkg/metricsutil/metrics_printer.go#L209
+func formatQuantity(resourceType corev1.ResourceName, quantity resource.Quantity) string {
+	switch resourceType {
+	case corev1.ResourceCPU:
+		return fmt.Sprintf("%vm", quantity.MilliValue())
+	case corev1.ResourceMemory:
+		return fmt.Sprintf("%vMiB", quantity.Value()/toMiB(1))
+	default:
+		return fmt.Sprintf("%v", quantity.Value())
+	}
+}
+
+func formatPercentage(val, total int64) string {
+	if total == 0 {
+		return util.NoneText
+	}
+	return fmt.Sprintf("%.1f", float64(val)/float64(total)*100) + "%"
+}
+
+func toMiB(val int64) int64 {
+	return val * 1024 * 1024
+}
+
+func formatExitCode(replica apps.ReplicaObservation) string {
+	lastExitCode := util.NoneText
+
+	if replica.LastExitCode != nil {
+		lastExitCode = strconv.Itoa(int(*replica.LastExitCode))
+		// not exactly guaranteed but 137 is usually caused by the OOM killer
+		if *replica.LastExitCode == 137 {
+			lastExitCode = lastExitCode + " (Out of memory)"
+		}
+	}
+	return lastExitCode
+}
+
+func formatRestartCount(replica apps.ReplicaObservation) string {
+	restartCount := util.NoneText
+	if replica.RestartCount != nil {
+		restartCount = strconv.Itoa(int(*replica.RestartCount))
+	}
+	return restartCount
+}

get/get.go

@@ -22,7 +22,7 @@ import (
 )
 
 type Cmd struct {
-	Output              output                `help:"Configures list output. ${enum}" short:"o" enum:"full,no-header,contexts,yaml" default:"full"`
+	Output              output                `help:"Configures list output. ${enum}" short:"o" enum:"full,no-header,contexts,yaml,stats" default:"full"`
 	AllProjects         bool                  `help:"apply the get over all projects." short:"A"`
 	Clusters            clustersCmd           `cmd:"" group:"infrastructure.nine.ch" aliases:"cluster,vcluster" help:"Get Kubernetes Clusters."`
 	APIServiceAccounts  apiServiceAccountsCmd `cmd:"" group:"iam.nine.ch" name:"apiserviceaccounts" aliases:"asa" help:"Get API Service Accounts."`
@@ -51,6 +51,7 @@ const (
 	noHeader output = "no-header"
 	contexts output = "contexts"
 	yamlOut  output = "yaml"
+	stats    output = "stats"
 )
 
 type listOpt func(cmd *Cmd)