From 09d6cd0567b96c8dc5aa9076d391792d806dadd0 Mon Sep 17 00:00:00 2001 From: sethiyash Date: Wed, 6 Dec 2023 00:50:59 +0530 Subject: [PATCH] gaugevec metrics for pkg repo, app, pkg install Signed-off-by: sethiyash --- cmd/controller/run.go | 4 +++- config/config/agg-api.yml | 5 +++++ config/config/deployment.yml | 3 +++ go.mod | 2 +- pkg/app/app.go | 1 + pkg/app/app_deploy.go | 7 ------- pkg/app/app_reconcile.go | 19 ++++++++++++++++- pkg/app/app_template.go | 9 +------- pkg/metrics/app_metrics.go | 9 ++++++++ pkg/metrics/reconcile_time_metrics.go | 7 ++++++- pkg/packageinstall/app.go | 5 ++--- pkg/packageinstall/packageinstall.go | 13 ++++++++++-- .../packageinstall_deletion_test.go | 3 ++- .../packageinstall_downgrade_test.go | 10 +++++---- pkg/packageinstall/packageinstall_test.go | 13 ++++++++---- pkg/packageinstall/reconciler.go | 8 ++++--- pkg/pkgrepository/app.go | 6 ++++-- pkg/pkgrepository/app_factory.go | 3 ++- pkg/pkgrepository/app_fetch.go | 4 ---- pkg/pkgrepository/app_reconcile.go | 21 ++++++++++++++++++- pkg/pkgrepository/app_reconcile_test.go | 9 +++++--- pkg/pkgrepository/app_template.go | 6 ------ pkg/pkgrepository/crd_app.go | 4 ++-- 23 files changed, 116 insertions(+), 55 deletions(-) diff --git a/cmd/controller/run.go b/cmd/controller/run.go index f16955f1e..433f7ea8b 100644 --- a/cmd/controller/run.go +++ b/cmd/controller/run.go @@ -231,7 +231,8 @@ func Run(opts Options, runLog logr.Logger) error { pkgToPkgInstallHandler := pkginstall.NewPackageInstallVersionHandler( kcClient, opts.PackagingGlobalNS, runLog.WithName("handler")) - reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler, runLog.WithName("pkgi"), compInfo, kcConfig) + reconciler := pkginstall.NewReconciler(kcClient, pkgClient, coreClient, pkgToPkgInstallHandler, + runLog.WithName("pkgi"), compInfo, kcConfig, reconcileTimeMetrics) ctrl, err := controller.New("pkgi", mgr, controller.Options{ Reconciler: reconciler, @@ -258,6 +259,7 @@ func Run(opts Options, runLog logr.Logger) error { CoreClient: coreClient, AppClient: kcClient, KcConfig: kcConfig, + AppMetrics: appMetrics, TimeMetrics: reconcileTimeMetrics, CmdRunner: sidecarCmdExec, Kubeconf: kubeconf, diff --git a/config/config/agg-api.yml b/config/config/agg-api.yml index 5a0912822..181b9708c 100644 --- a/config/config/agg-api.yml +++ b/config/config/agg-api.yml @@ -23,5 +23,10 @@ spec: - port: 443 protocol: TCP targetPort: api + name: main + - port: 8080 + protocol: TCP + targetPort: metrics + name: metrics selector: app: kapp-controller diff --git a/config/config/deployment.yml b/config/config/deployment.yml index 11b138032..6c56e3c7f 100644 --- a/config/config/deployment.yml +++ b/config/config/deployment.yml @@ -52,6 +52,9 @@ spec: - containerPort: #@ data.values.apiPort name: api protocol: TCP + - containerPort: 8080 + name: metrics + protocol: TCP securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/go.mod b/go.mod index 677508826..00f3d49eb 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14 github.com/go-logr/logr v1.2.4 github.com/k14s/semver/v4 v4.0.1-0.20210701191048-266d47ac6115 + github.com/prometheus/client_model v0.4.0 github.com/spf13/cobra v1.6.1 golang.org/x/sync v0.2.0 gopkg.in/yaml.v2 v2.4.0 @@ -86,7 +87,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/pkg/app/app.go b/pkg/app/app.go index 678c7cca6..8e8fb586a 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -60,6 +60,7 @@ type App struct { appMetrics *metrics.AppMetrics timeMetrics *metrics.ReconcileTimeMetrics + isFirstReconcile string pendingStatusUpdate bool flushAllStatusUpdates bool metadata *deploy.Meta diff --git a/pkg/app/app_deploy.go b/pkg/app/app_deploy.go index c9c0df564..33e0dea42 100644 --- a/pkg/app/app_deploy.go +++ b/pkg/app/app_deploy.go @@ -5,8 +5,6 @@ package app import ( "fmt" - "time" - "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" ctldep "github.com/vmware-tanzu/carvel-kapp-controller/pkg/deploy" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec" @@ -14,11 +12,6 @@ import ( ) func (a *App) deploy(tplOutput string, changedFunc func(exec.CmdRunResult)) exec.CmdRunResult { - reconcileStartTS := time.Now() - defer func() { - a.timeMetrics.RegisterDeployTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS)) - }() - err := a.blockDeletion() if err != nil { return exec.NewCmdRunResultWithErr(fmt.Errorf("Blocking for deploy: %s", err)) diff --git a/pkg/app/app_reconcile.go b/pkg/app/app_reconcile.go index 03997bed2..030690ba8 100644 --- a/pkg/app/app_reconcile.go +++ b/pkg/app/app_reconcile.go @@ -104,8 +104,14 @@ func (a *App) reconcileDeploy() error { func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { reconcileStartTS := time.Now() + a.isFirstReconcile = "false" + if a.appMetrics.GetReconcileAttemptCounterValue(a.app.Name, a.app.Namespace) == 1 { + a.isFirstReconcile = "true" + } + defer func() { - a.timeMetrics.RegisterOverallTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS)) + a.timeMetrics.RegisterOverallTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile, + time.Since(reconcileStartTS)) }() tmpDir := memdir.NewTmpDir("fetch-template-deploy") @@ -134,6 +140,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.timeMetrics.RegisterFetchTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile, + a.app.Status.Fetch.UpdatedAt.Sub(a.app.Status.Fetch.StartedAt.Time)) + err := a.updateStatus("marking fetch completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -144,6 +153,8 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { } } + templateStartTime := time.Now() + tplResult := a.template(assetsPath) a.app.Status.Template = &v1alpha1.AppStatusTemplate{ @@ -153,6 +164,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.timeMetrics.RegisterTemplateTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile, + a.app.Status.Template.UpdatedAt.Sub(templateStartTime)) + err = a.updateStatus("marking template completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -201,6 +215,9 @@ func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult { }, } + a.timeMetrics.RegisterDeployTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile, + a.Status().Deploy.UpdatedAt.Sub(a.Status().Deploy.StartedAt.Time)) + return result } diff --git a/pkg/app/app_template.go b/pkg/app/app_template.go index ec8b0fd0d..6c296a694 100644 --- a/pkg/app/app_template.go +++ b/pkg/app/app_template.go @@ -5,20 +5,13 @@ package app import ( "fmt" - "strings" - "time" - "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" "github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec" ctltpl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/template" + "strings" ) func (a *App) template(dirPath string) exec.CmdRunResult { - reconcileStartTS := time.Now() - defer func() { - a.timeMetrics.RegisterTemplateTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS)) - }() - if len(a.app.Spec.Template) == 0 { return exec.NewCmdRunResultWithErr(fmt.Errorf("Expected at least one template option")) } diff --git a/pkg/metrics/app_metrics.go b/pkg/metrics/app_metrics.go index 434bae711..771e8b7ef 100644 --- a/pkg/metrics/app_metrics.go +++ b/pkg/metrics/app_metrics.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -131,3 +132,11 @@ func (am *AppMetrics) RegisterReconcileDeleteAttempt(appName string, namespace s func (am *AppMetrics) RegisterReconcileDeleteFailed(appName string, namespace string) { am.reconcileDeleteFailedTotal.WithLabelValues(appName, namespace).Inc() } + +func (am *AppMetrics) GetReconcileAttemptCounterValue(appName, namespace string) int64 { + var m = &dto.Metric{} + if err := am.reconcileAttemptTotal.WithLabelValues(appName, namespace).Write(m); err != nil { + return 0 + } + return int64(m.Counter.GetValue()) +} diff --git a/pkg/metrics/reconcile_time_metrics.go b/pkg/metrics/reconcile_time_metrics.go index 7d7a10419..5f5762030 100644 --- a/pkg/metrics/reconcile_time_metrics.go +++ b/pkg/metrics/reconcile_time_metrics.go @@ -3,6 +3,7 @@ package metrics import ( "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/metrics" + "sync" "time" ) @@ -13,6 +14,10 @@ type ReconcileTimeMetrics struct { reconcileTemplateTimeSeconds *prometheus.GaugeVec } +var ( + timeMetricsOnce sync.Once +) + func NewReconcileTimeMetrics() *ReconcileTimeMetrics { const ( metricNamespace = "kappctrl_reconcile_time_seconds" @@ -58,7 +63,7 @@ func NewReconcileTimeMetrics() *ReconcileTimeMetrics { } func (tm *ReconcileTimeMetrics) RegisterAllMetrics() { - once.Do(func() { + timeMetricsOnce.Do(func() { metrics.Registry.MustRegister( tm.reconcileTimeSeconds, tm.reconcileFetchTimeSeconds, diff --git a/pkg/packageinstall/app.go b/pkg/packageinstall/app.go index 23ce547d7..23ce2c658 100644 --- a/pkg/packageinstall/app.go +++ b/pkg/packageinstall/app.go @@ -5,9 +5,6 @@ package packageinstall import ( "fmt" - "sort" - "strings" - "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" kcv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" pkgingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" @@ -15,6 +12,8 @@ import ( "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sort" + "strings" ) const ( diff --git a/pkg/packageinstall/packageinstall.go b/pkg/packageinstall/packageinstall.go index 9745f8498..5f0d89443 100644 --- a/pkg/packageinstall/packageinstall.go +++ b/pkg/packageinstall/packageinstall.go @@ -6,6 +6,7 @@ package packageinstall import ( "context" "fmt" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "time" "github.com/go-logr/logr" @@ -54,6 +55,8 @@ type PackageInstallCR struct { coreClient kubernetes.Interface compInfo ComponentInfo opts Opts + + timeMetrics *metrics.ReconcileTimeMetrics } // nolint: revive @@ -62,10 +65,11 @@ type Opts struct { } func NewPackageInstallCR(model *pkgingv1alpha1.PackageInstall, log logr.Logger, - kcclient kcclient.Interface, pkgclient pkgclient.Interface, coreClient kubernetes.Interface, compInfo ComponentInfo, opts Opts) *PackageInstallCR { + kcclient kcclient.Interface, pkgclient pkgclient.Interface, coreClient kubernetes.Interface, + compInfo ComponentInfo, opts Opts, timeMetrics *metrics.ReconcileTimeMetrics) *PackageInstallCR { return &PackageInstallCR{model: model, unmodifiedModel: model.DeepCopy(), log: log, - kcclient: kcclient, pkgclient: pkgclient, coreClient: coreClient, compInfo: compInfo, opts: opts} + kcclient: kcclient, pkgclient: pkgclient, coreClient: coreClient, compInfo: compInfo, opts: opts, timeMetrics: timeMetrics} } func (pi *PackageInstallCR) Reconcile() (reconcile.Result, error) { @@ -101,6 +105,11 @@ func (pi *PackageInstallCR) Reconcile() (reconcile.Result, error) { func (pi *PackageInstallCR) reconcile(modelStatus *reconciler.Status) (reconcile.Result, error) { pi.log.Info("Reconciling") + reconcileStartTS := time.Now() + defer func() { + pi.timeMetrics.RegisterOverallTime("pkgi", pi.model.Name, pi.model.Namespace, "", time.Since(reconcileStartTS)) + }() + err := pi.blockDeletion() if err != nil { return reconcile.Result{Requeue: true}, err diff --git a/pkg/packageinstall/packageinstall_deletion_test.go b/pkg/packageinstall/packageinstall_deletion_test.go index 0ef39fc80..0cdfd9b0c 100644 --- a/pkg/packageinstall/packageinstall_deletion_test.go +++ b/pkg/packageinstall/packageinstall_deletion_test.go @@ -4,6 +4,7 @@ package packageinstall import ( + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "testing" "github.com/k14s/semver/v4" @@ -65,7 +66,7 @@ func Test_PackageInstallDeletion(t *testing.T) { appClient := fakekappctrl.NewSimpleClientset(pkgInstall, existingApp) coreClient := fake.NewSimpleClientset() - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, coreClient, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, coreClient, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, &metrics.ReconcileTimeMetrics{}) _, err := ip.Reconcile() assert.Nil(t, err) diff --git a/pkg/packageinstall/packageinstall_downgrade_test.go b/pkg/packageinstall/packageinstall_downgrade_test.go index 7810d6f32..776f106e2 100644 --- a/pkg/packageinstall/packageinstall_downgrade_test.go +++ b/pkg/packageinstall/packageinstall_downgrade_test.go @@ -4,6 +4,7 @@ package packageinstall import ( + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "testing" "github.com/k14s/semver/v4" @@ -25,6 +26,7 @@ import ( func Test_PackageInstallVersionDowngrades(t *testing.T) { log := logf.Log.WithName("kc") + var reconcileTimeMetrics = metrics.NewReconcileTimeMetrics() pkg1 := datapkgingv1alpha1.Package{ ObjectMeta: metav1.ObjectMeta{ @@ -98,7 +100,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics) _, err := ip.Reconcile() assert.Nil(t, err) @@ -147,7 +149,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics) _, err := ip.Reconcile() assert.Nil(t, err) @@ -196,7 +198,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics) _, err := ip.Reconcile() assert.Nil(t, err) @@ -251,7 +253,7 @@ func Test_PackageInstallVersionDowngrades(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(pkgInstall, log, appClient, pkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, reconcileTimeMetrics) _, err := ip.Reconcile() assert.Nil(t, err) diff --git a/pkg/packageinstall/packageinstall_test.go b/pkg/packageinstall/packageinstall_test.go index 174eb302b..0fc71cb75 100644 --- a/pkg/packageinstall/packageinstall_test.go +++ b/pkg/packageinstall/packageinstall_test.go @@ -5,6 +5,7 @@ package packageinstall import ( "fmt" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "reflect" "testing" @@ -353,6 +354,7 @@ func Test_PackageRefUsesName(t *testing.T) { } func Test_PlaceHolderSecretCreated_WhenPackageHasNoSecretRef(t *testing.T) { + var timeMetrics = metrics.NewReconcileTimeMetrics() pkg := datapkgingv1alpha1.Package{ ObjectMeta: metav1.ObjectMeta{ Name: "expected-pkg", @@ -402,7 +404,7 @@ func Test_PlaceHolderSecretCreated_WhenPackageHasNoSecretRef(t *testing.T) { GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, timeMetrics) _, err := ip.Reconcile() assert.Nil(t, err) @@ -428,6 +430,7 @@ func Test_PlaceHolderSecretCreated_WhenPackageHasNoSecretRef(t *testing.T) { } func Test_PlaceHolderSecretsCreated_WhenPackageHasMultipleFetchStages(t *testing.T) { + var timeMetrics = metrics.NewReconcileTimeMetrics() pkg := datapkgingv1alpha1.Package{ ObjectMeta: metav1.ObjectMeta{ Name: "expected-pkg", @@ -481,7 +484,7 @@ func Test_PlaceHolderSecretsCreated_WhenPackageHasMultipleFetchStages(t *testing GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, timeMetrics) _, err := ip.Reconcile() assert.Nil(t, err) @@ -517,6 +520,7 @@ func Test_PlaceHolderSecretsCreated_WhenPackageHasMultipleFetchStages(t *testing } func Test_PlaceHolderSecretsNotCreated_WhenFetchStagesHaveSecrets(t *testing.T) { + var timeMetrics = metrics.NewReconcileTimeMetrics() pkg := datapkgingv1alpha1.Package{ ObjectMeta: metav1.ObjectMeta{ Name: "expected-pkg", @@ -571,7 +575,7 @@ func Test_PlaceHolderSecretsNotCreated_WhenFetchStagesHaveSecrets(t *testing.T) GitVersion: "v0.20.0", } - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, timeMetrics) _, err := ip.Reconcile() assert.Nil(t, err) @@ -591,6 +595,7 @@ func Test_PlaceHolderSecretsNotCreated_WhenFetchStagesHaveSecrets(t *testing.T) } func Test_PlaceHolderSecretCreated_WhenPackageInstallUpdated(t *testing.T) { + var timeMetrics = metrics.NewReconcileTimeMetrics() appSpec := v1alpha1.AppSpec{ Fetch: []v1alpha1.AppFetch{ { @@ -648,7 +653,7 @@ func Test_PlaceHolderSecretCreated_WhenPackageInstallUpdated(t *testing.T) { fakekctrl := fakekappctrl.NewSimpleClientset(model, existingApp) fakek8s := fake.NewSimpleClientset() - ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}) + ip := NewPackageInstallCR(model, log, fakekctrl, fakePkgClient, fakek8s, FakeComponentInfo{KCVersion: semver.MustParse("0.42.31337")}, Opts{}, timeMetrics) // mock the kubernetes server version fakeDiscovery, _ := fakek8s.Discovery().(*fakediscovery.FakeDiscovery) diff --git a/pkg/packageinstall/reconciler.go b/pkg/packageinstall/reconciler.go index 451106abd..f1ba01e3e 100644 --- a/pkg/packageinstall/reconciler.go +++ b/pkg/packageinstall/reconciler.go @@ -6,7 +6,6 @@ package packageinstall import ( "context" "fmt" - "github.com/go-logr/logr" kappctrlv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" pkgingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" @@ -14,6 +13,7 @@ import ( pkgclient "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/client/clientset/versioned" kcclient "github.com/vmware-tanzu/carvel-kapp-controller/pkg/client/clientset/versioned" kcconfig "github.com/vmware-tanzu/carvel-kapp-controller/pkg/config" + "github.com/vmware-tanzu/carvel-kapp-controller/pkg/metrics" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -33,12 +33,13 @@ type Reconciler struct { compInfo ComponentInfo log logr.Logger kcConfig *kcconfig.Config + timeMetrics *metrics.ReconcileTimeMetrics } // NewReconciler is the constructor for the Reconciler struct func NewReconciler(kcClient kcclient.Interface, pkgClient pkgclient.Interface, coreClient kubernetes.Interface, pkgToPkgInstallHandler *PackageInstallVersionHandler, - log logr.Logger, compInfo ComponentInfo, kcConfig *kcconfig.Config) *Reconciler { + log logr.Logger, compInfo ComponentInfo, kcConfig *kcconfig.Config, timeMetrics *metrics.ReconcileTimeMetrics) *Reconciler { return &Reconciler{kcClient: kcClient, pkgClient: pkgClient, @@ -47,6 +48,7 @@ func NewReconciler(kcClient kcclient.Interface, pkgClient pkgclient.Interface, compInfo: compInfo, log: log, kcConfig: kcConfig, + timeMetrics: timeMetrics, } } @@ -88,5 +90,5 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{}, err } - return NewPackageInstallCR(existingPkgInstall, log, r.kcClient, r.pkgClient, r.coreClient, r.compInfo, Opts{DefaultSyncPeriod: r.kcConfig.PackageInstallDefaultSyncPeriod()}).Reconcile() + return NewPackageInstallCR(existingPkgInstall, log, r.kcClient, r.pkgClient, r.coreClient, r.compInfo, Opts{DefaultSyncPeriod: r.kcConfig.PackageInstallDefaultSyncPeriod()}, r.timeMetrics).Reconcile() } diff --git a/pkg/pkgrepository/app.go b/pkg/pkgrepository/app.go index 4786e5e84..3a03920be 100644 --- a/pkg/pkgrepository/app.go +++ b/pkg/pkgrepository/app.go @@ -31,19 +31,21 @@ type App struct { deployFactory deploy.Factory timeMetrics *metrics.ReconcileTimeMetrics + appMetrics *metrics.AppMetrics log logr.Logger + isFirstReconcile string pendingStatusUpdate bool flushAllStatusUpdates bool } // NewApp creates a new instance of an App based on v1alpha1.App func NewApp(app v1alpha1.App, hooks Hooks, fetchFactory fetch.Factory, templateFactory template.Factory, deployFactory deploy.Factory, - log logr.Logger, timeMetrics *metrics.ReconcileTimeMetrics, pkgRepoUID types.UID) *App { + log logr.Logger, appMetrics *metrics.AppMetrics, timeMetrics *metrics.ReconcileTimeMetrics, pkgRepoUID types.UID) *App { return &App{app: app, appPrev: *(app.DeepCopy()), hooks: hooks, fetchFactory: fetchFactory, templateFactory: templateFactory, - deployFactory: deployFactory, log: log, timeMetrics: timeMetrics, pkgRepoUID: pkgRepoUID} + deployFactory: deployFactory, log: log, appMetrics: appMetrics, timeMetrics: timeMetrics, pkgRepoUID: pkgRepoUID} } func (a *App) Name() string { return a.app.Name } diff --git a/pkg/pkgrepository/app_factory.go b/pkg/pkgrepository/app_factory.go index faeb71b60..379d1c849 100644 --- a/pkg/pkgrepository/app_factory.go +++ b/pkg/pkgrepository/app_factory.go @@ -26,6 +26,7 @@ type AppFactory struct { CoreClient kubernetes.Interface AppClient kcclient.Interface TimeMetrics *metrics.ReconcileTimeMetrics + AppMetrics *metrics.AppMetrics KcConfig *config.Config CmdRunner exec.CmdRunner Kubeconf *kubeconfig.Kubeconfig @@ -41,5 +42,5 @@ func (f *AppFactory) NewCRDPackageRepo(app *kcv1alpha1.App, pkgr *pkgv1alpha1.Pa fetchFactory := fetch.NewFactory(f.CoreClient, vendirOpts, f.CmdRunner) templateFactory := template.NewFactory(f.CoreClient, fetchFactory, false, f.CmdRunner) deployFactory := deploy.NewFactory(f.CoreClient, f.Kubeconf, nil, f.CmdRunner, log) - return NewCRDApp(app, pkgr, log, f.AppClient, f.TimeMetrics, fetchFactory, templateFactory, deployFactory) + return NewCRDApp(app, pkgr, log, f.AppClient, f.TimeMetrics, f.AppMetrics, fetchFactory, templateFactory, deployFactory) } diff --git a/pkg/pkgrepository/app_fetch.go b/pkg/pkgrepository/app_fetch.go index 495f3b1b5..86839d787 100644 --- a/pkg/pkgrepository/app_fetch.go +++ b/pkg/pkgrepository/app_fetch.go @@ -14,10 +14,6 @@ import ( ) func (a *App) fetch(dstPath string) (string, exec.CmdRunResult) { - reconcileStartTS := time.Now() - defer func() { - a.timeMetrics.RegisterFetchTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS)) - }() if len(a.app.Spec.Fetch) == 0 { return "", exec.NewCmdRunResultWithErr(fmt.Errorf("Expected at least one fetch option")) } diff --git a/pkg/pkgrepository/app_reconcile.go b/pkg/pkgrepository/app_reconcile.go index bee2bb034..66461214e 100644 --- a/pkg/pkgrepository/app_reconcile.go +++ b/pkg/pkgrepository/app_reconcile.go @@ -21,6 +21,8 @@ func (a *App) Reconcile(force bool) (reconcile.Result, error) { var err error + a.appMetrics.InitMetrics(a.Name(), a.Namespace()) + switch { case a.app.Spec.Paused: a.log.Info("PackageRepository is paused, not reconciling") @@ -91,8 +93,12 @@ func (a *App) reconcileDeploy() error { func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { reconcileStartTS := time.Now() + a.isFirstReconcile = "false" + if a.appMetrics.GetReconcileAttemptCounterValue(a.app.Name, a.app.Namespace) == 1 { + a.isFirstReconcile = "true" + } defer func() { - a.timeMetrics.RegisterOverallTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS)) + a.timeMetrics.RegisterOverallTime(a.app.Kind, a.app.Name, a.app.Namespace, a.isFirstReconcile, time.Since(reconcileStartTS)) }() tmpDir := memdir.NewTmpDir("fetch-template-deploy") @@ -121,6 +127,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.timeMetrics.RegisterFetchTime("pkgr", a.app.Name, a.app.Namespace, a.isFirstReconcile, + a.app.Status.Fetch.UpdatedAt.Sub(a.app.Status.Fetch.StartedAt.Time)) + err := a.updateStatus("marking fetch completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -131,6 +140,8 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { } } + templateStartTime := time.Now() + tplResult := a.template(assetsPath) a.app.Status.Template = &v1alpha1.AppStatusTemplate{ @@ -140,6 +151,9 @@ func (a *App) reconcileFetchTemplateDeploy() exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.timeMetrics.RegisterTemplateTime("pkgr", a.app.Name, a.app.Namespace, a.isFirstReconcile, + a.app.Status.Template.UpdatedAt.Sub(templateStartTime)) + err = a.updateStatus("marking template completed") if err != nil { return exec.NewCmdRunResultWithErr(err) @@ -167,6 +181,9 @@ func (a *App) updateLastDeploy(result exec.CmdRunResult) exec.CmdRunResult { UpdatedAt: metav1.NewTime(time.Now().UTC()), } + a.timeMetrics.RegisterDeployTime("pkgr", a.app.Name, a.app.Namespace, a.isFirstReconcile, + a.Status().Deploy.UpdatedAt.Sub(a.Status().Deploy.StartedAt.Time)) + a.updateStatus("marking last deploy") return result @@ -198,6 +215,8 @@ func (a *App) setReconciling() { Status: corev1.ConditionTrue, }) + a.appMetrics.RegisterReconcileAttempt(a.app.Name, a.app.Namespace) + a.app.Status.FriendlyDescription = "Reconciling" } diff --git a/pkg/pkgrepository/app_reconcile_test.go b/pkg/pkgrepository/app_reconcile_test.go index e9268c8b1..78ebf7726 100644 --- a/pkg/pkgrepository/app_reconcile_test.go +++ b/pkg/pkgrepository/app_reconcile_test.go @@ -26,6 +26,7 @@ import ( func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) { log := logf.Log.WithName("kc") var timeMetrics = metrics.NewReconcileTimeMetrics() + var appMetrics = metrics.NewAppMetrics() // The url under fetch is invalid, which will cause this // app to fail before deploy. @@ -48,7 +49,7 @@ func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) { deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) pkgr := v1alpha12.PackageRepository{} - crdApp := NewCRDApp(&app, &pkgr, log, kappcs, timeMetrics, fetchFac, tmpFac, deployFac) + crdApp := NewCRDApp(&app, &pkgr, log, kappcs, timeMetrics, appMetrics, fetchFac, tmpFac, deployFac) _, err := crdApp.Reconcile(false) if err != nil { t.Fatalf("Unexpected error with reconciling: %v", err) @@ -81,6 +82,7 @@ func Test_NoInspectReconcile_IfNoDeployAttempted(t *testing.T) { func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing.T) { log := logf.Log.WithName("kc") var timeMetrics = metrics.NewReconcileTimeMetrics() + var appMetrics = metrics.NewAppMetrics() fetchInline := map[string]string{ "packages/file.yml": `foo: #@ data.values.nothere`, @@ -105,7 +107,7 @@ func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing. deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) pkgr := v1alpha12.PackageRepository{} - crdApp := NewCRDApp(&app, &pkgr, log, kappcs, timeMetrics, fetchFac, tmpFac, deployFac) + crdApp := NewCRDApp(&app, &pkgr, log, kappcs, timeMetrics, appMetrics, fetchFac, tmpFac, deployFac) _, err := crdApp.Reconcile(false) if err != nil { t.Fatalf("Unexpected error with reconciling: %v", err) @@ -139,6 +141,7 @@ func Test_TemplateError_DisplayedInStatus_UsefulErrorMessageProperty(t *testing. func TestInvalidPackageRepositoryFormat(t *testing.T) { log := logf.Log.WithName("kc") var timeMetrics = metrics.NewReconcileTimeMetrics() + var appMetrics = metrics.NewAppMetrics() fetchInline := map[string]string{ "file.yml": `foo: hi`, @@ -162,7 +165,7 @@ func TestInvalidPackageRepositoryFormat(t *testing.T) { deployFac := deploy.NewFactory(k8scs, kubeconfig.NewKubeconfig(k8scs, log), nil, exec.NewPlainCmdRunner(), log) pkgr := v1alpha12.PackageRepository{} - crdApp := NewCRDApp(&app, &pkgr, log, kappcs, timeMetrics, fetchFac, tmpFac, deployFac) + crdApp := NewCRDApp(&app, &pkgr, log, kappcs, timeMetrics, appMetrics, fetchFac, tmpFac, deployFac) _, err := crdApp.Reconcile(false) if err != nil { t.Fatalf("Unexpected error with reconciling: %v", err) diff --git a/pkg/pkgrepository/app_template.go b/pkg/pkgrepository/app_template.go index f98761764..241cee7ee 100644 --- a/pkg/pkgrepository/app_template.go +++ b/pkg/pkgrepository/app_template.go @@ -11,7 +11,6 @@ import ( "os" "path/filepath" "strings" - "time" kcv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" datapackagingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" @@ -26,11 +25,6 @@ import ( var errInvalidPackageRepo = exec.NewCmdRunResultWithErr(fmt.Errorf("Invalid package repository content: must contain 'packages/' directory but did not")) func (a *App) template(dirPath string) exec.CmdRunResult { - reconcileStartTS := time.Now() - defer func() { - a.timeMetrics.RegisterTemplateTime(a.app.Kind, a.app.Name, a.app.Namespace, "", time.Since(reconcileStartTS)) - }() - fileInfo, err := os.Lstat(filepath.Join(dirPath, "packages")) if err != nil { if os.IsNotExist(err) { diff --git a/pkg/pkgrepository/crd_app.go b/pkg/pkgrepository/crd_app.go index 5f02ae5d7..a99c99d1d 100644 --- a/pkg/pkgrepository/crd_app.go +++ b/pkg/pkgrepository/crd_app.go @@ -30,7 +30,7 @@ type CRDApp struct { } func NewCRDApp(appModel *kcv1alpha1.App, packageRepo *pkgingv1alpha1.PackageRepository, log logr.Logger, - appClient kcclient.Interface, timeMetrics *metrics.ReconcileTimeMetrics, fetchFactory fetch.Factory, + appClient kcclient.Interface, timeMetrics *metrics.ReconcileTimeMetrics, appMetrics *metrics.AppMetrics, fetchFactory fetch.Factory, templateFactory template.Factory, deployFactory deploy.Factory) *CRDApp { crdApp := &CRDApp{appModel: appModel, pkgrModel: packageRepo, log: log, appClient: appClient} @@ -39,7 +39,7 @@ func NewCRDApp(appModel *kcv1alpha1.App, packageRepo *pkgingv1alpha1.PackageRepo BlockDeletion: crdApp.blockDeletion, UnblockDeletion: crdApp.unblockDeletion, UpdateStatus: crdApp.updateStatus, - }, fetchFactory, templateFactory, deployFactory, log, timeMetrics, packageRepo.UID) + }, fetchFactory, templateFactory, deployFactory, log, appMetrics, timeMetrics, packageRepo.UID) return crdApp }