From 8a1763fb1b587b646d283ec3b7fe5a31d0ba3f8b Mon Sep 17 00:00:00 2001 From: Abhishek Veeramalla Date: Thu, 1 Feb 2024 17:19:52 +0530 Subject: [PATCH] feat: Notification service monitor (#1187) * feat: Expose notifications controller metrics to prometheus monitoring --------- Signed-off-by: iam-veeramalla --- build/util/Dockerfile | 2 +- common/defaults.go | 3 + controllers/argocd/notifications.go | 82 +++++++++++++++++ controllers/argocd/notifications_test.go | 87 +++++++++++++++++++ examples/argocd-notifications.yaml | 14 ++- .../02-assert.yaml | 16 ++++ .../03-change-appset-image.yaml | 2 +- 7 files changed, 200 insertions(+), 6 deletions(-) diff --git a/build/util/Dockerfile b/build/util/Dockerfile index 3e1b13e45..46ed94f7a 100644 --- a/build/util/Dockerfile +++ b/build/util/Dockerfile @@ -48,4 +48,4 @@ ENV USER_NAME=argocd ENV HOME=/home/argocd USER argocd -WORKDIR /home/argocd \ No newline at end of file +WORKDIR /home/argocd diff --git a/common/defaults.go b/common/defaults.go index 13768aa95..ab176bab7 100644 --- a/common/defaults.go +++ b/common/defaults.go @@ -306,6 +306,9 @@ vs-ssh.visualstudio.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOf ` // OperatorMetricsPort is the port that is used to expose default controller-runtime metrics for the operator pod. OperatorMetricsPort = 8080 + + // NotificationsControllerMetricsPort is the port that is used to expose notifications controller metrics. + NotificationsControllerMetricsPort = 9001 ) // DefaultLabels returns the default set of labels for controllers. diff --git a/controllers/argocd/notifications.go b/controllers/argocd/notifications.go index b7baa3ad0..f0e46f0e5 100644 --- a/controllers/argocd/notifications.go +++ b/controllers/argocd/notifications.go @@ -6,10 +6,12 @@ import ( "reflect" "time" + monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -53,6 +55,18 @@ func (r *ReconcileArgoCD) reconcileNotificationsController(cr *argoproj.ArgoCD) return err } + log.Info("reconciling notifications metrics service") + if err := r.reconcileNotificationsMetricsService(cr); err != nil { + return err + } + + if prometheusAPIFound { + log.Info("reconciling notifications metrics service monitor") + if err := r.reconcileNotificationsServiceMonitor(cr); err != nil { + return err + } + } + return nil } @@ -82,6 +96,16 @@ func (r *ReconcileArgoCD) deleteNotificationsResources(cr *argoproj.ArgoCD) erro return err } + log.Info("reconciling notifications service") + if err := r.reconcileNotificationsMetricsService(cr); err != nil { + return err + } + + log.Info("reconciling notifications service monitor") + if err := r.reconcileNotificationsServiceMonitor(cr); err != nil { + return err + } + log.Info("reconciling notifications secret") if err := r.reconcileNotificationsSecret(cr); err != nil { return err @@ -432,6 +456,64 @@ func (r *ReconcileArgoCD) reconcileNotificationsDeployment(cr *argoproj.ArgoCD, } +// reconcileNotificationsService will ensure that the Service for the Notifications controller metrics is present. +func (r *ReconcileArgoCD) reconcileNotificationsMetricsService(cr *argoproj.ArgoCD) error { + + var component = "notifications-controller" + var suffix = "notifications-controller-metrics" + + svc := newServiceWithSuffix(suffix, component, cr) + if argoutil.IsObjectFound(r.Client, cr.Namespace, svc.Name, svc) { + // Service found, do nothing + return nil + } + + svc.Spec.Selector = map[string]string{ + common.ArgoCDKeyName: nameWithSuffix(component, cr), + } + + svc.Spec.Ports = []corev1.ServicePort{ + { + Name: "metrics", + Port: common.NotificationsControllerMetricsPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(common.NotificationsControllerMetricsPort), + }, + } + + if err := controllerutil.SetControllerReference(cr, svc, r.Scheme); err != nil { + return err + } + return r.Client.Create(context.TODO(), svc) +} + +// reconcileNotificationsServiceMonitor will ensure that the ServiceMonitor for the Notifications controller metrics is present. +func (r *ReconcileArgoCD) reconcileNotificationsServiceMonitor(cr *argoproj.ArgoCD) error { + + name := fmt.Sprintf("%s-%s", cr.Name, "notifications-controller-metrics") + serviceMonitor := newServiceMonitorWithName(name, cr) + if argoutil.IsObjectFound(r.Client, cr.Namespace, serviceMonitor.Name, serviceMonitor) { + // Service found, do nothing + return nil + } + + serviceMonitor.Spec.Selector = v1.LabelSelector{ + MatchLabels: map[string]string{ + common.ArgoCDKeyName: name, + }, + } + + serviceMonitor.Spec.Endpoints = []monitoringv1.Endpoint{ + { + Port: "metrics", + Scheme: "http", + Interval: "30s", + }, + } + + return r.Client.Create(context.TODO(), serviceMonitor) +} + // reconcileNotificationsConfigMap only creates/deletes the argocd-notifications-cm based on whether notifications is enabled/disabled in the CR // It does not reconcile/overwrite any fields or information in the configmap itself func (r *ReconcileArgoCD) reconcileNotificationsConfigMap(cr *argoproj.ArgoCD) error { diff --git a/controllers/argocd/notifications_test.go b/controllers/argocd/notifications_test.go index 6418f6fc0..5446d599d 100644 --- a/controllers/argocd/notifications_test.go +++ b/controllers/argocd/notifications_test.go @@ -2,12 +2,15 @@ package argocd import ( "context" + "fmt" "testing" + monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -252,6 +255,90 @@ func TestReconcileNotifications_CreateDeployments(t *testing.T) { assert.True(t, errors.IsNotFound(err)) } +func TestReconcileNotifications_CreateMetricsService(t *testing.T) { + a := makeTestArgoCD(func(a *argoproj.ArgoCD) { + a.Spec.Notifications.Enabled = true + }) + + resObjs := []client.Object{a} + subresObjs := []client.Object{a} + runtimeObjs := []runtime.Object{} + sch := makeTestReconcilerScheme(argoproj.AddToScheme) + cl := makeTestReconcilerClient(sch, resObjs, subresObjs, runtimeObjs) + r := makeTestReconciler(cl, sch) + + err := monitoringv1.AddToScheme(r.Scheme) + assert.NoError(t, err) + + err = r.reconcileNotificationsMetricsService(a) + assert.NoError(t, err) + + testService := &corev1.Service{} + assert.NoError(t, r.Client.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"), + Namespace: a.Namespace, + }, testService)) + + assert.Equal(t, testService.ObjectMeta.Labels["app.kubernetes.io/name"], + fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics")) + + assert.Equal(t, testService.Spec.Selector["app.kubernetes.io/name"], + fmt.Sprintf("%s-%s", a.Name, "notifications-controller")) + + assert.Equal(t, testService.Spec.Ports[0].Port, int32(9001)) + assert.Equal(t, testService.Spec.Ports[0].TargetPort, intstr.IntOrString{ + IntVal: int32(9001), + }) + assert.Equal(t, testService.Spec.Ports[0].Protocol, v1.Protocol("TCP")) + assert.Equal(t, testService.Spec.Ports[0].Name, "metrics") +} + +func TestReconcileNotifications_CreateServiceMonitor(t *testing.T) { + + a := makeTestArgoCD(func(a *argoproj.ArgoCD) { + a.Spec.Notifications.Enabled = true + }) + + resObjs := []client.Object{a} + subresObjs := []client.Object{a} + runtimeObjs := []runtime.Object{} + sch := makeTestReconcilerScheme(argoproj.AddToScheme) + monitoringv1.AddToScheme(sch) + + cl := makeTestReconcilerClient(sch, resObjs, subresObjs, runtimeObjs) + r := makeTestReconciler(cl, sch) + + // Notifications controller service monitor should not be created when Prometheus API is not found. + prometheusAPIFound = false + err := r.reconcileNotificationsController(a) + assert.NoError(t, err) + + testServiceMonitor := &monitoringv1.ServiceMonitor{} + assert.Error(t, r.Client.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"), + Namespace: a.Namespace, + }, testServiceMonitor)) + + // Prometheus API found, Verify notification controller service monitor exists. + prometheusAPIFound = true + err = r.reconcileNotificationsController(a) + assert.NoError(t, err) + + testServiceMonitor = &monitoringv1.ServiceMonitor{} + assert.NoError(t, r.Client.Get(context.TODO(), types.NamespacedName{ + Name: fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics"), + Namespace: a.Namespace, + }, testServiceMonitor)) + + assert.Equal(t, testServiceMonitor.ObjectMeta.Labels["release"], "prometheus-operator") + + assert.Equal(t, testServiceMonitor.Spec.Endpoints[0].Port, "metrics") + assert.Equal(t, testServiceMonitor.Spec.Endpoints[0].Scheme, "http") + assert.Equal(t, testServiceMonitor.Spec.Endpoints[0].Interval, "30s") + assert.Equal(t, testServiceMonitor.Spec.Selector.MatchLabels["app.kubernetes.io/name"], + fmt.Sprintf("%s-%s", a.Name, "notifications-controller-metrics")) +} + func TestReconcileNotifications_CreateSecret(t *testing.T) { logf.SetLogger(ZapLogger(true)) a := makeTestArgoCD(func(a *argoproj.ArgoCD) { diff --git a/examples/argocd-notifications.yaml b/examples/argocd-notifications.yaml index 157ab6f05..4ba6ff4d0 100644 --- a/examples/argocd-notifications.yaml +++ b/examples/argocd-notifications.yaml @@ -1,10 +1,16 @@ -apiVersion: argoproj.io/v1alpha1 +apiVersion: argoproj.io/v1beta1 kind: ArgoCD metadata: name: example-argocd + labels: + example: route spec: notifications: - enabled: True + enabled: true + prometheus: + enabled: true + route: + enabled: true server: - ingress: - enabled: true \ No newline at end of file + route: + enabled: true diff --git a/tests/k8s/1-021_validate_notification_controller/02-assert.yaml b/tests/k8s/1-021_validate_notification_controller/02-assert.yaml index 4d554aeae..4c05fb038 100644 --- a/tests/k8s/1-021_validate_notification_controller/02-assert.yaml +++ b/tests/k8s/1-021_validate_notification_controller/02-assert.yaml @@ -6,6 +6,22 @@ status: phase: Available notificationsController: Running --- +kind: Service +apiVersion: v1 +metadata: + name: example-argocd-notifications-controller-metrics +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: example-argocd-notifications-controller +status: + conditions: + - type: Available + status: 'True' + - type: Progressing + status: 'True' +--- kind: Deployment apiVersion: apps/v1 metadata: diff --git a/tests/k8s/1-027_validate_applicationset_status/03-change-appset-image.yaml b/tests/k8s/1-027_validate_applicationset_status/03-change-appset-image.yaml index 407017280..a38804ccf 100644 --- a/tests/k8s/1-027_validate_applicationset_status/03-change-appset-image.yaml +++ b/tests/k8s/1-027_validate_applicationset_status/03-change-appset-image.yaml @@ -4,4 +4,4 @@ metadata: name: example-argocd spec: applicationSet: - image: quay.io/argoproj/argocd@sha256:8576d347f30fa4c56a0129d1c0a0f5ed1e75662f0499f1ed7e917c405fd909dc \ No newline at end of file + image: quay.io/argoproj/argocd@sha256:8576d347f30fa4c56a0129d1c0a0f5ed1e75662f0499f1ed7e917c405fd909dc