refactor: simplify the use of metrics

refactor: metrics names and use

refactor: simplify the use of metrics

refactor: metrics names and use

test: add unit test for metrics actions

fix: add create for mutatingwebhook

test: add unit test for counter

cmd/admission-controller/main.go

@@ -4,29 +4,25 @@ import (
 	"context"
 	"crypto/tls"
 	"flag"
-	"net"
 	"os"
 	"os/signal"
 	"syscall"
-	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 
 	"github.com/orange-cloudavenue/kube-image-updater/internal/httpserver"
 	client "github.com/orange-cloudavenue/kube-image-updater/internal/kubeclient"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/log"
-	"github.com/orange-cloudavenue/kube-image-updater/internal/metrics"
 )
 
 var (
 	insideCluster bool = true // running inside k8s cluster
 
-	webhookNamespace   string = "example.com"
-	webhookServiceName string = "your"
-	webhookConfigName  string = "webhookconfig"
+	webhookNamespace   string = "nip.io"
+	webhookServiceName string = "192-168-1-23"
+	webhookConfigName  string = "mutating-webhook-configuration"
 	webhookPathMutate  string = "/mutate"
 	webhookPort        string = ":8443"
 	webhookBase               = webhookServiceName + "." + webhookNamespace
@@ -36,13 +32,7 @@ var (
 	deserializer  = codecs.UniversalDeserializer()
 
 	kubeClient          client.Interface
-	manifestWebhookPath string = "./config/manifests/mutatingWebhookConfiguration.yaml"
-
-	// Prometheus metrics
-	promHTTPRequestsTotal prometheus.Counter   = metrics.NewCounter("http_requests_total", "The total number of handled HTTP requests.")
-	promHTTPErrorsTotal   prometheus.Counter   = metrics.NewCounter("http_errors_total", "The total number of handled HTTP errors.")
-	promHTTPDuration      prometheus.Histogram = metrics.NewHistogram("http_response_time_seconds", "The duration in seconds of HTTP requests.")
-	promPatchTotal        prometheus.Counter   = metrics.NewCounter("patch_total", "The total number of requests to a patch.")
+	manifestWebhookPath string = "./examples/mutatingWebhookConfiguration.yaml"
 )
 
 func init() {
@@ -96,14 +86,7 @@ func main() {
 	}
 
 	// * Config the webhook server
-	a, waitHTTP := httpserver.Init(ctx, httpserver.WithCustomHandlerForHealth(
-		func() (bool, error) {
-			_, err := net.DialTimeout("tcp", ":4444", 5*time.Second)
-			if err != nil {
-				return false, err
-			}
-			return true, nil
-		}))
+	a, waitHTTP := httpserver.Init(ctx)
 
 	s, err := a.Add("webhook", httpserver.WithTLS(tlsC), httpserver.WithAddr(webhookPort))
 	if err != nil {

cmd/admission-controller/webhook-configuration.go

@@ -49,7 +49,7 @@ func createOrUpdateMutatingWebhookConfiguration(caPEM *bytes.Buffer, webhookServ
 			Name: webhookConfigName,
 		},
 		Webhooks: []admissionregistrationv1.MutatingWebhook{{
-			Name:                    webhookService + "." + webhookNamespace + ".svc",
+			Name:                    webhookService + "." + webhookNamespace,
 			AdmissionReviewVersions: []string{"v1", "v1beta1"},
 			SideEffects:             &sideEffect,
 			ClientConfig:            clientConfig,
@@ -58,11 +58,14 @@ func createOrUpdateMutatingWebhookConfiguration(caPEM *bytes.Buffer, webhookServ
 				{
 					Operations: []admissionregistrationv1.OperationType{
 						admissionregistrationv1.Update,
+						admissionregistrationv1.Create,
 					},
 					Rule: admissionregistrationv1.Rule{
 						APIGroups:   []string{""},
 						APIVersions: []string{"v1"},
 						Resources:   []string{"pods"},
+						// TODO - add namespace scope
+						// Scope:       "*",
 					},
 				},
 			},

cmd/admission-controller/webhook.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	admissionv1 "k8s.io/api/admission/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -16,16 +15,16 @@ import (
 	"github.com/orange-cloudavenue/kube-image-updater/api/v1alpha1"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/annotations"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/log"
+	"github.com/orange-cloudavenue/kube-image-updater/internal/metrics"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/patch"
 )
 
 // func serveHandler
 func ServeHandler(w http.ResponseWriter, r *http.Request) {
-	// start the timer
-	timer := prometheus.NewTimer(promHTTPDuration)
-	defer timer.ObserveDuration()
-	// increment the totalRequests counter
-	promHTTPRequestsTotal.Inc()
+	// Prometheus metrics
+	metrics.AdmissionController().Total().Inc()
+	timeAC := metrics.AdmissionController().Duration()
+	defer timeAC.ObserveDuration()
 
 	var body []byte
 	if r.Body != nil {
@@ -34,7 +33,9 @@ func ServeHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	if len(body) == 0 {
-		promHTTPErrorsTotal.Inc()
+		// increment the total number of errors
+		metrics.AdmissionController().TotalErr().Inc()
+
 		log.Error("empty body")
 		http.Error(w, "empty body", http.StatusBadRequest)
 		return
@@ -43,15 +44,19 @@ func ServeHandler(w http.ResponseWriter, r *http.Request) {
 	// verify the content type is accurate
 	contentType := r.Header.Get("Content-Type")
 	if contentType != "application/json" {
-		promHTTPErrorsTotal.Inc()
+		// increment the total number of errors
+		metrics.AdmissionController().TotalErr().Inc()
+
 		http.Error(w, "invalid Content-Type, expect `application/json`", http.StatusUnsupportedMediaType)
 		return
 	}
 
 	var admissionResponse *admissionv1.AdmissionResponse
 	ar := admissionv1.AdmissionReview{}
 	if _, _, err := deserializer.Decode(body, nil, &ar); err != nil {
-		promHTTPErrorsTotal.Inc()
+		// increment the total number of errors
+		metrics.AdmissionController().TotalErr().Inc()
+
 		log.WithError(err).Warn("Can't decode body")
 		admissionResponse = &admissionv1.AdmissionResponse{
 			Result: &metav1.Status{
@@ -77,11 +82,15 @@ func ServeHandler(w http.ResponseWriter, r *http.Request) {
 
 	resp, err := json.Marshal(admissionReview)
 	if err != nil {
-		promHTTPErrorsTotal.Inc()
+		// increment the total number of errors
+		metrics.AdmissionController().TotalErr().Inc()
+
 		http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError)
 	}
 	if _, err := w.Write(resp); err != nil {
-		promHTTPErrorsTotal.Inc()
+		// increment the total number of errors
+		metrics.AdmissionController().TotalErr().Inc()
+
 		http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError)
 	}
 }
@@ -128,10 +137,18 @@ func mutate(ctx context.Context, ar *admissionv1.AdmissionReview) *admissionv1.A
 
 // create mutation patch for pod.
 func createPatch(ctx context.Context, pod *corev1.Pod) ([]byte, error) {
+	// Metrics - increment the total number of patch
+	metrics.AdmissionControllerPatch().Total().Inc()
+	timePatch := metrics.AdmissionControllerPatch().Duration()
+	defer timePatch.ObserveDuration()
+
 	var err error
 	// find annotation enabled
 	an := annotations.New(ctx, pod)
 	if !an.Enabled().Get() {
+		// increment the total number of errors
+		metrics.AdmissionControllerPatch().TotalErr().Inc()
+
 		return nil, fmt.Errorf("annotation not enabled")
 	}
 
@@ -154,6 +171,9 @@ func createPatch(ctx context.Context, pod *corev1.Pod) ([]byte, error) {
 			// find the image associated with the pod
 			image, err = kubeClient.Image().Find(ctx, pod.Namespace, container.Image)
 			if err != nil {
+				// increment the total number of errors
+				metrics.AdmissionControllerPatch().TotalErr().Inc()
+
 				log.
 					WithFields(logrus.Fields{
 						"Namespace": pod.Namespace,
@@ -166,6 +186,9 @@ func createPatch(ctx context.Context, pod *corev1.Pod) ([]byte, error) {
 		} else {
 			image, err = kubeClient.Image().Get(ctx, pod.Namespace, crdName)
 			if err != nil {
+				// increment the total number of errors
+				metrics.AdmissionControllerPatch().TotalErr().Inc()
+
 				log.
 					WithFields(logrus.Fields{
 						"Namespace": pod.Namespace,
@@ -179,8 +202,6 @@ func createPatch(ctx context.Context, pod *corev1.Pod) ([]byte, error) {
 		// Set the image to the pod
 		if image.ImageIsEqual(container.Image) {
 			p.AddPatch(patch.OpReplace, fmt.Sprintf("/spec/containers/%d/image", i), image.GetImageWithTag())
-			// increment the total number of patches
-			promPatchTotal.Inc()
 		}
 
 		// Annotations

cmd/kimup/scheduler.go

@@ -6,11 +6,13 @@ import (
 	"time"
 
 	"github.com/gookit/event"
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/client-go/util/retry"
 
 	"github.com/orange-cloudavenue/kube-image-updater/internal/actions"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/kubeclient"
+	"github.com/orange-cloudavenue/kube-image-updater/internal/metrics"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/models"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/registry"
 	"github.com/orange-cloudavenue/kube-image-updater/internal/rules"
@@ -28,6 +30,12 @@ func initScheduler(ctx context.Context, k kubeclient.Interface) {
 	crontab.New(ctx)
 	// Add event lock
 	event.On(triggers.RefreshImage.String(), event.ListenerFunc(func(e event.Event) (err error) {
+		// Increment the counter for the events
+		metrics.Events().Total().Inc()
+		// Start the timer for the event execution
+		timerEvents := metrics.Events().Duration()
+		defer timerEvents.ObserveDuration()
+
 		if l[e.Data()["namespace"].(string)+"/"+e.Data()["image"].(string)] == nil {
 			l[e.Data()["namespace"].(string)+"/"+e.Data()["image"].(string)] = &sync.RWMutex{}
 		}
@@ -64,6 +72,10 @@ func initScheduler(ctx context.Context, k kubeclient.Interface) {
 
 			i := utils.ImageParser(image.Spec.Image)
 
+			// Prometheus metrics - Increment the counter for the registry
+			metrics.Registry().Total().Inc()
+			timerRegistry := metrics.Registry().Duration()
+
 			re, err := registry.New(ctx, image.Spec.Image, registry.Settings{
 				InsecureTLS: image.Spec.InsecureSkipTLSVerify,
 				Username: func() string {
@@ -80,13 +92,25 @@ func initScheduler(ctx context.Context, k kubeclient.Interface) {
 				}(),
 			})
 			if err != nil {
+				// Prometheus metrics - Increment the counter for the registry with error
+				metrics.Registry().TotalErr().Inc()
+
 				return err
 			}
+			timerRegistry.ObserveDuration()
+
+			// Prometheus metrics - Increment the counter for the tags
+			metrics.Tags().Total().Inc()
+			timerTags := metrics.Tags().Duration()
 
 			tagsAvailable, err := re.Tags()
 			if err != nil {
+				// Prometheus metrics - Increment the counter for the tags with error
+				metrics.Tags().TotalErr().Inc()
+
 				return err
 			}
+			timerTags.ObserveDuration()
 
 			log.Debugf("[RefreshImage] %d tags available for %s", len(tagsAvailable), image.Spec.Image)
 
@@ -103,12 +127,23 @@ func initScheduler(ctx context.Context, k kubeclient.Interface) {
 				}
 
 				r.Init(tag, tagsAvailable, rule.Value)
+
+				// Prometheus metrics - Increment the counter for the rules
+				metrics.Rules().Total().Inc()
+				timerRules := prometheus.NewTimer(metrics.Rules().Duration())
+
 				match, newTag, err := r.Evaluate()
 				if err != nil {
+					// Prometheus metrics - Increment the counter for the evaluated rule with error
+					metrics.Rules().TotalErr().Inc()
+
 					log.Errorf("Error evaluating rule: %v", err)
 					continue
 				}
 
+				// Prometheus metrics - Observe the duration of the rule evaluation
+				timerRules.ObserveDuration()
+
 				if match {
 					for _, action := range rule.Actions {
 						a, err := actions.GetActionWithUntypedName(action.Type)
@@ -122,12 +157,22 @@ func initScheduler(ctx context.Context, k kubeclient.Interface) {
 							New:           newTag,
 							AvailableTags: tagsAvailable,
 						}, &image, action.Data)
+
+						// Prometheus metrics - Increment the counter for the actions
+						metrics.Actions().Total().Inc()
+						timerActions := metrics.Actions().Duration()
+
 						if err := a.Execute(ctx); err != nil {
+							// Prometheus metrics - Increment the counter for the executed action with error
+							metrics.Actions().TotalErr().Inc()
+
 							log.Errorf("Error executing action(%s): %v", action.Type, err)
 							continue
 						}
-					}
 
+						// Prometheus metrics - Observe the duration of the action execution
+						timerActions.ObserveDuration()
+					}
 					log.Debugf("[RefreshImage] Rule %s evaluated: %v -> %s", rule.Type, tag, newTag)
 				}
 			}
@@ -139,6 +184,8 @@ func initScheduler(ctx context.Context, k kubeclient.Interface) {
 			return nil
 		})
 
+		// Prometheus metrics - Increment the counter for the events evaluated with error
+		metrics.Events().TotalErr().Inc()
 		return retryErr
 	}), event.Normal)
 }

go.mod

@@ -66,6 +66,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect

internal/httpserver/httpserver.go

@@ -84,7 +84,7 @@ func init() {
 	// * Metrics
 	flag.Bool(models.MetricsFlagName, false, "Enable the metrics server.")
 	flag.StringVar(&metricsPort, models.MetricsPortFlagName, models.MetricsDefaultAddr, "Metrics server port.")
-	flag.StringVar(&metricsPath, models.MetricsPathFlagName, models.HealthzDefaultPath, "Metrics server path.")
+	flag.StringVar(&metricsPath, models.MetricsPathFlagName, models.MetricsDefaultPath, "Metrics server path.")
 }
 
 // Function to initialize application, return app struct and a func waitgroup.