From 64e5295c3136f0e4a4b42185afb6e92597243604 Mon Sep 17 00:00:00 2001 From: Michiel de Mare Date: Tue, 18 Jul 2023 10:21:06 +0200 Subject: [PATCH] Metrics for number of jobs per schac-home (#229) * Metrics for number of jobs per schac-home * Added test * Review feedback * Make /metrics public --- src/nl/surf/eduhub_rio_mapper/api.clj | 14 +++++++++++ .../eduhub_rio_mapper/api/authentication.clj | 25 +++++++++++-------- .../surf/eduhub_rio_mapper/clients_info.clj | 5 +++- src/nl/surf/eduhub_rio_mapper/metrics.clj | 17 +++++++++++++ src/nl/surf/eduhub_rio_mapper/redis.clj | 1 + src/nl/surf/eduhub_rio_mapper/worker.clj | 10 ++++++++ test/nl/surf/eduhub_rio_mapper/api_test.clj | 7 ++++++ .../surf/eduhub_rio_mapper/metrics_test.clj | 7 ++++++ 8 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 src/nl/surf/eduhub_rio_mapper/metrics.clj create mode 100644 test/nl/surf/eduhub_rio_mapper/metrics_test.clj diff --git a/src/nl/surf/eduhub_rio_mapper/api.clj b/src/nl/surf/eduhub_rio_mapper/api.clj index 460b709c..a244ce50 100644 --- a/src/nl/surf/eduhub_rio_mapper/api.clj +++ b/src/nl/surf/eduhub_rio_mapper/api.clj @@ -25,6 +25,7 @@ [nl.surf.eduhub-rio-mapper.clients-info :refer [wrap-client-info]] [nl.surf.eduhub-rio-mapper.job :as job] [nl.surf.eduhub-rio-mapper.logging :refer [wrap-logging with-mdc]] + [nl.surf.eduhub-rio-mapper.metrics :as metrics] [nl.surf.eduhub-rio-mapper.ooapi :as ooapi] [nl.surf.eduhub-rio-mapper.rio :as rio] [nl.surf.eduhub-rio-mapper.status :as status] @@ -55,6 +56,15 @@ res (update res :job assoc ::job/callback-url callback-url))))) +(defn wrap-metrics-getter + [app count-queues-fn] + (fn with-metrics-getter [req] + (let [res (app req)] + (cond-> res + (:metrics res) + (assoc :status http-status/ok + :body (metrics/render-metrics (count-queues-fn))))))) + (defn wrap-status-getter [app config] (fn with-status-getter [req] @@ -125,6 +135,9 @@ (GET "/status/:token" [token] {:token token}) + (GET "/metrics" [] + {:metrics true}) + (route/not-found nil)) (compojure.core/wrap-routes wrap-uuid-validator))) @@ -134,6 +147,7 @@ (wrap-callback-extractor) (wrap-job-enqueuer (partial worker/enqueue! config)) (wrap-status-getter config) + (wrap-metrics-getter (fn [] (metrics/count-queues #(worker/queue-counts-by-key % config)))) (wrap-client-info clients) (authentication/wrap-authentication (-> (authentication/make-token-authenticator auth-config) (authentication/cache-token-authenticator {:ttl-minutes 10}))) diff --git a/src/nl/surf/eduhub_rio_mapper/api/authentication.clj b/src/nl/surf/eduhub_rio_mapper/api/authentication.clj index 62a73790..262323e5 100644 --- a/src/nl/surf/eduhub_rio_mapper/api/authentication.clj +++ b/src/nl/surf/eduhub_rio_mapper/api/authentication.clj @@ -81,6 +81,9 @@ (assert (< 0 ttl-minutes)) (memo/ttl authenticator :ttl/threshold (* 1000 60 ttl-minutes))) +(defn public-request? [request] + (= (:uri request) "/metrics")) + (defn wrap-authentication "Authenticate calls to ring handler `f` using `token-authenticator`. @@ -94,13 +97,15 @@ response is returned." [f token-authenticator] (fn [request] - (if-let [token (bearer-token request)] - (if-let [client-id (token-authenticator token)] - ;; set client-id on request and response (for tracing) - (with-mdc {:client-id client-id} - (-> request - (assoc :client-id client-id) - f - (assoc :client-id client-id))) - (response/status http-status/forbidden)) - (response/status http-status/unauthorized)))) + (if (public-request? request) + (f request) + (if-let [token (bearer-token request)] + (if-let [client-id (token-authenticator token)] + ;; set client-id on request and response (for tracing) + (with-mdc {:client-id client-id} + (-> request + (assoc :client-id client-id) + f + (assoc :client-id client-id))) + (response/status http-status/forbidden)) + (response/status http-status/unauthorized))))) diff --git a/src/nl/surf/eduhub_rio_mapper/clients_info.clj b/src/nl/surf/eduhub_rio_mapper/clients_info.clj index 7c7820ea..9dba044d 100644 --- a/src/nl/surf/eduhub_rio_mapper/clients_info.clj +++ b/src/nl/surf/eduhub_rio_mapper/clients_info.clj @@ -22,6 +22,7 @@ [clojure.java.io :as io] [clojure.spec.alpha :as s] [nl.jomco.http-status-codes :as http-status] + [nl.surf.eduhub-rio-mapper.api.authentication :as authentication] [nl.surf.eduhub-rio-mapper.logging :refer [with-mdc]])) (s/def ::client-info @@ -72,4 +73,6 @@ (merge info) f (merge info))) - {:status http-status/forbidden}))) + (if (authentication/public-request? request) + (f request) + {:status http-status/forbidden})))) diff --git a/src/nl/surf/eduhub_rio_mapper/metrics.clj b/src/nl/surf/eduhub_rio_mapper/metrics.clj new file mode 100644 index 00000000..92500295 --- /dev/null +++ b/src/nl/surf/eduhub_rio_mapper/metrics.clj @@ -0,0 +1,17 @@ +(ns nl.surf.eduhub-rio-mapper.metrics + (:require [clojure.string :as str])) + +(defn render-metrics [queue-count] + {:pre [(map? queue-count) + (every? string? (keys queue-count)) + (every? integer? (vals queue-count))]} + (str/join "\n" (map (fn [[k v]] (format "active_and_queued_job_count{schac_home=\"%s\"} %s" k v)) + queue-count))) + +(defn count-queues [grouped-queue-counter] + {:post [(map? %) + (every? string? (keys %)) + (every? integer? (vals %))]} + (merge-with + + (grouped-queue-counter :queue) + (grouped-queue-counter :busy-queue))) diff --git a/src/nl/surf/eduhub_rio_mapper/redis.clj b/src/nl/surf/eduhub_rio_mapper/redis.clj index f7dcd836..fd4fcfeb 100644 --- a/src/nl/surf/eduhub_rio_mapper/redis.clj +++ b/src/nl/surf/eduhub_rio_mapper/redis.clj @@ -30,6 +30,7 @@ (defcmd del) (defcmd get) (defcmd keys) +(defcmd llen) (defcmd lpop) (defcmd lpush) (defcmd lrange) diff --git a/src/nl/surf/eduhub_rio_mapper/worker.clj b/src/nl/surf/eduhub_rio_mapper/worker.clj index a73dde58..8508bbf3 100644 --- a/src/nl/surf/eduhub_rio_mapper/worker.clj +++ b/src/nl/surf/eduhub_rio_mapper/worker.clj @@ -83,6 +83,16 @@ (defn- busy-queue-key [config queue] (prefix-key config (str "busy-queue:" queue))) +(defn queue-counts-by-key [query-type {:keys [redis-conn] :as config}] + (let [query (case query-type + :queue (queue-key config "*") + :busy-queue (busy-queue-key config "*")) + prefix-len (dec (count query))] + (->> (redis/keys redis-conn query) + (map (juxt #(subs % prefix-len) + #(redis/llen redis-conn %))) + (into {})))) + (defn- add-to-queue! [{:keys [redis-conn] {:keys [queue-fn diff --git a/test/nl/surf/eduhub_rio_mapper/api_test.clj b/test/nl/surf/eduhub_rio_mapper/api_test.clj index ab5a9e6e..e730a0b3 100644 --- a/test/nl/surf/eduhub_rio_mapper/api_test.clj +++ b/test/nl/surf/eduhub_rio_mapper/api_test.clj @@ -135,6 +135,13 @@ (is (= "12345678-1234-2345-3456-123456789abc" (-> :get (request "/status/12345678-1234-2345-3456-123456789abc") (api/routes) :token)))) +(deftest metrics + (let [app (api/wrap-metrics-getter api/routes (constantly {"foo" 1, "bar" 2})) + {:keys [status body]} (app (request :get "/metrics"))] + (is (= http-status/ok status)) + (is (= "active_and_queued_job_count{schac_home=\"foo\"} 1\nactive_and_queued_job_count{schac_home=\"bar\"} 2" + body)))) + (deftest wrap-job-queuer (let [queue-atom (atom []) app (api/wrap-job-enqueuer identity #(swap! queue-atom conj %))] diff --git a/test/nl/surf/eduhub_rio_mapper/metrics_test.clj b/test/nl/surf/eduhub_rio_mapper/metrics_test.clj new file mode 100644 index 00000000..336b6e25 --- /dev/null +++ b/test/nl/surf/eduhub_rio_mapper/metrics_test.clj @@ -0,0 +1,7 @@ +(ns nl.surf.eduhub-rio-mapper.metrics-test + (:require [clojure.test :refer :all] + [nl.surf.eduhub-rio-mapper.metrics :as metrics])) + +(deftest render-metrics + (is (= (metrics/render-metrics {"google" 12 "meta" 32}) + "active_and_queued_job_count{schac_home=\"google\"} 12\nactive_and_queued_job_count{schac_home=\"meta\"} 32")))