Metrics for number of jobs per schac-home (#229)

* Metrics for number of jobs per schac-home

* Added test

* Review feedback

* Make /metrics public

src/nl/surf/eduhub_rio_mapper/api.clj

@@ -25,6 +25,7 @@
             [nl.surf.eduhub-rio-mapper.clients-info :refer [wrap-client-info]]
             [nl.surf.eduhub-rio-mapper.job :as job]
             [nl.surf.eduhub-rio-mapper.logging :refer [wrap-logging with-mdc]]
+            [nl.surf.eduhub-rio-mapper.metrics :as metrics]
             [nl.surf.eduhub-rio-mapper.ooapi :as ooapi]
             [nl.surf.eduhub-rio-mapper.rio :as rio]
             [nl.surf.eduhub-rio-mapper.status :as status]
@@ -55,6 +56,15 @@
         res
         (update res :job assoc ::job/callback-url callback-url)))))
 
+(defn wrap-metrics-getter
+  [app count-queues-fn]
+  (fn with-metrics-getter [req]
+    (let [res (app req)]
+      (cond-> res
+              (:metrics res)
+              (assoc :status http-status/ok
+                     :body (metrics/render-metrics (count-queues-fn)))))))
+
 (defn wrap-status-getter
   [app config]
   (fn with-status-getter [req]
@@ -125,6 +135,9 @@
         (GET "/status/:token" [token]
           {:token token})
 
+        (GET "/metrics" []
+          {:metrics true})
+
         (route/not-found nil))
       (compojure.core/wrap-routes wrap-uuid-validator)))
 
@@ -134,6 +147,7 @@
       (wrap-callback-extractor)
       (wrap-job-enqueuer (partial worker/enqueue! config))
       (wrap-status-getter config)
+      (wrap-metrics-getter (fn [] (metrics/count-queues #(worker/queue-counts-by-key % config))))
       (wrap-client-info clients)
       (authentication/wrap-authentication (-> (authentication/make-token-authenticator auth-config)
                                               (authentication/cache-token-authenticator {:ttl-minutes 10})))

src/nl/surf/eduhub_rio_mapper/api/authentication.clj

@@ -81,6 +81,9 @@
   (assert (< 0 ttl-minutes))
   (memo/ttl authenticator :ttl/threshold (* 1000 60 ttl-minutes)))
 
+(defn public-request? [request]
+  (= (:uri request) "/metrics"))
+
 (defn wrap-authentication
   "Authenticate calls to ring handler `f` using `token-authenticator`.
 
@@ -94,13 +97,15 @@
   response is returned."
   [f token-authenticator]
   (fn [request]
-    (if-let [token (bearer-token request)]
-      (if-let [client-id (token-authenticator token)]
-        ;; set client-id on request and response (for tracing)
-        (with-mdc {:client-id client-id}
-          (-> request
-              (assoc :client-id client-id)
-              f
-              (assoc :client-id client-id)))
-        (response/status http-status/forbidden))
-      (response/status http-status/unauthorized))))
+    (if (public-request? request)
+      (f request)
+      (if-let [token (bearer-token request)]
+        (if-let [client-id (token-authenticator token)]
+          ;; set client-id on request and response (for tracing)
+          (with-mdc {:client-id client-id}
+                    (-> request
+                        (assoc :client-id client-id)
+                        f
+                        (assoc :client-id client-id)))
+          (response/status http-status/forbidden))
+        (response/status http-status/unauthorized)))))

src/nl/surf/eduhub_rio_mapper/clients_info.clj

@@ -22,6 +22,7 @@
             [clojure.java.io :as io]
             [clojure.spec.alpha :as s]
             [nl.jomco.http-status-codes :as http-status]
+            [nl.surf.eduhub-rio-mapper.api.authentication :as authentication]
             [nl.surf.eduhub-rio-mapper.logging :refer [with-mdc]]))
 
 (s/def ::client-info
@@ -72,4 +73,6 @@
             (merge info)
             f
             (merge info)))
-      {:status http-status/forbidden})))
+      (if (authentication/public-request? request)
+        (f request)
+        {:status http-status/forbidden}))))

src/nl/surf/eduhub_rio_mapper/metrics.clj

@@ -0,0 +1,17 @@
+(ns nl.surf.eduhub-rio-mapper.metrics
+  (:require [clojure.string :as str]))
+
+(defn render-metrics [queue-count]
+  {:pre [(map? queue-count)
+         (every? string? (keys queue-count))
+         (every? integer? (vals queue-count))]}
+  (str/join "\n" (map (fn [[k v]] (format "active_and_queued_job_count{schac_home=\"%s\"} %s" k v))
+                      queue-count)))
+
+(defn count-queues [grouped-queue-counter]
+  {:post [(map? %)
+          (every? string? (keys %))
+          (every? integer? (vals %))]}
+  (merge-with +
+              (grouped-queue-counter :queue)
+              (grouped-queue-counter :busy-queue)))

src/nl/surf/eduhub_rio_mapper/redis.clj

@@ -30,6 +30,7 @@
 (defcmd del)
 (defcmd get)
 (defcmd keys)
+(defcmd llen)
 (defcmd lpop)
 (defcmd lpush)
 (defcmd lrange)

src/nl/surf/eduhub_rio_mapper/worker.clj

@@ -83,6 +83,16 @@
 (defn- busy-queue-key [config queue]
   (prefix-key config (str "busy-queue:" queue)))
 
+(defn queue-counts-by-key [query-type {:keys [redis-conn] :as config}]
+  (let [query (case query-type
+                :queue (queue-key config "*")
+                :busy-queue (busy-queue-key config "*"))
+        prefix-len (dec (count query))]
+    (->> (redis/keys redis-conn query)
+         (map (juxt #(subs % prefix-len)
+                    #(redis/llen redis-conn %)))
+         (into {}))))
+
 (defn- add-to-queue!
   [{:keys [redis-conn]
     {:keys [queue-fn

test/nl/surf/eduhub_rio_mapper/api_test.clj

@@ -135,6 +135,13 @@
   (is (= "12345678-1234-2345-3456-123456789abc"
          (-> :get (request "/status/12345678-1234-2345-3456-123456789abc") (api/routes) :token))))
 
+(deftest metrics
+  (let [app (api/wrap-metrics-getter api/routes (constantly {"foo" 1, "bar" 2}))
+        {:keys [status body]} (app (request :get "/metrics"))]
+    (is (= http-status/ok status))
+    (is (= "active_and_queued_job_count{schac_home=\"foo\"} 1\nactive_and_queued_job_count{schac_home=\"bar\"} 2"
+           body))))
+
 (deftest wrap-job-queuer
   (let [queue-atom (atom [])
         app        (api/wrap-job-enqueuer identity #(swap! queue-atom conj %))]

test/nl/surf/eduhub_rio_mapper/metrics_test.clj

@@ -0,0 +1,7 @@
+(ns nl.surf.eduhub-rio-mapper.metrics-test
+  (:require [clojure.test :refer :all]
+            [nl.surf.eduhub-rio-mapper.metrics :as metrics]))
+
+(deftest render-metrics
+  (is (= (metrics/render-metrics {"google" 12 "meta" 32})
+         "active_and_queued_job_count{schac_home=\"google\"} 12\nactive_and_queued_job_count{schac_home=\"meta\"} 32")))