Skip to content

Commit

Permalink
Implement request limits
Browse files Browse the repository at this point in the history 
Two options added to main:

--max-total-requests
--max-requests-per-operation

This required a fairly large refactoring of the spidering
code. Removed all non-essential code from the base spider library.
  • Loading branch information
@joodie
joodie committed Nov 1, 2023
1 parent 4c4929e commit 1052fa3
Show file tree
Hide file tree
Showing 4 changed files with 170 additions and 65 deletions.
22 changes: 11 additions & 11 deletions deps.edn
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
{:paths ["src" "assets" "profiles"]
:deps
{org.clojure/tools.cli {:mvn/version "1.0.219"}
org.babashka/http-client {:mvn/version "0.4.15"}
hiccup/hiccup {:mvn/version "2.0.0-RC2"}
nl.jomco/clj-http-status-codes {:mvn/version "0.1"}
nl.jomco/openapi-v3-validator {:mvn/version "0.2.0"}
nl.jomco/spider {:mvn/version "0.1.0"}}

{:paths ["src" "assets" "profiles"]
:deps {org.clojure/tools.cli {:mvn/version "1.0.219"}
ring/ring-core {:mvn/version "1.10.0"}
org.babashka/http-client {:mvn/version "0.4.15"}
org.clojure/data.json {:mvn/version "2.4.0"}
hiccup/hiccup {:mvn/version "2.0.0-RC2"}
nl.jomco/clj-http-status-codes {:mvn/version "0.1"}
nl.jomco/openapi-v3-validator {:mvn/version "0.2.1"}
nl.jomco/spider {:mvn/version "0.2.0"}}
:aliases {:test {:extra-deps {lambdaisland/kaocha {:mvn/version "RELEASE"}}
:main-opts ["-m" "kaocha.runner"]}
:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "RELEASE"}}
:main-opts ["-m" "clj-kondo.main"]}
:main-opts ["-m" "clj-kondo.main"]}
:outdated {:replace-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}}
:main-opts ["-m" "antq.core"]}}}
:main-opts ["-m" "antq.core"]}}}
24 changes: 21 additions & 3 deletions src/nl/jomco/eduhub_validator/main.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@
["-b" "--bearer-token TOKEN"
"Add bearer token to request."
:default nil]
["-M" "--max-total-requests N"
"Maximum number of requests."
:default ##Inf
:parse-fn parse-long]
["-m" "--max-requests-per-operation N"
"Maximum number of requests per operation in OpenAPI spec."
:default ##Inf
:parse-fn parse-long]
["-a" "--basic-auth 'USER:PASS'" "Send basic authentication header."
:default nil
:parse-fn (fn [s]
Expand All @@ -65,6 +73,7 @@
Returns nil if neither resource or file are present."
[f]
(assert f)
(let [file (io/file f)]
(if (.exists file)
file
Expand All @@ -90,17 +99,17 @@
(println "Spidering" base-url)
(with-open [w (io/writer observations-path :encoding "UTF-8")]
(.write w "[")
(run! #(do (println (:url (:request %)))
(run! #(do (println (:status (:response %)) (name (:method (:request %))) (:uri (:request %)))
(pprint/pprint % w)) (spider/spider-and-validate spec-data rules-data options))
(.write w "]")))

(defn- report
[spec-data {:keys [observations-path report-path]}]
[spec-data {:keys [observations-path report-path base-url]}]
(println "Writing report to" report-path)
(binding [*out* (io/writer report-path :encoding "UTF-8")]
(println
;; str needed to coerce hiccup "rawstring"
(str (report/report spec-data (read-edn observations-path))))))
(str (report/report spec-data (read-edn observations-path) base-url)))))

(defn -main
[& args]
Expand Down Expand Up @@ -128,3 +137,12 @@
(spider spec-data profile-data options))
(when-not no-report?
(report spec-data options)))))

(comment

(-main "-M" "5" "-u" "https://demo04.test.surfeduhub.nl/")

(def ooapi-rules (read-edn (file-or-resource "rio")))
(def ooapi-spec (read-json (file-or-resource (:openapi-spec ooapi-rules))))

(def interactions (spider/spider-and-validate ooapi-spec ooapi-rules {:base-url "https://demo04.test.surfeduhub.nl"})))
36 changes: 18 additions & 18 deletions src/nl/jomco/eduhub_validator/report.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
(filter :issues interactions))

(defn- score-percent [interactions]
(* 100.0 (/ (count (with-issues interactions))
(count interactions))))
(if (empty? interactions)
100.0
(* 100.0 (/ (count (with-issues interactions))
(count interactions)))))

(defn- score-summary [interactions]
(let [score (score-percent interactions)]
Expand All @@ -31,29 +33,22 @@
(count (with-issues interactions))))]))


(def report-title "SURFeduhub validation report")
(def css-resource "style.css")

(defn- interactions-summary [interactions]
(let [server-names (->> interactions
(map :request)
(map :server-name)
set)
start-at (->> interactions
(let [responses (->> interactions
(map :response))
start-at (->> responses
(map :start-at)
(sort)
(first))
finish-at (->> interactions
finish-at (->> responses
(map :finish-at)
(sort)
(last))]
[:section.summary
[:h3 "Summary"]
[:dl
[:div
[:dt "Server" (if (> (count server-names) 1) "s" "")]
[:dd (interpose ", "
(map #(vector :strong %) (sort server-names)))]]
[:div
[:dt "Run time"]
[:dd "From "[:strong start-at] " till " [:strong finish-at]]]]]))
Expand All @@ -74,11 +69,12 @@
[:dt "Validation score"]
[:dd (score-summary interactions)]]])])

(defn- interaction-summary [{{:keys [method url]} :request}]
(defn- interaction-summary [{{:keys [method uri query-string]} :request}]
[:span.interaction-summary
[:code.method (string/upper-case (name method))]
" "
[:code.url url]])
[:code.url uri (when query-string
(str "?" query-string))]])

(defn- value-type [v]
(cond
Expand Down Expand Up @@ -378,15 +374,19 @@
(defn- raw-css [css]
(hiccup.util/raw-string "/*<![CDATA[*/\n" css "/*]]>*/"))

(defn report-title
[base-url]
(str "Validation report for " base-url))

(defn report
[openapi interactions]
[openapi interactions base-url]
(hiccup.page/html5
[:html
[:head [:title report-title]
[:head [:title (report-title base-url)]
[:style (-> css-resource (io/resource) (slurp) (raw-css))]]
[:body
[:header
[:h1 report-title]]
[:h1 (report-title base-url)]]

[:main
[:section.general
Expand Down
153 changes: 120 additions & 33 deletions src/nl/jomco/eduhub_validator/spider.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,26 @@
(:require [nl.jomco.openapi.v3.path-matcher :refer [paths-matcher]]
[nl.jomco.openapi.v3.validator :as validator]
[nl.jomco.spider :as spider]
[clojure.string :as string]
[clojure.data.json :as json]
[babashka.http-client :as http-client]
[ring.middleware.params :as params]))

(defn fixup-request
"Convert spider request to format expected by validator"
[{:keys [path] :as request}]
(let [[uri query-string] (string/split path #"\?" 2)]
(-> request
(dissoc :path)
(assoc :uri uri
:query-string query-string)
(params/assoc-query-params "UTF-8"))))

(defn fixup-interaction
"Convert spider interaction to format expected by validator."
[interaction]
(-> interaction
(update :request #(params/assoc-query-params % "UTF-8"))
(update :request fixup-request)
;; remove java.net.URI - not needed and doesn't print cleanly
(update :response dissoc :uri)))

Expand All @@ -34,40 +47,114 @@
(merge-with merge-header-values headers1 headers2))

(defn wrap-timing
"Middleware adding timing info to interactor"
[interactor]
"Middleware adding timing info to exec-request"
[f]
(fn [request]
(let [start-at (java.util.Date.)]
(assoc (interactor request)
(let [start-at (java.util.Date.)]
(assoc (f request)
:start-at start-at
:finish-at (java.util.Date.)))))

(defn wrap-update-req
"Middleware for updating request prior to sending"
[interactor update-fn & args]
[f update-fn & args]
(fn [request]
(interactor (apply update-fn request args))))
(f (apply update-fn request args))))

(defn wrap-headers
[interactor headers]
(wrap-update-req interactor update :headers
[f headers]
(wrap-update-req f update :headers
(fn [existing-headers]
(merge-with merge-header-values existing-headers headers))))

(defn wrap-basic-auth
[interactor basic-auth]
(wrap-update-req interactor assoc :basic-auth basic-auth))
[f basic-auth]
(wrap-update-req f assoc :basic-auth basic-auth))

(defn wrap-bearer-token
[interactor token]
(wrap-headers interactor {:authorization (str "Bearer " token)}))
[f token]
(wrap-headers f {:authorization (str "Bearer " token)}))

(defn fixup-babashka-request
[req]
(-> req
(assoc :uri (select-keys req [:host :port :path :scheme]))
(dissoc :host :port :path :scheme)))

(defn fixup-validator-request
[req]
(assoc req :uri (:path req)))

(defn wrap-client
[f client]
(fn [request]
(-> request
(fixup-babashka-request)
(assoc :client client
:throw false)
(f)
(dissoc :request))))

(defn wrap-base-url
"Ensure base-url info is added to request."
[f base-url]
(let [u (java.net.URL. base-url)
base-request {:host (.getHost u)
:port (.getPort u)
:path "/"
:scheme (.getProtocol u)}
base-path (.getPath u)
path-prefix (if (= base-path "/")
""
base-path)]
(fn [{:keys [path] :as request}]
(f (-> base-request
(merge request)
(assoc :path (str path-prefix path)))))))

(defn wrap-json-body
[f]
(fn [request]
(let [response (f request)]
(if (string/starts-with? (get-in response [:headers "content-type"]) "application/json")
(update response :body json/read-str)
response))))

(defn wrap-max-requests-per-operation
[f max-requests-per-operation op-path]
(let [counters (atom {})]
(fn [request]
(let [operation (op-path request)
n (get @counters operation 0)]
(if (< n max-requests-per-operation)
(do (swap! counters update operation (fnil inc 0))
(f request))
::spider/skip)))))

(defn wrap-max-requests
[f max-requests]
(let [counter (atom 0)]
(fn [request]
(if (< @counter max-requests)
(do (swap! counter inc)
(f request))
::spider/skip))))

(defn mk-exec-request
[{:keys [base-url headers basic-auth bearer-token]}]
(cond-> http-client/request
:always
(wrap-client (http-client/client (assoc http-client/default-client-opts
:follow-redirects :never)))
:always
(wrap-base-url base-url)

(defn mk-interact
[{:keys [headers basic-auth bearer-token]}]
(cond-> spider/interact
:always
(wrap-timing)

:always
(wrap-json-body)

(seq headers)
(wrap-headers headers)

Expand All @@ -77,27 +164,27 @@
bearer-token
(wrap-bearer-token bearer-token)))

(defn path
[{:keys [uri path url]}]
(or path uri (.getPath (java.net.URI/create url))))

(defn spider-and-validate
[openapi-spec {:keys [rules seeds] :as _rules} {:keys [base-url] :as options}]
(let [interact (mk-interact options)
seeds (map (fn [{:keys [path] :as seed}]
(-> seed
(assoc :url (spider/make-url base-url path))))
seeds)
[openapi-spec
{:keys [rules seeds] :as _rules}
{:keys [base-url max-requests-per-operation max-total-requests] :as options}]
(let [
validate (-> (validator/validator-context openapi-spec {})
(validator/interaction-validator))
matcher (paths-matcher (keys (get openapi-spec "paths")))
op-path (fn [interaction]
(when-let [template (:template (matcher (:uri (:request interaction))))]
[:paths template (:method (:request interaction))]))]
(->> (iterate (fn [state]
(spider/step state rules
:interact interact))
(spider/step {:pool (set seeds), :seen #{}} rules
:interact interact))
(take-while some?)
(map :interaction)
op-path (fn [request]
(when-let [template (:template (matcher (path request)))]
[:paths template (:method request)]))
exec-request (-> (mk-exec-request options)
(wrap-max-requests max-total-requests)
(wrap-max-requests-per-operation max-requests-per-operation op-path))]

(->> (spider/spider {:rules rules :seeds seeds :exec-request exec-request})
(map fixup-interaction)
(map #(assoc %
:issues (validate % [])
:operation-path (op-path %))))))
:operation-path (op-path (:request %)))))))

0 comments on commit 1052fa3

Please sign in to comment.