GH-1528: Release 0.11.0 (#548)

See the CHANGELOG.
broadinstitute · Dec 15, 2021 · c613d70 · c613d70
1 parent 1f7f4ed
commit c613d70
Show file tree

Hide file tree

Showing 17 changed files with 262 additions and 114 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,12 @@
+# Release 0.11.0
+- [GH-1526] Release "Arrays_v2.6.0" ([#547](https://github.com/broadinstitute/wfl/pull/547))
+- [GH-1546] Increment minor version to 0.11.0 ([#545](https://github.com/broadinstitute/wfl/pull/545))
+- [GH-1506] Add some documentation for changing the logging level ([#546](https://github.com/broadinstitute/wfl/pull/546))
+- [GH-1465] Specify TDR Polling Interval ([#542](https://github.com/broadinstitute/wfl/pull/542))
+- [GH-1521] Pull v0.10.0 changelog into develop ([#544](https://github.com/broadinstitute/wfl/pull/544))
+- [GH-1525] Document TerraWorkspaceSink identifier changes ([#535](https://github.com/broadinstitute/wfl/pull/535))
+- [GH-1516] Documentation updates - TerraExecutor retry ([#539](https://github.com/broadinstitute/wfl/pull/539))
+
 # Release 0.10.0
 - [GH-1475] Update test inputs buckets ([#538](https://github.com/broadinstitute/wfl/pull/538))
 - [GH-1529] Bump WFL develop version to 0.10.0 ([#537](https://github.com/broadinstitute/wfl/pull/537))

diff --git a/api/src/wfl/module/aou.clj b/api/src/wfl/module/aou.clj
@@ -33,7 +33,7 @@
 
 (def workflow-wdl
   "The top-level WDL file and its version."
-  {:release "Arrays_v2.4.1"
+  {:release "Arrays_v2.6.0"
    :path    "pipelines/broad/arrays/single_sample/Arrays.wdl"})
 
 (def cromwell-label-map

diff --git a/api/src/wfl/source.clj b/api/src/wfl/source.clj
@@ -22,13 +22,15 @@
 (s/def ::column string?)
 (s/def ::snapshotReaders (s/* util/email-address?))
 (s/def ::snapshots (s/* ::all/uuid))
+(s/def ::pollingIntervalMinutes int?)
 (s/def ::tdr-source
   (s/keys :req-un [::all/name
                    ::column
                    ::all/dataset
                    ::all/table
                    ::snapshotReaders]
-          :opt-un [::snapshots]))
+          :opt-un [::snapshots
+                   ::pollingIntervalMinutes]))
 
 (s/def ::snapshot-list-source
   (s/keys :req-un [::all/name ::snapshots]))
@@ -96,10 +98,11 @@
 (def ^:private ^:const tdr-source-type  "TerraDataRepoSource")
 (def ^:private ^:const tdr-source-table "TerraDataRepoSource")
 (def ^:private ^:const tdr-source-serialized-fields
-  {:dataset         :dataset
-   :table           :dataset_table
-   :column          :table_column_name
-   :snapshotReaders :snapshot_readers})
+  {:dataset                :dataset
+   :table                  :dataset_table
+   :column                 :table_column_name
+   :snapshotReaders        :snapshot_readers
+   :pollingIntervalMinutes :polling_interval_minutes})
 
 (def ^:private bigquery-datetime-format
   (DateTimeFormatter/ofPattern "yyyy-MM-dd'T'HH:mm:ss"))
@@ -305,19 +308,22 @@
 ;; but overpolling the TDR increases the chances of:
 ;; - creating many single-row / low-cardinality snapshots
 ;; - locking the dataset / running into dataset locks
-(def ^:private tdr-source-polling-interval-minutes 20)
+(def ^:private tdr-source-default-polling-interval-minutes 20)
 
 (defn ^:private tdr-source-should-poll?
-  "Return true if it's been at least `tdr-source-polling-interval-minutes`
+  "Return true if it's been at least the specified `polling_interval_minutes`
+   or, if unspecified, `tdr-source-default-polling-interval-minutes`
    since `last_checked` -- when we last checked for new rows in the TDR."
-  [{:keys [type id last_checked] :as _source} utc-now]
+  [{:keys [type id last_checked pollingIntervalMinutes] :as _source} utc-now]
   (let [checked            (timestamp-to-offsetdatetime last_checked)
-        minutes-since-poll (.between ChronoUnit/MINUTES checked utc-now)]
+        minutes-since-poll (.between ChronoUnit/MINUTES checked utc-now)
+        polling-interval (or pollingIntervalMinutes
+                             tdr-source-default-polling-interval-minutes)]
     (log/debug {:type               type
                 :id                 id
                 :minutes-since-poll minutes-since-poll
-                :polling-interval   tdr-source-polling-interval-minutes})
-    (<= tdr-source-polling-interval-minutes minutes-since-poll)))
+                :polling-interval   polling-interval})
+    (<= polling-interval minutes-since-poll)))
 
 (defn ^:private update-tdr-source
   "Check for new data in TDR from `source`, create new snapshots,

diff --git a/api/test/wfl/system/v1_endpoint_test.clj b/api/test/wfl/system/v1_endpoint_test.clj
@@ -258,11 +258,12 @@
   []
   (let [terra-ns  (comp (partial str/join "/") (partial vector "wfl-dev"))
         workspace (terra-ns "CDC_Viral_Sequencing")
-        source    {:name            "Terra DataRepo"
-                   :dataset         "79fc88f5-dcf4-48b0-8c01-615dfbc1c63a"
-                   :table           "flowcells"
-                   :column          "last_modified_date"
-                   :snapshotReaders ["[email protected]"]}
+        source    {:name                   "Terra DataRepo"
+                   :dataset                "79fc88f5-dcf4-48b0-8c01-615dfbc1c63a"
+                   :table                  "flowcells"
+                   :column                 "last_modified_date"
+                   :snapshotReaders        ["[email protected]"]
+                   :pollingIntervalMinutes 1}
         executor  {:name                       "Terra"
                    :workspace                  workspace
                    :methodConfiguration        (terra-ns "sarscov2_illumina_full")
@@ -428,11 +429,12 @@
        [{:email (env/getenv "WFL_TDR_SERVICE_ACCOUNT")
          :accessLevel "OWNER"}])]
     (fn [[dataset workspace]]
-      (let [source   {:name            "Terra DataRepo"
-                      :dataset         dataset
-                      :table           "inputs"
-                      :column          "ingested"
-                      :snapshotReaders ["[email protected]"]}
+      (let [source   {:name                   "Terra DataRepo"
+                      :dataset                dataset
+                      :table                  "inputs"
+                      :column                 "ingested"
+                      :snapshotReaders        ["[email protected]"]
+                      :pollingIntervalMinutes 1}
             executor {:name                       "Terra"
                       :workspace                  workspace
                       :methodConfiguration        "warp-pipelines/IlluminaGenotypingArray"

diff --git a/api/test/wfl/unit/source_test.clj b/api/test/wfl/unit/source_test.clj
@@ -9,7 +9,7 @@
   (let [now    (utc-now)
         source {:last_checked (Timestamp/from (.toInstant now))}]
     (letfn [(end-of-interval [min-from-interval]
-              (->> #'source/tdr-source-polling-interval-minutes
+              (->> #'source/tdr-source-default-polling-interval-minutes
                    var-get
                    (+ min-from-interval)
                    (.addTo ChronoUnit/MINUTES now)))]

diff --git a/database/changelog.xml b/database/changelog.xml
@@ -45,4 +45,5 @@
     <include file="changesets/20210823_ConvertWatchersArrayToText.xml"                relativeToChangelogFile="true"/>
     <include file="changesets/20210910_AddRetryToTerraDataRepoSourceDetails.xml"      relativeToChangelogFile="true"/>
     <include file="changesets/20211102_WorkloadWatchersDataCompatibilityUpdate.xml"   relativeToChangelogFile="true"/>
+    <include file="changesets/20211110_AddPollingIntervalColumn.xml"                  relativeToChangelogFile="true"/>
 </databaseChangeLog>
diff --git a/database/changesets/20211110_AddPollingIntervalColumn.xml b/database/changesets/20211110_AddPollingIntervalColumn.xml
@@ -0,0 +1,26 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext
+                        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd
+                        http://www.liquibase.org/xml/ns/dbchangelog
+                        http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+    <changeSet author="[email protected]"
+               dbms="postgresql"
+               id="20211110"
+               logicalFilePath="20211110_AddPollingIntervalColumn.xml"
+               runInTransaction="false">
+        <comment>
+            We want to be able to allow each terra data repo source to be able to specify
+            the rate at which they poll TDR for new rows to snapshot. The new column added
+            to TerraDataRepoSource will allow us to keep track of what the interval is for
+            each workload's source.
+        </comment>
+        <sql>
+            ALTER TABLE TerraDataRepoSource
+            ADD polling_interval_minutes INTEGER;
+        </sql>
+    </changeSet>
+</databaseChangeLog>
diff --git a/docs/md/assets/executor/terra-method-configuration.png b/docs/md/assets/executor/terra-method-configuration.png
diff --git a/docs/md/assets/sink/terra-identifier-in-data-table.png b/docs/md/assets/sink/terra-identifier-in-data-table.png
diff --git a/docs/md/dev-logging.md b/docs/md/dev-logging.md
@@ -48,6 +48,49 @@ This example produces the following json log:
 ```
 The logging can also be disabled if you see fit for whatever reason by binding the `*logger*` instance to `disabled-logger`.
 
+Example:
+```clojure
+(binding [log/*logger* log/disabled-logger]
+  (log/info "This message will not be written"))
+```
+
+## Logging Levels
+The severity levels currently supported for WFL logs are, in order: DEBUG, INFO, NOTICE, ERROR, CRITICAL, ALERT, EMERGENCY.
+
+By default, all logs of severity INFO and higher are written to stdout. If you desire to change this level, i.e. write only logs ERROR and higher or DEBUG and higher, you can set this configuration with the `logging_level` endpoint.
+
+Calling `logging_level` with a GET request will return the current level in which the api is writing. Example on a local server below:
+
+```
+curl -X GET http://localhost:3000/api/v1/logging_level \
+     -H 'accept: application/json' \
+     -H "authorization: Bearer "$(gcloud auth print-access-token)
+```
+
+The result will look something like this:
+```
+{
+  "level" : "INFO"
+}
+```
+
+In order to change this level as desired would be done like so:
+```
+curl -X POST http://localhost:3000/api/v1/logging_level?level=DEBUG \
+     -H 'accept: application/json' \
+     -H "authorization: Bearer "$(gcloud auth print-access-token)
+```
+
+The result would be similar:
+```
+{
+  "level" : "DEBUG"
+}
+```
+
+The above change would allow all logs DEBUG and higher to be written, i.e. DEBUG, INFO, NOTICE,
+WARNING, ERROR, CRITICAL, ALERT, EMERGENCY.
+
 Example:
 ```clojure
 (binding [log/*logger* log/disabled-logger]
@@ -64,3 +107,9 @@ In order to be able to search for specific logs locally that could be useful in
 2. Run the server with `./ops/server.sh >> path/to/wfl/log 2>&1`
 3. Look up logs by severity and only show the message: `tail -f path/to/wfl/log | grep --line-buffered -w '"severity":"[YOUR_SEVERITY_HERE]"' | jq '.message'`
 4. Look up logs by a label and print the message: `tail -f path/to/wfl/log | grep --line-buffered -w 'my-label' | jq '.message'`
+
+You may also wish to check the Logging Level section for changing the severity of messages being written. An example being that you want to have debug messages that WFL writes that are not always displayed when the app is deployed on a production server. You could set the logging level to DEBUG and then tail the messages like so:
+
+`tail -f path/to/wfl/log | grep --line-buffered -w '"severity":"DEBUG"' | jq '.message'`
+
+Now you can read only the debug messages in stdout as they come and filter out all other severities such as INFO.
diff --git a/docs/md/dev-process.md b/docs/md/dev-process.md
@@ -235,8 +235,6 @@ open http://localhost:3000/swagger
 
 ### debugging Liquibase locally
 
-### debugging Liquibase locally
-
 Running `liquibase update`:
 ```bash
 liquibase --classpath=$(clojure -Spath)          \

diff --git a/docs/md/executor.md b/docs/md/executor.md
@@ -43,7 +43,7 @@ And a real-life example for a known method configuration:
 }
 ```
 
-![](./assets/terra-method-configuration.png)
+![](assets/executor/terra-method-configuration.png)
 
 The table below summarises the purpose of each attribute in the above request.
 
@@ -77,13 +77,25 @@ Prerequisites:
   workload creation.
 
 #### `methodConfigurationVersion`
-The expected version of `methodConfiguration`, stored as an integer
-in Firecloud.
+The expected version of `methodConfiguration`, stored as an integer in
+[Firecloud](https://firecloud-orchestration.dsde-prod.broadinstitute.org/#/Method%20Configurations/getWorkspaceMethodConfig).
+
+Example fetch of a method configuration's version from the appropriate
+Firecloud instance (prod):
+
+```shell
+curl -X 'GET' \
+  'https://firecloud-orchestration.dsde-prod.broadinstitute.org/api/workspaces/emerge_prod/Arrays_test/method_configs/warp-pipelines/Arrays' \
+  -H 'accept: */*' \
+  -H 'Authorization: Bearer '$(gcloud auth print-access-token) \
+  | jq .methodConfigVersion
+7
+```
 
 Prerequisites:
 
-- The `methodConfiguration` version when fetched from Firecloud should match
-  `methodConfigurationVersion`.
+- The `methodConfiguration` version when fetched from Firecloud
+  at workload creation should match `methodConfigurationVersion`.
 
 !!! warning "Implications of Version Mismatch"
     A version mismatch may indicate a possible concurrent modification of the
@@ -127,14 +139,21 @@ An executor is a `Queue` that satisfies the `Executor` protocol below:
      ^Executor   executor     ;; This executor instance
      ]
     "Use database `transaction` to return workflows created by the `executor`.")
-  (executor-workflows-by-status
+  (executor-workflows-by-filters
     ^IPersistentVector
-    [^Connection transaction  ;; JDBC Connection
-     ^Executor   executor     ;; This executor instance
-     ^String     status       ;; workflow status to match
+    [^Connection        transaction  ;; JDBC Connection
+     ^Executor          executor     ;; This executor instance
+     ^IPersistentVector filters      ;; Workflow filters to match
+                                     ;; (ex. status, submission)
     ]
     "Use database `transaction` to return workflows created by the `executor`
-     matching the workflow `status`.")
+     matching the workflow `filters` (ex. status, submission).")
+  (executor-throw-if-invalid-retry-filters
+    ;; Executed for side effects
+    [^IPersistentHashmap workload  ;; Workload for which a retry is requested
+     ^IPersistentVector  filters   ;; Workflow filters
+    ]
+    "Throw if workflow `filters` are invalid for `workload`'s retry request.")
   (executor-retry-workflows!
     ;; Executed for side effects
     [^Executor          executor   ;; This executor instance