Monitoring integration

docs/configuration.md

@@ -2,24 +2,26 @@
 
 ## Core configuration
 
-| Key | Description | Default value | Mandatory? | Values examples |
-|---|---|---|---|---|
-| name | Allow to name your config. For now will only be used to logging purpose | None | no |  |
-| working.directory.path | Indicates the directory to use to find relative paths | Java default working directory path | no |  |
-| parent.config.path | Path to an other configuration file to inherit properties. Can be absolute or relative to `working.directory.path` | None | no |  |
-| default.script.action | The default script action | ```RUN``` | no | ```RUN``` or ```MARK_AS_EXECUTED``` |
-| scan.path | Path to the folder containing all your scripts. Can be absolute or relative to `working.directory.path` | ```./scripts/``` | yes |  |
-| scan.identifier.regex | Regex that will be used to determine an identifier for each file. It has to contain a capturing group. Identifiers are then used to sort the scripts before running them. | ```(.*)``` (with this regex, the script's whole name will be its identifier) | no | With the regex ```(.*?)_.*```, a script named ```1.23_my-script.js``` will have ```1.23``` as its identifier  |
-| scan.tags.createFromFolder | If true, scripts will have their parent folders names as tags. Relative path to ```scan.path``` is used.  | ```false``` | no | ```false``` or ```true``` |
-| tag.*your_tag* | Glob paths to your scripts that you want to apply the tag "your_tag" on. To declare multiple tags, you will have to add multiple properties in your settings. A tag ```my_tag``` will have as as property name ```tag.my_tag``` **WARNING:** ALWAYS declare your tags using absolute paths. Relative paths and even using a tilde (~) won't do the trick. |  | no | ```[data/*, script1.js, old/old_script1.js]``` |
-| filter.tags.whitelisted | Scripts that have these tags will be considered | None | no | ```DATA,tag``` |
-| filter.tags.blacklisted | Scripts that have these tags will be ignored. A script having a whitelisted tag and a blacklisted tag will be ignored | None | no | ```DATA,tag``` |
-| prune.tags.to.run.again | Scripts that have these tags will be run, even they were already executed  | None | no | ```tag,again``` |
-| execution.mode | Execution mode. Possible values:<br />- ```NORMAL```: Regular execution: your scripts will be run on your database.<br />- ```DRY```: Scripts will not be executed. A full report of what would happen is you ran Datamaintain normally will be logged.<br /> | ```NORMAL``` | no | ```NORMAL```, ```DRY``` |
-| verbose | If true, more logs will be printed. **WARNING:** Can't be used alongside with porcelain. If both are set, porcelain will prevail | ```false``` | no | ```true``` or ```false``` |
-| prune.scripts.override.executed | Allow datamaintain to override a script if it detect a checksum change on a script already runned (assuming its filename) | ```false``` | no | ```true``` or ```false``` |
-| db.trust.uri | Bypass all checks that could be done on your URI because you are very sure of it and think our checks are just liars | ```false``` | no | ```true``` or ```false``` |
-| porcelain | For each executed script, print path relative to scan path **WARNING:** Can't be used alongside with verbose. If both are set, porcelain will prevail | ```false``` | no | ```true``` or ```false``` |
+| Key | Description                                                                                                                                                                                                                                                                                                                                               | Default value                                                                | Mandatory? | Values examples |
+|---|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|---|---|
+| name | Allow to name your config. For now will only be used to logging purpose                                                                                                                                                                                                                                                                                   | None                                                                         | no |  |
+| working.directory.path | Indicates the directory to use to find relative paths                                                                                                                                                                                                                                                                                                     | Java default working directory path                                          | no |  |
+| parent.config.path | Path to an other configuration file to inherit properties. Can be absolute or relative to `working.directory.path`                                                                                                                                                                                                                                        | None                                                                         | no |  |
+| default.script.action | The default script action                                                                                                                                                                                                                                                                                                                                 | ```RUN```                                                                    | no | ```RUN``` or ```MARK_AS_EXECUTED``` |
+| scan.path | Path to the folder containing all your scripts. Can be absolute or relative to `working.directory.path`                                                                                                                                                                                                                                                   | ```./scripts/```                                                             | yes |  |
+| scan.identifier.regex | Regex that will be used to determine an identifier for each file. It has to contain a capturing group. Identifiers are then used to sort the scripts before running them.                                                                                                                                                                                 | ```(.*)``` (with this regex, the script's whole name will be its identifier) | no | With the regex ```(.*?)_.*```, a script named ```1.23_my-script.js``` will have ```1.23``` as its identifier  |
+| scan.tags.createFromFolder | If true, scripts will have their parent folders names as tags. Relative path to ```scan.path``` is used.                                                                                                                                                                                                                                                  | ```false```                                                                  | no | ```false``` or ```true``` |
+| tag.*your_tag* | Glob paths to your scripts that you want to apply the tag "your_tag" on. To declare multiple tags, you will have to add multiple properties in your settings. A tag ```my_tag``` will have as as property name ```tag.my_tag``` **WARNING:** ALWAYS declare your tags using absolute paths. Relative paths and even using a tilde (~) won't do the trick. |                                                                              | no | ```[data/*, script1.js, old/old_script1.js]``` |
+| filter.tags.whitelisted | Scripts that have these tags will be considered                                                                                                                                                                                                                                                                                                           | None                                                                         | no | ```DATA,tag``` |
+| filter.tags.blacklisted | Scripts that have these tags will be ignored. A script having a whitelisted tag and a blacklisted tag will be ignored                                                                                                                                                                                                                                     | None                                                                         | no | ```DATA,tag``` |
+| prune.tags.to.run.again | Scripts that have these tags will be run, even they were already executed                                                                                                                                                                                                                                                                                 | None                                                                         | no | ```tag,again``` |
+| execution.mode | Execution mode. Possible values:<br />- ```NORMAL```: Regular execution: your scripts will be run on your database.<br />- ```DRY```: Scripts will not be executed. A full report of what would happen is you ran Datamaintain normally will be logged.<br />                                                                                             | ```NORMAL```                                                                 | no | ```NORMAL```, ```DRY``` |
+| verbose | If true, more logs will be printed. **WARNING:** Can't be used alongside with porcelain. If both are set, porcelain will prevail                                                                                                                                                                                                                          | ```false```                                                                  | no | ```true``` or ```false``` |
+| prune.scripts.override.executed | Allow datamaintain to override a script if it detect a checksum change on a script already runned (assuming its filename)                                                                                                                                                                                                                                 | ```false```                                                                  | no | ```true``` or ```false``` |
+| db.trust.uri | Bypass all checks that could be done on your URI because you are very sure of it and think our checks are just liars                                                                                                                                                                                                                                      | ```false```                                                                  | no | ```true``` or ```false``` |
+| porcelain | For each executed script, print path relative to scan path **WARNING:** Can't be used alongside with verbose. If both are set, porcelain will prevail                                                                                                                                                                                                     | ```false```                                                                  | no | ```true``` or ```false``` |
+| datamaintain.monitoring.api.url | Url to the Datamaintain monitoring application you want your data to be sent to **WARNING:** Make sure to also set `datamaintain.monitoring.module.environment.token` if you want to send data to the monitoring app as both settings are necessary                                                                                                       | ```https://datamaintain-monitoring.com```                                    | no |  |
+| datamaintain.monitoring.module.environment.token | Token that identifies your module and environment, given by Datamaintain monitoring **WARNING:** Make sure to also set `datamaintain.monitoring.api.url` if you want to send data to the monitoring app as both settings are necessary                                                                                                                                                                                                                                                              | ```109a2c08-e836-451a-86ef-d67be8ffc648```                                                                  | no |  |
 
 ### Common driver configuration
 

gradle/conventions/base/src/main/kotlin/kotlin.gradle.kts

@@ -30,6 +30,7 @@ version = if (lastTag != "") {
 
 repositories {
     mavenCentral()
+    mavenLocal()
 }
 
 kotlin {

gradle/libs.versions.toml

@@ -6,6 +6,7 @@ kotlinLogging = "3.0.5"
 logbackClassic = "1.5.3"
 junit = "5.10.2"
 testContainers = "1.19.7"
+mockServer = "5.15.0"
 strikt = "0.34.1"
 mockk = "1.13.10"
 clickt = "2.8.0"
@@ -20,6 +21,9 @@ postgresql = "42.1.4"
 json-path = "2.9.0"
 json-smart = "2.5.0"
 
+http4k = "4.3.4.1"
+jacksonDatatype = "2.12.2"
+
 # use this to specify latest release
 versionLatest = "latest.integration"
 
@@ -52,6 +56,10 @@ jsonSmart = { module = "net.minidev:json-smart", version.ref = "json-smart" }
 h2database = { module = "com.h2database:h2", version.ref = "h2Database" }
 mongoDriver-sync = { module = "org.mongodb:mongodb-driver-sync", version.ref = "mongoDriver" }
 
+http4kCore = { module = "org.http4k:http4k-core", version.ref = "http4k" }
+http4kJackson = { module = "org.http4k:http4k-format-jackson", version.ref = "http4k" }
+jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jacksonDatatype" }
+
 junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" }
 junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit" }
 junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" }
@@ -61,6 +69,8 @@ clikt = { module = "com.github.ajalt:clikt", version.ref = "clickt" }
 testContainers = { module = "org.testcontainers:testcontainers", version.ref = "testContainers" }
 testContainers-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testContainers" }
 testContainers-mongodb = { module = "org.testcontainers:mongodb", version.ref = "testContainers" }
+testContainers-mockserver = { module = "org.testcontainers:mockserver", version.ref = "testContainers" }
+mockserver = { module = "org.mock-server:mockserver-client-java", version.ref = "mockServer" }
 
 # Plugins imported in convention plugins must be imported as libs for now
 gradle-palantir-git = { module = "com.palantir.gradle.gitversion:gradle-git-version", version.ref = "gradle-palantir-git" }

modules/cli/src/main/kotlin/datamaintain/cli/app/update/db/DatamaintainCliUpdateDbCommand.kt

@@ -4,6 +4,7 @@ import datamaintain.cli.app.DatamaintainCliCommand
 import datamaintain.core.Datamaintain
 import datamaintain.core.config.DatamaintainConfig
 import datamaintain.core.exception.DatamaintainException
+import datamaintain.core.report.print
 import kotlin.system.exitProcess
 
 fun defaultUpdateDbRunner(config: DatamaintainConfig) {

modules/cli/src/main/kotlin/datamaintain/cli/app/update/db/MarkOneScriptAsExecuted.kt

@@ -5,7 +5,7 @@ import com.github.ajalt.clikt.parameters.options.option
 import datamaintain.cli.app.utils.detailedOption
 import datamaintain.core.config.CoreConfigKey
 import datamaintain.core.config.DatamaintainConfig
-import datamaintain.core.script.ScriptAction
+import datamaintain.domain.script.ScriptAction
 import java.util.*
 
 class MarkOneScriptAsExecuted(runner: (DatamaintainConfig) -> Unit = ::defaultUpdateDbRunner) :

modules/cli/src/main/kotlin/datamaintain/cli/app/update/db/UpdateDb.kt

@@ -9,11 +9,11 @@ import datamaintain.cli.app.utils.detailedOption
 import datamaintain.core.config.CoreConfigKey
 import datamaintain.core.config.DatamaintainConfig
 import datamaintain.core.db.driver.DriverConfigKey
-import datamaintain.core.script.ScriptAction
 import datamaintain.core.step.check.allCheckRuleNames
 import datamaintain.core.step.executor.ExecutionMode
 import datamaintain.db.driver.mongo.MongoConfigKey
 import datamaintain.db.driver.mongo.MongoShell
+import datamaintain.domain.script.ScriptAction
 import java.util.*
 
 class UpdateDb(runner: (DatamaintainConfig) -> Unit = ::defaultUpdateDbRunner) : DatamaintainCliUpdateDbCommand(
@@ -124,6 +124,12 @@ class UpdateDb(runner: (DatamaintainConfig) -> Unit = ::defaultUpdateDbRunner) :
             "To define multiple rules, use option multiple times.")
         .multiple()
 
+    private val datamaintainMonitoringApiUrl: String? by option(help = "Url to contact the datamaintain monitoring app." +
+            "Will be used to send all reports on executions")
+
+    private val datamaintainMonitoringModuleEnvironmentToken: String? by option(help = "Token given in the Datamaintain monitoring app " +
+            "to identify the module and environment on which you are executing your scripts")
+
     override fun overloadProps(props: Properties) {
         path?.let { props.put(CoreConfigKey.SCAN_PATH.key, it) }
         identifierRegex?.let { props.put(CoreConfigKey.SCAN_IDENTIFIER_REGEX.key, it) }
@@ -144,6 +150,8 @@ class UpdateDb(runner: (DatamaintainConfig) -> Unit = ::defaultUpdateDbRunner) :
         porcelain?.let { props.put(CoreConfigKey.PRINT_RELATIVE_PATH_OF_SCRIPT.key, it.toString()) }
         mongoShell?.let { props.put(MongoConfigKey.DB_MONGO_SHELL.key, it.toUpperCase()) }
         flags?.let { props.put(CoreConfigKey.FLAGS.key, it.optionListToString()) }
+        datamaintainMonitoringApiUrl?.let { props.put(CoreConfigKey.DATAMAINTAIN_MONITORING_API_URL.key, it) }
+        datamaintainMonitoringModuleEnvironmentToken?.let { props.put(CoreConfigKey.DATAMAINTAIN_MONITORING_MODULE_ENVIRONMENT_TOKEN.key, it) }
         mongoClient?.let { props.put(MongoConfigKey.DB_MONGO_CLIENT_PATH.key, mongoClient) }
     }
 

modules/cli/src/test/kotlin/datamaintain/cli/app/update/db/MarkOneScriptAsExecutedTest.kt

@@ -1,7 +1,7 @@
 package datamaintain.cli.app.update.db
 
 import datamaintain.cli.app.BaseCliTest
-import datamaintain.core.script.ScriptAction
+import datamaintain.domain.script.ScriptAction
 import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
 import strikt.api.expectThat