DHFPROD-1896: Merge pull request

Automated PR for Release Branch

gradle.properties

@@ -1,2 +1,4 @@
 version=4.2.1
 publishUrl=file:../marklogic-data-hub/releases
+systemProp.file.encoding=utf-8
+org.gradle.jvmargs='-Dfile.encoding=UTF-8'

marklogic-data-hub/src/main/java/com/marklogic/hub/collector/DiskQueue.java

@@ -25,6 +25,7 @@
 package com.marklogic.hub.collector;
 
 import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.security.InvalidParameterException;
 import java.text.MessageFormat;
 import java.util.AbstractQueue;
@@ -165,12 +166,14 @@ private void openFile() throws IOException {
             fileQueue = File.createTempFile(DiskQueue.class.getSimpleName() + "-backingstore-", null, tempDir);
             fileQueue.deleteOnExit();
             LOG.log(Level.INFO, "created backing store {0}", fileQueue.getAbsolutePath());
-            fileOut = new BufferedWriter(new FileWriter(fileQueue));
+            fileOut = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileQueue), StandardCharsets.UTF_8));
 
             // Flush output file, so there's something written when we open the input stream.
             fileOut.flush();
 
-            fileIn = new BufferedReader(new FileReader(fileQueue));
+            fileIn = new BufferedReader( new InputStreamReader(
+                new FileInputStream(fileQueue), StandardCharsets.UTF_8)
+            );
         }
     }
 

marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java

@@ -53,6 +53,7 @@
 import java.io.InputStreamReader;
 import java.net.URI;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateParsingException;
 import java.security.cert.X509Certificate;
@@ -138,7 +139,7 @@ public DiskQueue<String> run(String jobId, String entity, String flow, int threa
 
             // Streams the response instead of loading it all in memory
             ResponseExtractor<Void> responseExtractor = response -> {
-                BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(response.getBody()));
+                BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(response.getBody(), StandardCharsets.UTF_8));
                 String line;
                 while((line = bufferedReader.readLine()) != null) {
                     results.add(line);

marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java

@@ -318,7 +318,8 @@ else if (failedEvents.get() + successfulEvents.get() != uriCount) {
             else if (failedEvents.get() > 0 && successfulEvents.get() > 0) {
                 status = JobStatus.FINISHED_WITH_ERRORS;
             }
-            else if (failedEvents.get() == 0 && successfulEvents.get() > 0) {
+            //empty collector or no failure events => JobStatus.FINISHED
+            else if ((failedEvents.get() == 0 && successfulEvents.get() > 0) || uriCount == 0) {
                 status = JobStatus.FINISHED;
             }
             else {

marklogic-data-hub/src/test/java/com/marklogic/hub/PiiE2E.java

@@ -108,6 +108,7 @@ public static void setupAll() throws Exception
     public void setup()
     {
         Assumptions.assumeTrue(!(isCertAuth() || isSslRun()));
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME);
         installHubModules();
         installUserModules(getHubAdminConfig(), true);

marklogic-data-hub/src/test/java/com/marklogic/hub/collector/EmptyCollectorTest.java

@@ -86,7 +86,7 @@ public void runCollector() {
         assertEquals(0, node.get("successfulEvents").asInt());
         assertEquals(0, node.get("failedEvents").asInt());
         assertEquals(0, node.get("failedBatches").asInt());
-        assertEquals("FAILED", node.get("status").asText());
+        assertEquals("FINISHED", node.get("status").asText());
 
     }
 }

marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java

@@ -20,10 +20,7 @@
 import com.marklogic.hub.ApplicationConfig;
 import org.apache.commons.io.FileUtils;
 import org.custommonkey.xmlunit.XMLUnit;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
@@ -89,6 +86,7 @@ public void testTelemetryInstallCount() throws IOException {
 
     @Test
     public void testInstallHubModules() throws IOException {
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         assertTrue(getDataHub().isInstalled().isInstalled());
 
         assertTrue(getModulesFile("/com.marklogic.hub/config.xqy").startsWith(getResource("data-hub-test/core-modules/config.xqy")));
@@ -108,6 +106,7 @@ public void getHubModulesVersion() throws IOException {
 
     @Test
     public void testInstallUserModules() throws IOException, ParserConfigurationException, SAXException, URISyntaxException {
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         URL url = DataHubInstallTest.class.getClassLoader().getResource("data-hub-test");
         String path = Paths.get(url.toURI()).toFile().getAbsolutePath();
         File srcDir = new File(path);

marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubConfigTest.java

@@ -11,6 +11,7 @@
 import com.marklogic.hub.error.DataHubConfigurationException;
 
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -53,7 +54,8 @@ public void testAppConfigDefaultProps() {
 
     @Test
     public void applyFinalConnectionPropsToDefaultRestConnection() {
-
+
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         AppConfig config = adminHubConfig.getAppConfig();
 
         assertEquals(new Integer(8011), config.getRestPort(),

marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java

@@ -163,7 +163,8 @@ public void testUserModulesDeployTimestampFilePath() {
 
     @Test
     public void upgrade300To403ToCurrentVersion() throws Exception {
-        Assumptions.assumeFalse((isCertAuth() || isSslRun())); 
+        Assumptions.assumeFalse((isCertAuth() || isSslRun()));
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         final String projectPath = "build/tmp/upgrade-projects/dhf403from300";
         final File projectDir = Paths.get(projectPath).toFile();
 

marklogic-data-hub/src/test/java/com/marklogic/hub/entity/EntityServicesAlignmentTest.java

@@ -13,6 +13,7 @@
 import com.marklogic.hub.HubTestBase;
 import com.marklogic.hub.util.FileUtil;
 import com.marklogic.hub.util.HubModuleManager;
+import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -77,6 +78,7 @@ public void testDeployTDEWithNoEntities() {
 
     @Test
     public void testDeployTDE() throws Exception {
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         installEntities();
 
         getDataHub().clearDatabase(HubConfig.DEFAULT_FINAL_SCHEMAS_DB_NAME);

marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java

@@ -30,6 +30,7 @@
 import org.custommonkey.xmlunit.XMLAssert;
 import org.custommonkey.xmlunit.XMLUnit;
 import org.junit.Assert;
+import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -250,6 +251,7 @@ public void testEnvelopeSJSXML() throws IOException, SAXException {
     @Test
     public void testCreateandDeployFlowWithHubUser() throws IOException {
 
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         scaffolding.createFlow(ENTITY, "FlowWithHubUser", FlowType.HARMONIZE,
                 CodeFormat.XQUERY, DataFormat.JSON, false);
         Files.copy(getResourceStream("flow-runner-test/collector2.xqy"),

marklogic-data-hub/src/test/java/com/marklogic/hub_integration/StreamCollectorTest.java

@@ -32,6 +32,7 @@
 import com.marklogic.hub.util.FileUtil;
 import org.custommonkey.xmlunit.XMLUnit;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -144,6 +145,7 @@ public void runCollector() {
         // there is a custom content plugin that throws an error. This code uses the stopOnFailure
         // option to halt execution. This allows us to test that the collector runs to completion while not
         // having to wait for the entire harmonize flow to finish.
+        Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment());
         assertEquals(DOC_COUNT, getStagingDocCount());
         assertEquals(0, getFinalDocCount());
         Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize",

ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/RunFlowTask.groovy

@@ -116,7 +116,7 @@ class RunFlowTask extends HubTask {
         Map<String, Object> options = new HashMap<>()
         project.ext.properties.each { key, value ->
             if (key.toString().startsWith("dhf.")) {
-                options.put(key, value)
+                options.put(key.minus("dhf."), value)
             }
         }
         println("Running Flow: [" + entityName + ":" + flowName + "]" +

ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy

@@ -3,6 +3,7 @@ package com.marklogic.gradle.task
 import com.marklogic.hub.HubConfig
 import org.apache.commons.io.FileUtils
 import org.gradle.testkit.runner.UnexpectedBuildFailure
+import spock.lang.IgnoreIf
 
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -16,6 +17,7 @@ class GenerateTDETemplateFromEntityTaskTest extends BaseTest {
         clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME);
     }
 
+    @IgnoreIf({ System.getProperty('mlIsProvisionedEnvironment') })
     def "GenerateTDETEmplates"() {
         given:
         def pluginDir = Paths.get(hubConfig().hubProject.projectDirString).resolve("plugins")

ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/InstalledTests.groovy

@@ -91,16 +91,8 @@ class InstalledTests extends BaseTest {
     }
 
     def "test run flow with invalid flow"() {
-        setup: "append properties for task name and flow name"
-        propertiesFile << """
-                ext {
-                    entityName=my-entity
-                    flowName=my-flow-not-found
-                }
-            """
-
         when: "hubRunFlow is Run"
-        def result = runFailTask('hubRunFlow', '-i')
+        def result = runFailTask('hubRunFlow', '-PentityName=my-new-entity', '-PflowName=my-flow-not-found', '-i')
 
         then: "it should run with errors"
         notThrown(UnexpectedBuildSuccess)
@@ -124,16 +116,19 @@ class InstalledTests extends BaseTest {
         installStagingDoc("/employee2.xml", meta, new File("src/test/resources/run-flow-test/employee2.xml").text)
         assert (getStagingDocCount() == 2)
         assert (getFinalDocCount() == 0)
-
+        String result;
         installModule("/entities/my-new-entity/harmonize/my-new-harmonize-flow/content/content.xqy", "run-flow-test/content.xqy")
 
         when:
-        println(runTask('hubRunFlow', '-PentityName=my-new-entity', '-PflowName=my-new-harmonize-flow', '-i').getOutput())
+        result = runTask('hubRunFlow', '-Pdhf.key=value', '-PshowOptions=true','-PentityName=my-new-entity', '-PflowName=my-new-harmonize-flow', '-i').getOutput()
+        println(result)
 
         then:
         notThrown(UnexpectedBuildFailure)
         getStagingDocCount() == 2
         getFinalDocCount() == 2
+        assert(result.contains("key = value"))
+        assert(! result.contains("dhf.key = value"))
         assertXMLEqual(getXmlFromResource("run-flow-test/harmonized1.xml"), hubConfig().newFinalClient().newDocumentManager().read("/employee1.xml").next().getContent(new DOMHandle()).get())
         assertXMLEqual(getXmlFromResource("run-flow-test/harmonized2.xml"), hubConfig().newFinalClient().newDocumentManager().read("/employee2.xml").next().getContent(new DOMHandle()).get())
     }