diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..f0743b2d --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,510 @@ +@Library('shared-libraries') _ +import groovy.json.JsonSlurper +import groovy.json.JsonSlurperClassic +def JIRA_ID=""; +def commitMessage=""; +def prResponse=""; +def prNumber; +def githubAPIUrl="https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub" +pipeline{ + agent none; + options { + checkoutToSubdirectory 'data-hub' + } + environment{ + JAVA_HOME_DIR="~/java/jdk1.8.0_72" + GRADLE_DIR="/.gradle" + MAVEN_HOME="/usr/local/maven" + DMC_USER = credentials('MLBUILD_USER') + DMC_PASSWORD= credentials('MLBUILD_PASSWORD') + } + parameters{ + string(name: 'Email', defaultValue: 'stadikon@marklogic.com', description: 'Who should I say send the email to?') + } + stages{ + stage('Build-datahub'){ + agent { label 'dhfLinuxAgent'} + steps{ + script{ + if(env.CHANGE_TITLE){ + JIRA_ID=env.CHANGE_TITLE.split(':')[0]; + def transitionInput =[transition: [id: '41']] + //jiraTransitionIssue idOrKey: JIRA_ID, input: transitionInput, site: 'JIRA' + } + } + println(BRANCH_NAME) + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean --stacktrace;./gradlew build -x test -Pskipui=true;' + archiveArtifacts artifacts: 'data-hub/marklogic-data-hub/build/libs/* , data-hub/ml-data-hub-plugin/build/libs/* , data-hub/quick-start/build/libs/', onlyIfSuccessful: true } + post{ + failure { + println("Datahub Build FAILED") + script{ + def email; + if(env.CHANGE_AUTHOR){ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + email=getEmailFromGITUser author + }else{ + email=Email + } + sendMail email,'Check: ${BUILD_URL}/console',false,'Data Hub Build for $BRANCH_NAME Failed' + } + } + } + } + stage('Unit-Tests'){ + agent { label 'dhfLinuxAgent'} + steps{ + copyRPM 'Latest' + setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.mapping.MappingManagerTest -Pskipui=true' + junit '**/TEST-*.xml' + script{ + if(env.CHANGE_TITLE){ + JIRA_ID=env.CHANGE_TITLE.split(':')[0] + jiraAddComment comment: 'Jenkins Unit Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + } + post{ + always{ + sh 'rm -rf $WORKSPACE/xdmp' + } + success { + println("Unit Tests Completed") + script{ + def email; + if(env.CHANGE_AUTHOR){ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + email=getEmailFromGITUser author + }else{ + email=Email + } + sendMail email,'Check: ${BUILD_URL}/console',false,'Unit Tests for $BRANCH_NAME Passed' + } + } + failure { + println("Unit Tests Failed") + script{ + def email; + if(env.CHANGE_AUTHOR){ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + email=getEmailFromGITUser author + }else{ + email=Email + } + sendMail email,'Check: ${BUILD_URL}/console',false,'Unit Tests for $BRANCH_NAME Failed' + } + } + } + } + stage('code-review'){ + when { + allOf { + changeRequest author: '', authorDisplayName: '', authorEmail: '', branch: '', fork: '', id: '', target: '', title: '', url: '' + } + beforeAgent true + } + agent {label 'master'}; + steps{ + script{ + if(env.CHANGE_TITLE.split(':')[1].contains("Automated PR")){ + println("Automated PR") + sh 'exit 0' + }else{ + script{ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + def email=getEmailFromGITUser author + sendMail email,'Check: ${BUILD_URL}/console',false,'Waiting for code review $BRANCH_NAME ' + } + try{ + timeout(time:5, unit:'MINUTES') { + input message:'Review Done?' + } + }catch(err){ + currentBuild.result = "SUCCESS" + } + } + } + } + } + stage('PR'){ + when { + changeRequest() + beforeAgent true + } + agent {label 'master'}; + steps{ + retry(5){ + withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { + script{ + JIRA_ID=env.CHANGE_TITLE.split(':')[0] + def response = sh (returnStdout: true, script:'''curl -u $Credentials --header "application/vnd.github.merge-info-preview+json" "'''+githubAPIUrl+'''/pulls/$CHANGE_ID" | grep '"mergeable_state":' | cut -d ':' -f2 | cut -d ',' -f1 | tr -d '"' ''') + response=response.trim(); + println(response) + if(response.equals("clean")){ + println("merging can be done") + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: merging PR\"}' -u $Credentials "+ githubAPIUrl+"/pulls/$CHANGE_ID/merge | tail -1 > mergeResult.txt" + def mergeResult = readFile('mergeResult.txt').trim() + if(mergeResult==200){ + println("Merge successful") + }else{ + println("Merge Failed") + } + }else if(response.equals("blocked")){ + println("retry blocked"); + sleep time: 1, unit: 'MINUTES' + throw new Exception("Waiting for all the status checks to pass"); + }else if(response.equals("unstable")){ + println("retry unstable") + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: merging PR\"}' -u $Credentials "+githubAPIUrl+"/pulls/$CHANGE_ID/merge | tail -1 > mergeResult.txt" + def mergeResult = readFile('mergeResult.txt').trim() + println("Result is"+ mergeResult) + }else{ + println("merging not possible") + currentBuild.result = "FAILURE" + sh 'exit 1'; + } + } + } + } + } + post{ + success { + println("Merge Successful") + script{ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + def email=getEmailFromGITUser author + sendMail email,'Check: ${BUILD_URL}/console',false,' $BRANCH_NAME is Merged' + } + } + failure { + println("Retried 5times") + script{ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + def email=getEmailFromGITUser author + sendMail email,'Check: ${BUILD_URL}/console',false,' $BRANCH_NAME Cannot be Merged' + } + } + } + } + stage('rh7-singlenode'){ + when { + branch '4.x-develop' + } + agent { label 'dhfLinuxAgent'} + steps{ + copyRPM 'Latest' + setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.mapping.MappingManagerTest -Pskipui=true' + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins rh7-singlenode Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + post{ + always{ + sh 'rm -rf $WORKSPACE/xdmp' + } + success { + println("End-End Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7-singlenode Tests for $BRANCH_NAME Passed' + + } + failure { + println("End-End Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7-singlenode Tests for $BRANCH_NAME Failed' + } + } + } + stage('Merge PR to Integration Branch'){ + when { + branch 'FeatureBranch' + beforeAgent true + } + agent {label 'master'} + steps{ + withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { + script{ + //JIRA_ID=env.CHANGE_TITLE.split(':')[0] + prResponse = sh (returnStdout: true, script:''' + curl -u $Credentials -X POST -H 'Content-Type:application/json' -d '{\"title\": \"Automated PR for Integration Branch\" , \"head\": \"FeatureBranch\" , \"base\": \"IntegrationBranch\" }' '''+githubAPIUrl+'''/pulls ''') + println(prResponse) + def slurper = new JsonSlurper().parseText(prResponse) + println(slurper.number) + prNumber=slurper.number; + } + } + withCredentials([usernameColonPassword(credentialsId: 'rahul-git', variable: 'Credentials')]) { + sh "curl -u $Credentials -X POST -d '{\"event\": \"APPROVE\"}' "+githubAPIUrl+"/pulls/${prNumber}/reviews" + } + withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { + script{ + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: Merge pull request\"}' -u $Credentials "+githubAPIUrl+"/pulls/${prNumber}/merge | tail -1 > mergeResult.txt" + def mergeResult = readFile('mergeResult.txt').trim() + if(mergeResult==200){ + println("Merge successful") + }else{ + println("Merge Failed") + } + } + } + + } + post{ + success { + println("Automated PR For Integration branch Completed") + } + failure { + println("Creation of Automated PR Failed") + + } + } + } + stage('Parallel Execution'){ + when { + branch '4.x-develop' + beforeAgent true + } + parallel{ + stage('rh7_cluster_9.0-6'){ + agent { label 'dhfLinuxAgent'} + steps{ + copyRPM 'Release','9.0-6' + script{ + def dockerhost=setupMLDockerCluster 3 + sh 'docker exec -u builder -i '+dockerhost+' /bin/sh -c "su -builder;echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.mapping.MappingManagerTest -Pskipui=true"' + } + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins rh7_cluster_9.0-6 Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + post{ + always{ + sh 'rm -rf $WORKSPACE/xdmp' + } + success { + println("Upgrade Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7_cluster_9.0-6 Tests $BRANCH_NAME Passed' + // sh './gradlew publish' + } + failure { + println("Upgrade Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7_cluster_9.0-6 Tests for $BRANCH_NAME Failed' + } + } + } + stage('rh7_cluster_9.0-7'){ + agent { label 'dhfLinuxAgent'} + steps{ + copyRPM 'Release','9.0-7' + script{ + def dockerhost=setupMLDockerCluster 3 + sh 'docker exec -u builder -i '+dockerhost+' /bin/sh -c "su -builder;echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.mapping.MappingManagerTest -Pskipui=true"' + } + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins rh7_cluster_9.0-7 Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + post{ + always{ + sh 'rm -rf $WORKSPACE/xdmp' + } + success { + println("Upgrade Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7_cluster_9.0-7 Tests for $BRANCH_NAME Passed' + } + failure { + println("Upgrade Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7_cluster_9.0-7 Tests for $BRANCH_NAME Failed' + } + } + } + stage('rh7_cluster_9.0-8'){ + agent { label 'dhfLinuxAgent'} + steps{ + copyRPM 'Release','9.0-8' + script{ + def dockerhost=setupMLDockerCluster 3 + sh 'docker exec -u builder -i '+dockerhost+' /bin/sh -c "echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.mapping.MappingManagerTest -Pskipui=true"' + } + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins rh7_cluster_9.0-8 Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + post{ + always{ + sh 'rm -rf $WORKSPACE/xdmp' + } + success { + println("Upgrade Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7_cluster_9.0-8 Tests for $BRANCH_NAME Passed' + } + failure { + println("Upgrade Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'rh7_cluster_9.0-8 Tests for $BRANCH_NAME Failed' + } + } + } + stage('w12_cluster_9.0-6'){ + agent { label 'dhfWindowsAgent'} + steps{ + copyRPM 'Release','9.0-6' + script{ + def dockerhost=setupMLDockerCluster 3 + sh 'docker exec -u builder -i '+dockerhost+' /bin/sh -c "echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.mapping.MappingManagerTest -Pskipui=true"' + } + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins w12_cluster_9.0-6 Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + post{ + always{ + sh 'rm -rf $WORKSPACE/xdmp' + } + success { + println("Upgrade Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'w12_cluster_9.0-6 Tests for $BRANCH_NAME Passed' + } + failure { + println("Upgrade Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'w12_cluster_9.0-6 Tests for $BRANCH_NAME Failed' + } + } + } + } + post{ + success{ + node('dhfLinuxAgent'){ + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew publish' + } + } + } + } + stage('Merge PR to Release Branch'){ + when { + branch '4.x-develop' + beforeAgent true + } + agent {label 'master'} + steps{ + withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { + script{ + //JIRA_ID=env.CHANGE_TITLE.split(':')[0] + prResponse = sh (returnStdout: true, script:''' + curl -u $Credentials -X POST -H 'Content-Type:application/json' -d '{\"title\": \"Automated PR for Release Branch\" , \"head\": \"4.x-develop\" , \"base\": \"release/4.2.2\" }' '''+githubAPIUrl+'''/pulls ''') + println(prResponse) + def slurper = new JsonSlurper().parseText(prResponse) + println(slurper.number) + prNumber=slurper.number; + } + } + withCredentials([usernameColonPassword(credentialsId: 'rahul-git', variable: 'Credentials')]) { + sh "curl -u $Credentials -X POST -d '{\"event\": \"APPROVE\"}' "+githubAPIUrl+"/pulls/${prNumber}/reviews" + } + withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { + script{ + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: Merge pull request\"}' -u $Credentials "+githubAPIUrl+"/pulls/${prNumber}/merge | tail -1 > mergeResult.txt" + def mergeResult = readFile('mergeResult.txt').trim() + if(mergeResult==200){ + println("Merge successful") + }else{ + println("Merge Failed") + } + } + } + + } + post{ + success { + println("Automated PR For Release branch created") + + } + failure { + println("Creation of Automated PR Failed") + } + } + } + stage('Sanity Tests'){ + when { + branch 'Release/4.2.2' + beforeAgent true + } + agent { label 'dhfLinuxAgent'} + steps{ + copyRPM 'Latest' + setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.mapping.MappingManagerTest -Pskipui=true' + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins Sanity Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + post{ + always{ + sh 'rm -rf $WORKSPACE/xdmp' + } + success { + println("Sanity Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'Sanity Tests for $BRANCH_NAME Passed' + script{ + def transitionInput =[transition: [id: '31']] + //JIRA_ID=env.CHANGE_TITLE.split(':')[0] + //jiraTransitionIssue idOrKey: JIRA_ID, input: transitionInput, site: 'JIRA' + } + sendMail Email,'Click approve to release',false,'Datahub is ready for Release' + + } + failure { + println("Sanity Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'Sanity Tests for $BRANCH_NAME Failed' + } + } + } + } +} \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 9ba63feb..400ab412 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,4 @@ version=4.2.1 publishUrl=file:../marklogic-data-hub/releases +systemProp.file.encoding=utf-8 +org.gradle.jvmargs='-Dfile.encoding=UTF-8' diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/DiskQueue.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/DiskQueue.java index a61d65cd..c8ab3bbf 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/DiskQueue.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/DiskQueue.java @@ -25,6 +25,7 @@ package com.marklogic.hub.collector; import java.io.*; +import java.nio.charset.StandardCharsets; import java.security.InvalidParameterException; import java.text.MessageFormat; import java.util.AbstractQueue; @@ -165,12 +166,14 @@ private void openFile() throws IOException { fileQueue = File.createTempFile(DiskQueue.class.getSimpleName() + "-backingstore-", null, tempDir); fileQueue.deleteOnExit(); LOG.log(Level.INFO, "created backing store {0}", fileQueue.getAbsolutePath()); - fileOut = new BufferedWriter(new FileWriter(fileQueue)); + fileOut = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileQueue), StandardCharsets.UTF_8)); // Flush output file, so there's something written when we open the input stream. fileOut.flush(); - fileIn = new BufferedReader(new FileReader(fileQueue)); + fileIn = new BufferedReader( new InputStreamReader( + new FileInputStream(fileQueue), StandardCharsets.UTF_8) + ); } } diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java index ee664a7c..194fcb39 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java @@ -53,6 +53,7 @@ import java.io.InputStreamReader; import java.net.URI; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.security.cert.Certificate; import java.security.cert.CertificateParsingException; import java.security.cert.X509Certificate; @@ -138,7 +139,7 @@ public DiskQueue run(String jobId, String entity, String flow, int threa // Streams the response instead of loading it all in memory ResponseExtractor responseExtractor = response -> { - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(response.getBody())); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(response.getBody(), StandardCharsets.UTF_8)); String line; while((line = bufferedReader.readLine()) != null) { results.add(line); diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java index 518ae475..39a6c2e6 100644 --- a/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java +++ b/marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java @@ -318,7 +318,8 @@ else if (failedEvents.get() + successfulEvents.get() != uriCount) { else if (failedEvents.get() > 0 && successfulEvents.get() > 0) { status = JobStatus.FINISHED_WITH_ERRORS; } - else if (failedEvents.get() == 0 && successfulEvents.get() > 0) { + //empty collector or no failure events => JobStatus.FINISHED + else if ((failedEvents.get() == 0 && successfulEvents.get() > 0) || uriCount == 0) { status = JobStatus.FINISHED; } else { diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/PiiE2E.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/PiiE2E.java index efddb4f2..efbab0b5 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/PiiE2E.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/PiiE2E.java @@ -108,6 +108,7 @@ public static void setupAll() throws Exception public void setup() { Assumptions.assumeTrue(!(isCertAuth() || isSslRun())); + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME); installHubModules(); installUserModules(getHubAdminConfig(), true); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/EmptyCollectorTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/EmptyCollectorTest.java index 7f5d2c6a..e3d41109 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/EmptyCollectorTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/collector/EmptyCollectorTest.java @@ -86,7 +86,7 @@ public void runCollector() { assertEquals(0, node.get("successfulEvents").asInt()); assertEquals(0, node.get("failedEvents").asInt()); assertEquals(0, node.get("failedBatches").asInt()); - assertEquals("FAILED", node.get("status").asText()); + assertEquals("FINISHED", node.get("status").asText()); } } diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java index 7d4314ee..573db5c5 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubInstallTest.java @@ -20,10 +20,7 @@ import com.marklogic.hub.ApplicationConfig; import org.apache.commons.io.FileUtils; import org.custommonkey.xmlunit.XMLUnit; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -89,6 +86,7 @@ public void testTelemetryInstallCount() throws IOException { @Test public void testInstallHubModules() throws IOException { + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); assertTrue(getDataHub().isInstalled().isInstalled()); assertTrue(getModulesFile("/com.marklogic.hub/config.xqy").startsWith(getResource("data-hub-test/core-modules/config.xqy"))); @@ -108,6 +106,7 @@ public void getHubModulesVersion() throws IOException { @Test public void testInstallUserModules() throws IOException, ParserConfigurationException, SAXException, URISyntaxException { + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); URL url = DataHubInstallTest.class.getClassLoader().getResource("data-hub-test"); String path = Paths.get(url.toURI()).toFile().getAbsolutePath(); File srcDir = new File(path); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubTest.java index a53c7769..0341c4af 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/DataHubTest.java @@ -27,6 +27,7 @@ import org.custommonkey.xmlunit.XMLUnit; import org.easymock.EasyMock; import org.jdom2.Namespace; +import org.junit.jupiter.api.Assumptions; import org.springframework.beans.factory.annotation.Autowired; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -239,6 +240,7 @@ public void testPreFlightCheckHubAlreadyInstalled() { @Test public void testPreFlightCheckStagingPortTaken() { + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); List list = new ArrayList<>(); expect(serverManager.getAsXml()).andReturn(new ResourcesFragment(new Fragment(" /manage/v2/servers 2017-10-01T19:15:38.140598-04:00 0.030447 groups 1 /manage/v2/groups/Default 3341249095562999141 Default 8 3341249095562999141 Default /manage/v2/servers/Admin?group-id=Default http Security 7776582106827683360 Admin 3341249095562999141 Default /manage/v2/servers/App-Services?group-id=Default http Documents Modules 4626187627163603518 App-Services 3341249095562999141 Default /manage/v2/servers/port-stealer?group-id=Default http data-hub-STAGING data-hub-MODULES 10842453788764821876 port-stealer 3341249095562999141 Default /manage/v2/servers/HealthCheck?group-id=Default http App-Services 16530884482127520539 HealthCheck 3341249095562999141 Default /manage/v2/servers/Manage?group-id=Default http App-Services 15392081774558336614 Manage root default /manage/v2 list metrics /manage/v2/servers?view=metrics list package /manage/v2/servers?view=package list schema /manage/v2/servers?view=schema list status /manage/v2/servers?view=status ", list.toArray(new Namespace[] {})))); expect(serverManager.getPropertiesAsXml("Admin")).andReturn(new ResourcesFragment(new Fragment(" Admin Default http true Admin/ digest 8001 false true false
0.0.0.0
512 32 30 5 3600 10800 600 500 100 3600 1000 10000 http://marklogic.com/collation/ true 0 true false false false 0.9-ml contemporaneous fast none UTF-8 default default default default default default default none default omit default default compatible /error-switch.xqy rewriter.xqy false true true false false false false ALL:!LOW:@STRENGTH true Security nobody ", list.toArray(new Namespace[] {})))); @@ -260,6 +262,7 @@ public void testPreFlightCheckStagingPortTaken() { @Test public void testPreFlightCheckStagingPortTakenAndBadVersion() { + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); List list = new ArrayList<>(); expect(serverManager.getAsXml()).andReturn(new ResourcesFragment(new Fragment(" /manage/v2/servers 2017-10-01T19:15:38.140598-04:00 0.030447 groups 1 /manage/v2/groups/Default 3341249095562999141 Default 8 3341249095562999141 Default /manage/v2/servers/Admin?group-id=Default http Security 7776582106827683360 Admin 3341249095562999141 Default /manage/v2/servers/App-Services?group-id=Default http Documents Modules 4626187627163603518 App-Services 3341249095562999141 Default /manage/v2/servers/port-stealer?group-id=Default http data-hub-STAGING data-hub-MODULES 10842453788764821876 port-stealer 3341249095562999141 Default /manage/v2/servers/HealthCheck?group-id=Default http App-Services 16530884482127520539 HealthCheck 3341249095562999141 Default /manage/v2/servers/Manage?group-id=Default http App-Services 15392081774558336614 Manage root default /manage/v2 list metrics /manage/v2/servers?view=metrics list package /manage/v2/servers?view=package list schema /manage/v2/servers?view=schema list status /manage/v2/servers?view=status ", list.toArray(new Namespace[] {})))); expect(serverManager.getPropertiesAsXml("Admin")).andReturn(new ResourcesFragment(new Fragment(" Admin Default http true Admin/ digest 8001 false true false
0.0.0.0
512 32 30 5 3600 10800 600 500 100 3600 1000 10000 http://marklogic.com/collation/ true 0 true false false false 0.9-ml contemporaneous fast none UTF-8 default default default default default default default none default omit default default compatible /error-switch.xqy rewriter.xqy false true true false false false false ALL:!LOW:@STRENGTH true Security nobody ", list.toArray(new Namespace[] {})))); @@ -281,6 +284,7 @@ public void testPreFlightCheckStagingPortTakenAndBadVersion() { @Test public void testPreFlightCheckFinalPortTaken() { + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); List list = new ArrayList<>(); expect(serverManager.getAsXml()).andReturn(new ResourcesFragment(new Fragment(" /manage/v2/servers 2017-10-01T19:15:38.140598-04:00 0.030447 groups 1 /manage/v2/groups/Default 3341249095562999141 Default 8 3341249095562999141 Default /manage/v2/servers/Admin?group-id=Default http Security 7776582106827683360 Admin 3341249095562999141 Default /manage/v2/servers/App-Services?group-id=Default http Documents Modules 4626187627163603518 App-Services 3341249095562999141 Default /manage/v2/servers/port-stealer?group-id=Default http data-hub-STAGING data-hub-MODULES 10842453788764821876 port-stealer 3341249095562999141 Default /manage/v2/servers/HealthCheck?group-id=Default http App-Services 16530884482127520539 HealthCheck 3341249095562999141 Default /manage/v2/servers/Manage?group-id=Default http App-Services 15392081774558336614 Manage root default /manage/v2 list metrics /manage/v2/servers?view=metrics list package /manage/v2/servers?view=package list schema /manage/v2/servers?view=schema list status /manage/v2/servers?view=status ", list.toArray(new Namespace[] {})))); expect(serverManager.getPropertiesAsXml("Admin")).andReturn(new ResourcesFragment(new Fragment(" Admin Default http true Admin/ digest 8001 false true false
0.0.0.0
512 32 30 5 3600 10800 600 500 100 3600 1000 10000 http://marklogic.com/collation/ true 0 true false false false 0.9-ml contemporaneous fast none UTF-8 default default default default default default default none default omit default default compatible /error-switch.xqy rewriter.xqy false true true false false false false ALL:!LOW:@STRENGTH true Security nobody ", list.toArray(new Namespace[] {})))); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubConfigTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubConfigTest.java index 11d39977..80bbe89b 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubConfigTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubConfigTest.java @@ -11,6 +11,7 @@ import com.marklogic.hub.error.DataHubConfigurationException; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -53,7 +54,8 @@ public void testAppConfigDefaultProps() { @Test public void applyFinalConnectionPropsToDefaultRestConnection() { - + + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); AppConfig config = adminHubConfig.getAppConfig(); assertEquals(new Integer(8011), config.getRestPort(), diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java index bd7cd57a..4c394f65 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/core/HubProjectTest.java @@ -163,7 +163,8 @@ public void testUserModulesDeployTimestampFilePath() { @Test public void upgrade300To403ToCurrentVersion() throws Exception { - Assumptions.assumeFalse((isCertAuth() || isSslRun())); + Assumptions.assumeFalse((isCertAuth() || isSslRun())); + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); final String projectPath = "build/tmp/upgrade-projects/dhf403from300"; final File projectDir = Paths.get(projectPath).toFile(); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/entity/EntityServicesAlignmentTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/entity/EntityServicesAlignmentTest.java index ad69451c..7a5fceea 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/entity/EntityServicesAlignmentTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/entity/EntityServicesAlignmentTest.java @@ -13,6 +13,7 @@ import com.marklogic.hub.HubTestBase; import com.marklogic.hub.util.FileUtil; import com.marklogic.hub.util.HubModuleManager; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -77,6 +78,7 @@ public void testDeployTDEWithNoEntities() { @Test public void testDeployTDE() throws Exception { + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); installEntities(); getDataHub().clearDatabase(HubConfig.DEFAULT_FINAL_SCHEMAS_DB_NAME); diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java index 93357f50..86e56d00 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java @@ -30,6 +30,7 @@ import org.custommonkey.xmlunit.XMLAssert; import org.custommonkey.xmlunit.XMLUnit; import org.junit.Assert; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -250,6 +251,7 @@ public void testEnvelopeSJSXML() throws IOException, SAXException { @Test public void testCreateandDeployFlowWithHubUser() throws IOException { + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); scaffolding.createFlow(ENTITY, "FlowWithHubUser", FlowType.HARMONIZE, CodeFormat.XQUERY, DataFormat.JSON, false); Files.copy(getResourceStream("flow-runner-test/collector2.xqy"), diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/StreamCollectorTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/StreamCollectorTest.java index e9d7eaa5..6da851d6 100644 --- a/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/StreamCollectorTest.java +++ b/marklogic-data-hub/src/test/java/com/marklogic/hub_integration/StreamCollectorTest.java @@ -32,6 +32,7 @@ import com.marklogic.hub.util.FileUtil; import org.custommonkey.xmlunit.XMLUnit; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -144,6 +145,7 @@ public void runCollector() { // there is a custom content plugin that throws an error. This code uses the stopOnFailure // option to halt execution. This allows us to test that the collector runs to completion while not // having to wait for the entire harmonize flow to finish. + Assumptions.assumeFalse(getHubAdminConfig().getIsProvisionedEnvironment()); assertEquals(DOC_COUNT, getStagingDocCount()); assertEquals(0, getFinalDocCount()); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize", diff --git a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/RunFlowTask.groovy b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/RunFlowTask.groovy index 345ea793..95a84049 100644 --- a/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/RunFlowTask.groovy +++ b/ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/RunFlowTask.groovy @@ -116,7 +116,7 @@ class RunFlowTask extends HubTask { Map options = new HashMap<>() project.ext.properties.each { key, value -> if (key.toString().startsWith("dhf.")) { - options.put(key, value) + options.put(key.minus("dhf."), value) } } println("Running Flow: [" + entityName + ":" + flowName + "]" + diff --git a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy index e010eb87..81ada4c0 100644 --- a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy +++ b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/GenerateTDETemplateFromEntityTaskTest.groovy @@ -3,6 +3,7 @@ package com.marklogic.gradle.task import com.marklogic.hub.HubConfig import org.apache.commons.io.FileUtils import org.gradle.testkit.runner.UnexpectedBuildFailure +import spock.lang.IgnoreIf import java.nio.file.Path import java.nio.file.Paths @@ -16,6 +17,7 @@ class GenerateTDETemplateFromEntityTaskTest extends BaseTest { clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_JOB_NAME); } + @IgnoreIf({ System.getProperty('mlIsProvisionedEnvironment') }) def "GenerateTDETEmplates"() { given: def pluginDir = Paths.get(hubConfig().hubProject.projectDirString).resolve("plugins") diff --git a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/InstalledTests.groovy b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/InstalledTests.groovy index de6d4025..5de8e2ab 100644 --- a/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/InstalledTests.groovy +++ b/ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/InstalledTests.groovy @@ -91,16 +91,8 @@ class InstalledTests extends BaseTest { } def "test run flow with invalid flow"() { - setup: "append properties for task name and flow name" - propertiesFile << """ - ext { - entityName=my-entity - flowName=my-flow-not-found - } - """ - when: "hubRunFlow is Run" - def result = runFailTask('hubRunFlow', '-i') + def result = runFailTask('hubRunFlow', '-PentityName=my-new-entity', '-PflowName=my-flow-not-found', '-i') then: "it should run with errors" notThrown(UnexpectedBuildSuccess) @@ -124,16 +116,19 @@ class InstalledTests extends BaseTest { installStagingDoc("/employee2.xml", meta, new File("src/test/resources/run-flow-test/employee2.xml").text) assert (getStagingDocCount() == 2) assert (getFinalDocCount() == 0) - + String result; installModule("/entities/my-new-entity/harmonize/my-new-harmonize-flow/content/content.xqy", "run-flow-test/content.xqy") when: - println(runTask('hubRunFlow', '-PentityName=my-new-entity', '-PflowName=my-new-harmonize-flow', '-i').getOutput()) + result = runTask('hubRunFlow', '-Pdhf.key=value', '-PshowOptions=true','-PentityName=my-new-entity', '-PflowName=my-new-harmonize-flow', '-i').getOutput() + println(result) then: notThrown(UnexpectedBuildFailure) getStagingDocCount() == 2 getFinalDocCount() == 2 + assert(result.contains("key = value")) + assert(! result.contains("dhf.key = value")) assertXMLEqual(getXmlFromResource("run-flow-test/harmonized1.xml"), hubConfig().newFinalClient().newDocumentManager().read("/employee1.xml").next().getContent(new DOMHandle()).get()) assertXMLEqual(getXmlFromResource("run-flow-test/harmonized2.xml"), hubConfig().newFinalClient().newDocumentManager().read("/employee2.xml").next().getContent(new DOMHandle()).get()) }