diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..090ddfba --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,188 @@ +pipeline { + options { + buildDiscarder(logRotator(numToKeepStr: '10')) + disableConcurrentBuilds() + } + + agent { + kubernetes { + cloud 'gke-autopilot' + workspaceVolume dynamicPVC(storageClassName: 'premium-rwo', accessModes: 'ReadWriteOnce', requestsSize: '6Gi') + defaultContainer 'openjdk' + yamlFile 'jenkins-k8s-pod.yaml' + } + } + + stages { + stage('Scale SolrCloud') { + steps { + container('kubectl') {} + } + } + + stage('Provision Gradle') { + options { + timeout (time: 20, unit: "MINUTES") + } + steps { + sh './gradlew --no-watch-fs' + } + } + + stage('–– Core lib ––') { + stages { + stage('Compile') { + options { + timeout (time: 1, unit: "HOURS") + } + steps { + sh './gradlew --no-watch-fs ' + + '-Pflyway.url=jdbc:postgresql://localhost:5432/postgres ' + + '-Pflyway.user=postgres ' + + '-Pflyway.password=postgres ' + + '-Pflyway.locations=filesystem:./schemas/flyway/gxa ' + + '-Pflyway.schemas=gxa ' + + 'flywayMigrate' + sh './gradlew --no-watch-fs ' + + '-PdataFilesLocation=/test-data ' + + '-PexperimentFilesLocation=/test-data/gxa ' + + '-PexperimentDesignLocation=/root/expdesign-rw ' + + '-PjdbcUrl=jdbc:postgresql://localhost:5432/postgres?currentSchema=gxa ' + + '-PjdbcUsername=postgres ' + + '-PjdbcPassword=postgres ' + + '-PzkHosts=gxa-solrcloud-zookeeper-0.gxa-solrcloud-zookeeper-headless.jenkins-gene-expression.svc.cluster.local:2181,gxa-solrcloud-zookeeper-1.gxa-solrcloud-zookeeper-headless.jenkins-gene-expression.svc.cluster.local:2181,gxa-solrcloud-zookeeper-2.gxa-solrcloud-zookeeper-headless.jenkins-gene-expression.svc.cluster.local:2181 ' + + '-PsolrHosts=http://gxa-solrcloud-0.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr,http://gxa-solrcloud-1.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr,http://gxa-solrcloud-2.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr,http://gxa-solrcloud-3.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr ' + + '-PsolrUser=solr ' + + '-PsolrPassword=SolrRocks ' + + ':atlas-web-core:testClasses' + } + } + + stage('Test') { + options { + timeout (time: 2, unit: "HOURS") + } + steps { + sh './gradlew --no-watch-fs -PtestResultsPath=ut :atlas-web-core:test --tests *Test' + sh './gradlew --no-watch-fs -PtestResultsPath=it :atlas-web-core:test --tests *IT' + sh './gradlew --no-watch-fs :atlas-web-core:jacocoTestReport' + } + } + } + } + + + stage('–– Web app ––') { + stages { + stage('Compile') { + options { + timeout (time: 1, unit: "HOURS") + } + steps { + sh './gradlew --no-watch-fs ' + + '-Pflyway.url=jdbc:postgresql://localhost:5432/postgres ' + + '-Pflyway.user=postgres ' + + '-Pflyway.password=postgres ' + + '-Pflyway.locations=filesystem:./schemas/flyway/gxa ' + + '-Pflyway.schemas=gxa ' + + 'flywayMigrate' + sh './gradlew --no-watch-fs ' + + '-PdataFilesLocation=/test-data ' + + '-PexperimentFilesLocation=/test-data/gxa ' + + '-PexperimentDesignLocation=/root/expdesign-rw ' + + '-PjdbcUrl=jdbc:postgresql://localhost:5432/postgres?currentSchema=gxa ' + + '-PjdbcUsername=postgres ' + + '-PjdbcPassword=postgres ' + + '-PzkHosts=gxa-solrcloud-zookeeper-0.gxa-solrcloud-zookeeper-headless.jenkins-gene-expression.svc.cluster.local:2181,gxa-solrcloud-zookeeper-1.gxa-solrcloud-zookeeper-headless.jenkins-gene-expression.svc.cluster.local:2181,gxa-solrcloud-zookeeper-2.gxa-solrcloud-zookeeper-headless.jenkins-gene-expression.svc.cluster.local:2181 ' + + '-PsolrHosts=http://gxa-solrcloud-0.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr,http://gxa-solrcloud-1.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr,http://gxa-solrcloud-2.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr,http://gxa-solrcloud-3.gxa-solrcloud-headless.jenkins-gene-expression.svc.cluster.local:8983/solr ' + + '-PsolrUser=solr ' + + '-PsolrPassword=SolrRocks ' + + ':app:testClasses' + } + } + + stage('Test') { + options { + timeout (time: 2, unit: "HOURS") + } + steps { + sh './gradlew --no-watch-fs -PtestResultsPath=ut :app:test --tests *Test' + sh './gradlew -PsolrUser=solr -PsolrPassword=SolrRocks --no-watch-fs -PtestResultsPath=it -PexcludeTests=**/*WIT.class :app:test --tests *IT' + sh './gradlew -PsolrUser=solr -PsolrPassword=SolrRocks --no-watch-fs -PtestResultsPath=e2e :app:test --tests *WIT' + sh './gradlew --no-watch-fs :app:jacocoTestReport' + } + } + + stage('–– Build ––') { + when { anyOf { + branch 'develop'; branch 'main'; branch 'release/*' + } } + stages { + stage('Provision Node.js build environment') { + options { + timeout (time: 1, unit: "HOURS") + } + steps { + // To avoid the unpleasant: + // Err:4 http://deb.debian.org/debian bullseye-updates InRelease + // Connection timed out [IP: 199.232.174.132 80] + sh 'echo \'APT::Acquire::Retries "10";\' > /etc/apt/apt.conf.d/80-retries' + + // Required by node_modules/cwebp-bin + // /home/jenkins/agent/workspace/298051-test-and-build-in-jenkins/app/src/main/javascript/node_modules/cwebp-bin/vendor/cwebp: + // error while loading shared libraries: libGL.so.1: cannot open shared object file: No such file or directory + // + // ⚠ cwebp pre-build test failed + // ℹ compiling from source + // ✖ Error: Command failed: /bin/sh -c ./configure --disable-shared --prefix="/home/jenkins/agent/workspace/298051-test-and-build-in-jenkins/app/src/main/javascript/bundles/experiment-page/node_modules/cwebp-bin/vendor" --bindir="/home/jenkins/agent/workspace/298051-test-and-build-in-jenkins/app/src/main/javascript/bundles/experiment-page/node_modules/cwebp-bin/vendor" + // configure: error: in `/home/jenkins/agent/workspace/298051-test-and-build-in-jenkins/app/src/main/javascript/bundles/experiment-page/node_modules/cwebp-bin/2525557b-9d4c-4886-93b3-8cbfa3b76a32': + // configure: error: no acceptable C compiler found in $PATH + sh 'apt update && apt install -y libglu1-mesa gcc' + sh 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash' + sh '. ~/.bashrc && nvm install 14 --lts' + sh '. ~/.bashrc && npm install -g npm-check-updates' + } + } + + stage('Update and build ES bundles') { + options { + timeout (time: 1, unit: "HOURS") + } + steps { + sh 'if [ env.BRANCH_NAME = "develop" ]; then WEBPACK_OPTS=-i; else WEBPACK_OPTS=-ip; fi; ' + + '. ~/.bashrc && ./compile-front-end-packages.sh ${WEBPACK_OPTS}' + } + } + + stage('Assemble WAR file') { + options { + timeout (time: 1, unit: "HOURS") + } + steps { + sh './gradlew --no-watch-fs :app:war' + archiveArtifacts artifacts: 'webapps/gxa.war', fingerprint: true + } + } + } + } + } + } + } + + + + post { + always { + junit 'atlas-web-core/build/ut/**/*.xml' + junit 'atlas-web-core/build/it/**/*.xml' + + junit 'app/build/ut/**/*.xml' + junit 'app/build/it/**/*.xml' + junit 'app/build/e2e/**/*.xml' + + archiveArtifacts artifacts: 'atlas-web-core/build/reports/**', fingerprint: true, allowEmptyArchive: true + archiveArtifacts artifacts: 'app/build/reports/**', fingerprint: true, allowEmptyArchive: true + archiveArtifacts artifacts: 'app/src/main/webapp/resources/js-bundles/report.html', fingerprint: true, allowEmptyArchive: true + } + } +} diff --git a/app/src/main/java/uk/ac/ebi/atlas/controllers/rest/DASFeaturesController.java b/app/src/main/java/uk/ac/ebi/atlas/controllers/rest/DASFeaturesController.java index fc3b081b..88881e76 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/controllers/rest/DASFeaturesController.java +++ b/app/src/main/java/uk/ac/ebi/atlas/controllers/rest/DASFeaturesController.java @@ -63,8 +63,7 @@ public String getDifferentialJsonResults( Experiment experiment = experimentTrader.getPublicExperiment(dbe.getExperimentAccession()); FactorSet factorsForAssayGroup = - FactorSet.create(experiment.getExperimentDesign() - .getFactorValues(testAssayGroup.getFirstAssayId())); + FactorSet.create(experiment.getFactorValues(testAssayGroup.getFirstAssayId())); for (Factor factor : factorsForAssayGroup) { factorValuesByType.put(factor.getType(), factor.getValue()); } diff --git a/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLines.java b/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLines.java index 7c5b68e1..db62be88 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLines.java +++ b/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLines.java @@ -35,7 +35,7 @@ private LinkedHashSet> buildAssayGroupsDetails(BaselineExp } private void populateSamples(BaselineExperiment experiment, String assayAccession, AssayGroup assayGroup) { - for (SampleCharacteristic sample : experiment.getExperimentDesign().getSampleCharacteristics(assayAccession)) { + for (SampleCharacteristic sample : experiment.getSampleCharacteristics(assayAccession)) { ImmutableList line = ImmutableList.of( experiment.getAccession(), @@ -49,7 +49,7 @@ private void populateSamples(BaselineExperiment experiment, String assayAccessio } private void populateFactors(BaselineExperiment experiment, String assayAccession, AssayGroup assayGroup) { - FactorSet factorSet = experiment.getExperimentDesign().getFactors(assayAccession); + FactorSet factorSet = experiment.getFactors(assayAccession); if (factorSet != null) { for (Factor factor : factorSet) { ImmutableList line = ImmutableList.of(experiment.getAccession(), assayGroup.getId(), "factor", diff --git a/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/DifferentialExperimentContrastLines.java b/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/DifferentialExperimentContrastLines.java index c3f7ffc9..e7f08577 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/DifferentialExperimentContrastLines.java +++ b/app/src/main/java/uk/ac/ebi/atlas/ebeyedump/DifferentialExperimentContrastLines.java @@ -43,7 +43,7 @@ private void populateSamples(DifferentialExperiment experiment, String assayAccession, Contrast contrast, String value) { - for (SampleCharacteristic sample : experiment.getExperimentDesign().getSampleCharacteristics(assayAccession)) { + for (SampleCharacteristic sample : experiment.getSampleCharacteristics(assayAccession)) { ImmutableList line = ImmutableList.of( experiment.getAccession(), @@ -61,7 +61,7 @@ private void populateFactors(DifferentialExperiment experiment, String assayAccession, Contrast contrast, String value) { - FactorSet factorSet = experiment.getExperimentDesign().getFactors(assayAccession); + FactorSet factorSet = experiment.getFactors(assayAccession); if (factorSet != null) { for (Factor factor : factorSet) { ImmutableList line = diff --git a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/baseline/BaselineExperimentPageService.java b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/baseline/BaselineExperimentPageService.java index 6a07d183..d32737fe 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/baseline/BaselineExperimentPageService.java +++ b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/baseline/BaselineExperimentPageService.java @@ -101,7 +101,7 @@ private JsonArray constructColumnHeaders(BaselineRequestContext requestContex o.add("assayGroupSummary", new AssayGroupSummaryBuilder() .forAssayGroup(experiment.getDataColumnDescriptor(dataColumnDescriptor.getId())) - .withExperimentDesign(experiment.getExperimentDesign()) + .withExperimentDesign(experiment) .build().toJson()); result.add(o); } diff --git a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/content/ExperimentPageContentService.java b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/content/ExperimentPageContentService.java index 982c1584..e5ab77df 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/content/ExperimentPageContentService.java +++ b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/content/ExperimentPageContentService.java @@ -10,14 +10,12 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Component; import uk.ac.ebi.atlas.commons.readers.TsvStreamer; -import uk.ac.ebi.atlas.experimentpage.ExperimentDesignFile; import uk.ac.ebi.atlas.experimentpage.ExternallyAvailableContentService; import uk.ac.ebi.atlas.experimentpage.json.JsonBaselineExperimentController; import uk.ac.ebi.atlas.experimentpage.qc.MicroarrayQcFiles; import uk.ac.ebi.atlas.experimentpage.qc.QcReportController; import uk.ac.ebi.atlas.model.download.ExternallyAvailableContent; import uk.ac.ebi.atlas.model.experiment.Experiment; -import uk.ac.ebi.atlas.model.experiment.ExperimentDesignTable; import uk.ac.ebi.atlas.model.experiment.ExperimentType; import uk.ac.ebi.atlas.model.experiment.sample.ReportsGeneExpression; import uk.ac.ebi.atlas.resource.DataFileHub; @@ -85,12 +83,6 @@ private JsonObject experimentPageContentForExperiment(final Experiment getExperimentVariablesAsHeatmapF Streams.concat( Stream.of(experiment.getDisplayDefaults().getDefaultQueryFactorType()), experiment.getDisplayDefaults().getFactorTypes().stream(), - experiment.getExperimentDesign().getFactorHeaders().stream()) + experiment.getExperimentalFactorHeaders().stream()) .filter(StringUtils::isNotBlank) // defaultQueryFactorType in diff experiments is "" .map(Factor::normalize) .collect(toImmutableSet()); @@ -104,7 +104,7 @@ public static ImmutableList getExperimentVariablesAsHeatmapF mapAssayGroupsToFactors(assayId2AssayGroup, factorHeader, experiment))); Stream sampleCharacteristicGroups = - experiment.getExperimentDesign().getSampleCharacteristicHeaders().stream() + experiment.getSampleCharacteristicHeaders().stream() .filter(sampleCharacteristicHeader -> !orderedFactors.contains(Factor.normalize(sampleCharacteristicHeader))) .map(sampleCharacteristicHeader -> HeatmapFilterGroup.create( @@ -126,9 +126,9 @@ private static ImmutableSetMultimap mapAssayGroupsToFactors( @NotNull String factorHeader, @NotNull Experiment<@NotNull ?> experiment) { return assayId2AssayGroup.keySet().stream() - .filter(assayId -> experiment.getExperimentDesign().getFactor(assayId, factorHeader) != null) + .filter(assayId -> experiment.getFactor(assayId, factorHeader) != null) .collect(flatteningToImmutableSetMultimap( - assayId -> experiment.getExperimentDesign().getFactor(assayId, factorHeader).getValue(), + assayId -> experiment.getFactor(assayId, factorHeader).getValue(), assayId -> assayId2AssayGroup.get(assayId).stream())); } @@ -139,14 +139,10 @@ private static ImmutableSetMultimap mapAssayGroupsToSampleCharac @NotNull Experiment<@NotNull?> experiment) { return assayId2AssayGroup.keySet().stream() .filter(assayId -> - experiment - .getExperimentDesign() - .getSampleCharacteristic(assayId, sampleCharacteristicHeader) != null) + experiment.getSampleCharacteristic(assayId, sampleCharacteristicHeader) != null) .collect(flatteningToImmutableSetMultimap( assayId -> - experiment - .getExperimentDesign() - .getSampleCharacteristic(assayId, sampleCharacteristicHeader).getValue(), + experiment.getSampleCharacteristic(assayId, sampleCharacteristicHeader).getValue(), assayId -> assayId2AssayGroup.get(assayId).stream())); } } diff --git a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/differential/DifferentialExperimentPageService.java b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/differential/DifferentialExperimentPageService.java index 44b0f4d6..7ee34e79 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/differential/DifferentialExperimentPageService.java +++ b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/differential/DifferentialExperimentPageService.java @@ -76,7 +76,7 @@ private JsonArray constructColumnHeaders(Iterable contrasts, Different JsonObject o = contrast.toJson(); o.add("contrastSummary", new ContrastSummaryBuilder() .forContrast(contrast) - .withExperimentDesign(differentialExperiment.getExperimentDesign()) + .withExperiment(differentialExperiment) .withExperimentDescription(differentialExperiment.getDescription()) .build().toJson()); o.add("resources", contrastImages.get(contrast.getId())); diff --git a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/json/opentargets/EvidenceService.java b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/json/opentargets/EvidenceService.java index 54217e4d..13d143f0 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/json/opentargets/EvidenceService.java +++ b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/json/opentargets/EvidenceService.java @@ -9,6 +9,7 @@ import org.apache.commons.lang3.tuple.Pair; import uk.ac.ebi.atlas.commons.readers.TsvStreamer; import uk.ac.ebi.atlas.commons.streams.ObjectInputStream; +import uk.ac.ebi.atlas.model.experiment.Experiment; import uk.ac.ebi.atlas.model.experiment.sample.AssayGroup; import uk.ac.ebi.atlas.model.GeneProfilesList; import uk.ac.ebi.atlas.model.OntologyTerm; @@ -108,7 +109,7 @@ public void evidenceForExperiment(E experiment, private boolean shouldSkip(E experiment) { return !experiment.getSpecies().isUs() || experiment.getType().isMicroRna() || - cellLineAsSampleCharacteristicButNoDiseaseAsFactor(experiment.getExperimentDesign()); + cellLineAsSampleCharacteristicButNoDiseaseAsFactor(experiment); } private void piecesOfEvidence(E experiment, @@ -424,14 +425,14 @@ enum CONFIDENCE { public abstract SampleCharacteristic organismPart(); public static Optional tryCreate(DifferentialExperiment experiment, Contrast contrast) { - var biosampleInfo = getBiosampleInfo(experiment.getExperimentDesign(), contrast.getTestAssayGroup()); - var diseaseInfo = getDiseaseInfo(experiment.getExperimentDesign(), contrast.getTestAssayGroup()); + var biosampleInfo = getBiosampleInfo(experiment, contrast.getTestAssayGroup()); + var diseaseInfo = getDiseaseInfo(experiment, contrast.getTestAssayGroup()); if (biosampleInfo.isPresent() && diseaseInfo.isPresent()) { return Optional.of( DiseaseAssociation.create( biosampleInfo.get(), - experiment.getExperimentDesign(), + experiment, contrast, experiment.doesContrastHaveCttvPrimaryAnnotation(contrast), diseaseInfo.get())); @@ -441,18 +442,18 @@ public static Optional tryCreate(DifferentialExperiment expe } public static DiseaseAssociation create(SampleCharacteristic biosampleInfo, - ExperimentDesign experimentDesign, + Experiment experiment, Contrast contrast, boolean isCttvPrimary, SampleCharacteristic diseaseInfo) { - var referenceSampleLabel = factorBasedSummaryLabel(experimentDesign, contrast.getReferenceAssayGroup()); - var testSampleLabel = factorBasedSummaryLabel(experimentDesign, contrast.getTestAssayGroup()); + var referenceSampleLabel = factorBasedSummaryLabel(experiment, contrast.getReferenceAssayGroup()); + var testSampleLabel = factorBasedSummaryLabel(experiment, contrast.getTestAssayGroup()); var confidence = determineStudyConfidence( - experimentDesign, diseaseInfo, contrast.getTestAssayGroup(), isCttvPrimary); + experiment, diseaseInfo, contrast.getTestAssayGroup(), isCttvPrimary); var organismPart = Optional.ofNullable( - experimentDesign.getSampleCharacteristic( + experiment.getSampleCharacteristic( contrast.getTestAssayGroup().getFirstAssayId(), "organism part")) .orElse(SampleCharacteristic.create("organism part", "")); @@ -474,10 +475,9 @@ public static DiseaseAssociation create(SampleCharacteristic biosampleInfo, https://github.com/opentargets/json_schema/blob/master/src/evidence/expression.json "test_sample", "reference_sample" */ - private static String factorBasedSummaryLabel(final ExperimentDesign experimentDesign, AssayGroup assayGroup) { - return experimentDesign.getFactorValues(assayGroup.getFirstAssayId()).values().stream() - .filter(StringUtils::isNotEmpty) - .collect(Collectors.joining("; ")); + private static String factorBasedSummaryLabel(final Experiment experiment, AssayGroup assayGroup) { + Stream factorValueStream = experiment.getFactorValues(assayGroup.getFirstAssayId()).values().stream(); + return factorValueStream.filter(StringUtils::isNotEmpty).collect(Collectors.joining("; ")); } /* @@ -485,12 +485,12 @@ private static String factorBasedSummaryLabel(final ExperimentDesign experimentD This will either be an organism part, a cell line, or a cell type. When we couldn't determine this we used to issue a warning asking curators to curate this. */ - private static Optional getBiosampleInfo(final ExperimentDesign experimentDesign, + private static Optional getBiosampleInfo(final Experiment experiment, AssayGroup testAssayGroup) { return Stream.of("organism part", "cell line", "cell type").flatMap( experimentalVariable -> { var matchingSampleCharacteristic = - experimentDesign.getSampleCharacteristic( + experiment.getSampleCharacteristic( testAssayGroup.getFirstAssayId(), experimentalVariable); return matchingSampleCharacteristic == null ? Stream.empty() : @@ -503,9 +503,11 @@ private static Optional getBiosampleInfo(final ExperimentD Original comment: # Go through the types (should probably always only be one)... */ - private static Optional getDiseaseInfo(final ExperimentDesign experimentDesign, + private static Optional getDiseaseInfo(final Experiment experiment, AssayGroup testAssayGroup) { - return experimentDesign.getSampleCharacteristics(testAssayGroup.getFirstAssayId()).stream() + Stream sampleCharacteristicStream = + experiment.getSampleCharacteristics(testAssayGroup.getFirstAssayId()).stream(); + return sampleCharacteristicStream .filter( sampleCharacteristic -> sampleCharacteristic.getHeader().toLowerCase().contains("disease") && @@ -526,11 +528,11 @@ private static Optional getDiseaseInfo(final ExperimentDes characteristics and something else is the factor e.g a treatment, the confidence is "low". */ - private static DiseaseAssociation.CONFIDENCE determineStudyConfidence(ExperimentDesign experimentDesign, + private static DiseaseAssociation.CONFIDENCE determineStudyConfidence(Experiment experiment, SampleCharacteristic diseaseCharacteristic, AssayGroup testAssayGroup, boolean isCttvPrimary) { - var factorSet = experimentDesign.getFactors(testAssayGroup.getFirstAssayId()); + var factorSet = experiment.getFactors(testAssayGroup.getFirstAssayId()); if (factorSet != null) { if (!factorSet.factorsByType.containsKey(Factor.normalize(diseaseCharacteristic.getHeader()))) { return DiseaseAssociation.CONFIDENCE.LOW; @@ -597,9 +599,9 @@ private String getMethodDescriptionFromAnalysisMethodsFile(E experiment) { If something's a factor then it is also a characteristic unless we've made a mistake. Example mistake was E-GEOD-23764. */ - private boolean cellLineAsSampleCharacteristicButNoDiseaseAsFactor(ExperimentDesign experimentDesign) { - return (experimentDesign.getSampleCharacteristicHeaders().contains("cell line") || - experimentDesign.getFactorHeaders().contains("cell line")) && - !experimentDesign.getFactorHeaders().contains("disease"); + private boolean cellLineAsSampleCharacteristicButNoDiseaseAsFactor(Experiment experiment) { + return (experiment.getSampleCharacteristicHeaders().contains("cell line") || + experiment.getExperimentalFactorHeaders().contains("cell line")) && + !experiment.getExperimentalFactorHeaders().contains("disease"); } } diff --git a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastSummaryController.java b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastSummaryController.java index 6ded26ad..f3280ef0 100644 --- a/app/src/main/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastSummaryController.java +++ b/app/src/main/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastSummaryController.java @@ -7,7 +7,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; -import uk.ac.ebi.atlas.model.experiment.ExperimentDesign; import uk.ac.ebi.atlas.model.experiment.sample.Contrast; import uk.ac.ebi.atlas.model.experiment.differential.DifferentialExperiment; import uk.ac.ebi.atlas.model.experiment.summary.ContrastSummary; @@ -44,10 +43,8 @@ public String getTooltipContrastContent(@RequestParam(value = "experimentAccessi throw new IllegalArgumentException("No contrast with ID " + contrastId + " found."); } - ExperimentDesign experimentDesign = differentialExperiment.getExperimentDesign(); - ContrastSummary contrastSummary = new ContrastSummaryBuilder() - .withExperimentDesign(experimentDesign) + .withExperiment(differentialExperiment) .forContrast(contrast) .withExperimentDescription(differentialExperiment.getDescription()) .build(); diff --git a/app/src/test/java/uk/ac/ebi/atlas/configuration/TestConfig.java b/app/src/test/java/uk/ac/ebi/atlas/configuration/TestConfig.java index 7d4c541d..6a64d674 100644 --- a/app/src/test/java/uk/ac/ebi/atlas/configuration/TestConfig.java +++ b/app/src/test/java/uk/ac/ebi/atlas/configuration/TestConfig.java @@ -6,6 +6,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.FilterType; import org.springframework.web.client.RestTemplate; +import uk.ac.ebi.atlas.experiments.ExperimentCellCountDao; @Configuration // Enabling component scanning will also load BasePathsConfig, JdbcConfig and SolrConfig, so just using this class as @@ -22,4 +23,9 @@ public class TestConfig { public RestTemplate restTemplate() { return new RestTemplate(); } + + @Bean + public ExperimentCellCountDao experimentCellCountDao() { + return __ -> 0; + } } diff --git a/app/src/test/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLinesTest.java b/app/src/test/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLinesTest.java index 77265f61..da9676a4 100644 --- a/app/src/test/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLinesTest.java +++ b/app/src/test/java/uk/ac/ebi/atlas/ebeyedump/BaselineExperimentAssayGroupsLinesTest.java @@ -6,29 +6,20 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import uk.ac.ebi.atlas.model.experiment.sample.AssayGroup; import uk.ac.ebi.atlas.model.experiment.sample.BiologicalReplicate; import uk.ac.ebi.atlas.model.OntologyTerm; +import uk.ac.ebi.atlas.model.experiment.sdrf.Factor; +import uk.ac.ebi.atlas.model.experiment.sdrf.FactorSet; import uk.ac.ebi.atlas.model.experiment.sdrf.SampleCharacteristic; -import uk.ac.ebi.atlas.model.experiment.ExperimentDesign; import uk.ac.ebi.atlas.model.experiment.baseline.BaselineExperiment; -import java.util.Arrays; -import java.util.Iterator; - -import static java.util.stream.Collectors.joining; -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.wrap; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class BaselineExperimentAssayGroupsLinesTest { - private static final Logger LOGGER = LoggerFactory.getLogger(BaselineExperimentAssayGroupsLinesTest.class); - private static final String EXPERIMENT_ACCESSION = "EXPERIMENT_ACCESSION"; private static final String ASSAY1 = "ASSAY1"; private static final String ASSAY2 = "ASSAY2"; @@ -72,39 +63,34 @@ public class BaselineExperimentAssayGroupsLinesTest { @Test public void lines() { - ExperimentDesign experimentDesign = new ExperimentDesign(); - - SampleCharacteristic sampleCharacteristic1 = - SampleCharacteristic.create(SAMPLE_HEADER, SAMPLE_VALUE1, SAMPLE_ONTOLOGY_TERM1); - SampleCharacteristic sampleCharacteristic2 = - SampleCharacteristic.create(SAMPLE_HEADER, SAMPLE_VALUE2, SAMPLE_ONTOLOGY_TERM2); - - - experimentDesign.putSampleCharacteristic(ASSAY1, SAMPLE_HEADER, sampleCharacteristic1); - experimentDesign.putFactor(ASSAY1, FACTOR_HEADER, FACTOR_VALUE1, FACTOR_ONTOLOGY_TERM1); + when(baselineExperiment.getAccession()) + .thenReturn(EXPERIMENT_ACCESSION); + when(baselineExperiment.getDataColumnDescriptors()) + .thenReturn(ImmutableList.of(ASSAY_GROUP1, ASSAY_GROUP2, ASSAY_GROUP3)); - experimentDesign.putSampleCharacteristic(ASSAY2, SAMPLE_HEADER, sampleCharacteristic2); - experimentDesign.putFactor(ASSAY2, FACTOR_HEADER, FACTOR_VALUE2, FACTOR_ONTOLOGY_TERM2); + var sampleCharacteristic1 = SampleCharacteristic.create(SAMPLE_HEADER, SAMPLE_VALUE1, SAMPLE_ONTOLOGY_TERM1); + when(baselineExperiment.getSampleCharacteristics(ASSAY1)).thenReturn(ImmutableList.of(sampleCharacteristic1)); - experimentDesign.putSampleCharacteristic(ASSAY3, SAMPLE_HEADER, SAMPLE_VALUE3); - experimentDesign.putFactor(ASSAY3, FACTOR_HEADER, FACTOR_VALUE3); + var sampleCharacteristic2 = SampleCharacteristic.create(SAMPLE_HEADER, SAMPLE_VALUE2, SAMPLE_ONTOLOGY_TERM2); + when(baselineExperiment.getSampleCharacteristics(ASSAY2)).thenReturn(ImmutableList.of(sampleCharacteristic2)); + var sampleCharacteristic3 = SampleCharacteristic.create(SAMPLE_HEADER, SAMPLE_VALUE3); + when(baselineExperiment.getSampleCharacteristics(ASSAY3)).thenReturn(ImmutableList.of(sampleCharacteristic3)); - when(baselineExperiment.getAccession()).thenReturn(EXPERIMENT_ACCESSION); - when(baselineExperiment.getDataColumnDescriptors()).thenReturn(ImmutableList.of(ASSAY_GROUP1, ASSAY_GROUP2, - ASSAY_GROUP3)); - when(baselineExperiment.getExperimentDesign()).thenReturn(experimentDesign); + var factorSet1 = new FactorSet(); + factorSet1.add(new Factor(FACTOR_HEADER, FACTOR_VALUE1, FACTOR_ONTOLOGY_TERM1)); + when(baselineExperiment.getFactors(ASSAY1)).thenReturn(factorSet1); - BaselineExperimentAssayGroupsLines subject = new BaselineExperimentAssayGroupsLines(baselineExperiment); + var factorSet2 = new FactorSet(); + factorSet2.add(new Factor(FACTOR_HEADER, FACTOR_VALUE2, FACTOR_ONTOLOGY_TERM2)); + when(baselineExperiment.getFactors(ASSAY2)).thenReturn(factorSet2); - Iterator lines = subject.iterator(); + var factorSet3 = new FactorSet(); + factorSet3.add(new Factor(FACTOR_HEADER, FACTOR_VALUE3)); + when(baselineExperiment.getFactors(ASSAY3)).thenReturn(factorSet3); - subject.forEach(line -> - LOGGER.info( - Arrays.stream(line) - .map(field -> wrap(field, "\"")) - .map(field -> isBlank(field) ? "\"\"" : field) - .collect(joining(",")))); + var subject = new BaselineExperimentAssayGroupsLines(baselineExperiment); + var lines = subject.iterator(); assertThat( lines.next(), diff --git a/app/src/test/java/uk/ac/ebi/atlas/experimentimport/admin/ExperimentOpsExecutionServiceTest.java b/app/src/test/java/uk/ac/ebi/atlas/experimentimport/admin/ExperimentOpsExecutionServiceTest.java index 1aa0e07d..bea063f8 100644 --- a/app/src/test/java/uk/ac/ebi/atlas/experimentimport/admin/ExperimentOpsExecutionServiceTest.java +++ b/app/src/test/java/uk/ac/ebi/atlas/experimentimport/admin/ExperimentOpsExecutionServiceTest.java @@ -6,14 +6,10 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import uk.ac.ebi.atlas.experimentimport.ExperimentCrud; -import uk.ac.ebi.atlas.experimentimport.ExperimentDto; import uk.ac.ebi.atlas.experimentimport.analyticsindex.AnalyticsIndexerManager; import uk.ac.ebi.atlas.experimentimport.coexpression.BaselineCoexpressionProfileLoader; -import java.util.Optional; - -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doNothing; @RunWith(MockitoJUnitRunner.class) public class ExperimentOpsExecutionServiceTest { @@ -28,14 +24,10 @@ public class ExperimentOpsExecutionServiceTest { @Mock private AnalyticsIndexerManager analyticsIndexerManager; - @Mock - private ExperimentDto experimentDto; - private ExperimentOpsExecutionService subject; @Before public void setUp() { - when(experimentCrudMock.readExperiment(ACCESSION)).thenReturn(Optional.of(experimentDto)); subject = new ExpressionAtlasExperimentOpsExecutionService( experimentCrudMock, @@ -43,21 +35,26 @@ public void setUp() { analyticsIndexerManager); } + + // No need to verify interactions: if any stub isn’t used Mockito will throw an exception + @Test public void updateExperimentDesignShouldRemoveExperimentFromCache() throws Exception { + doNothing().when(experimentCrudMock).updateExperimentDesign(ACCESSION); subject.attemptExecuteStatefulOp(ACCESSION, Op.UPDATE_DESIGN); -// verify(experimentTrader).removeExperimentFromCache(ACCESSION); } @Test public void updateExperimentToPrivateShouldRemoveExperimentFromAnalyticsIndex() throws Exception { + doNothing().when(analyticsIndexerManager).deleteFromAnalyticsIndex(ACCESSION); + doNothing().when(experimentCrudMock).updateExperimentPrivate(ACCESSION, true); subject.attemptExecuteStatefulOp(ACCESSION, Op.UPDATE_PRIVATE); - verify(analyticsIndexerManager).deleteFromAnalyticsIndex(ACCESSION); } @Test public void deleteExperimentShouldRemoveExperimentFromAnalyticsIndex() throws Exception { + doNothing().when(analyticsIndexerManager).deleteFromAnalyticsIndex(ACCESSION); + doNothing().when(experimentCrudMock).deleteExperiment(ACCESSION); subject.attemptExecuteStatefulOp(ACCESSION, Op.DELETE); - verify(analyticsIndexerManager).deleteFromAnalyticsIndex(ACCESSION); } } diff --git a/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/AssayGroupSummaryBuilderTest.java b/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/AssayGroupSummaryBuilderTest.java deleted file mode 100644 index 9760b9df..00000000 --- a/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/AssayGroupSummaryBuilderTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package uk.ac.ebi.atlas.experimentpage.tooltip; - -import org.junit.Test; -import uk.ac.ebi.atlas.model.experiment.sample.AssayGroup; -import uk.ac.ebi.atlas.model.experiment.ExperimentDesign; -import uk.ac.ebi.atlas.model.experiment.summary.AssayGroupSummary; -import uk.ac.ebi.atlas.model.experiment.summary.AssayGroupSummaryBuilder; -import uk.ac.ebi.atlas.model.experiment.summary.AssayProperty; -import uk.ac.ebi.atlas.model.experiment.summary.ContrastPropertyType; -import uk.ac.ebi.atlas.testutils.AssayGroupFactory; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.collection.IsIterableContainingInOrder.contains; - -public class AssayGroupSummaryBuilderTest { - private static final String ASSAY1 = "ASSAY1"; - private static final String FACTOR_HEADER = "factor header"; - private static final String FACTOR_VALUE = "factor value"; - - private static final String ASSAY2 = "ASSAY2"; - private static final String FACTOR_HEADER2 = "factor header 2"; - private static final String FACTOR_VALUE2 = "factor value 2"; - - private static final AssayGroup ASSAY_GROUP = AssayGroupFactory.create("id", ASSAY1, ASSAY2); - private static final String SAMPLE_HEADER = "sample header"; - private static final String SAMPLE_VALUE1 = "sample value 2"; - private static final String SAMPLE_VALUE2 = "sample value 2"; - - @Test - public void build() { - ExperimentDesign experimentDesign = new ExperimentDesign(); - experimentDesign.putFactor(ASSAY1, FACTOR_HEADER, FACTOR_VALUE); - experimentDesign.putFactor(ASSAY1, FACTOR_HEADER2, FACTOR_VALUE2); - - experimentDesign.putFactor(ASSAY2, FACTOR_HEADER, FACTOR_VALUE); - experimentDesign.putFactor(ASSAY2, FACTOR_HEADER2, FACTOR_VALUE2); - - experimentDesign.putSampleCharacteristic(ASSAY1, SAMPLE_HEADER, SAMPLE_VALUE1); - experimentDesign.putSampleCharacteristic(ASSAY2, SAMPLE_HEADER, SAMPLE_VALUE2); - - AssayGroupSummaryBuilder subject = new AssayGroupSummaryBuilder(). - forAssayGroup(ASSAY_GROUP). - withExperimentDesign(experimentDesign); - - AssayGroupSummary assayGroupSummary = subject.build(); - - assertThat(assayGroupSummary, contains( - new AssayProperty(FACTOR_HEADER, FACTOR_VALUE, ContrastPropertyType.FACTOR), - new AssayProperty(FACTOR_HEADER2, FACTOR_VALUE2, ContrastPropertyType.FACTOR), - new AssayProperty(SAMPLE_HEADER, SAMPLE_VALUE2, ContrastPropertyType.SAMPLE))); - } -} diff --git a/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastPropertyTest.java b/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastPropertyTest.java deleted file mode 100644 index 085e43d8..00000000 --- a/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastPropertyTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package uk.ac.ebi.atlas.experimentpage.tooltip; - -import com.google.common.collect.Sets; -import org.junit.Test; -import uk.ac.ebi.atlas.model.experiment.summary.AssayProperty; -import uk.ac.ebi.atlas.model.experiment.summary.ContrastProperty; -import uk.ac.ebi.atlas.model.experiment.summary.ContrastPropertyType; - -import java.util.SortedSet; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.is; - -public class ContrastPropertyTest { - - private static final String PROPERTY_NAME = "propertyName"; - private static final String TEST_VALUE = "testValue"; - private static final String REFERENCE_VALUE = "referenceValue"; - private static final ContrastPropertyType CONTRAST_PROPERTY_TYPE = ContrastPropertyType.FACTOR; - - @Test - public void testGetters() { - ContrastProperty subject = - new ContrastProperty(PROPERTY_NAME, TEST_VALUE, REFERENCE_VALUE, CONTRAST_PROPERTY_TYPE); - assertThat(subject.getContrastPropertyType(), is(CONTRAST_PROPERTY_TYPE)); - assertThat(subject.getPropertyName(), is(PROPERTY_NAME)); - assertThat(subject.getTestValue(), is(TEST_VALUE)); - assertThat(subject.getReferenceValue(), is(REFERENCE_VALUE)); - assertThat( - subject.toString(), - is( - "ContrastProperty{propertyName=" + PROPERTY_NAME + ", referenceValue=" + REFERENCE_VALUE + - ", testValue=" + TEST_VALUE + "}")); - } - - @Test - public void testCompareTo() { - AssayProperty property1 = new ContrastProperty("b", "z", "a", CONTRAST_PROPERTY_TYPE); - AssayProperty property2 = new ContrastProperty("a", "a", "a", CONTRAST_PROPERTY_TYPE); - AssayProperty property3 = new ContrastProperty("c", null, "a", CONTRAST_PROPERTY_TYPE); - AssayProperty property4 = new ContrastProperty("q", "a", "z", CONTRAST_PROPERTY_TYPE); - AssayProperty property5 = new ContrastProperty("d", "a", "a", CONTRAST_PROPERTY_TYPE); - - SortedSet properties = - Sets.newTreeSet(Sets.newHashSet(property1, property2, property3, property4, property5)); - - assertThat(properties, contains(property2, property1, property3, property5, property4)); - } -} diff --git a/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastSummaryBuilderTest.java b/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastSummaryBuilderTest.java deleted file mode 100644 index 89a29bfc..00000000 --- a/app/src/test/java/uk/ac/ebi/atlas/experimentpage/tooltip/ContrastSummaryBuilderTest.java +++ /dev/null @@ -1,83 +0,0 @@ -package uk.ac.ebi.atlas.experimentpage.tooltip; - -import org.junit.Test; -import uk.ac.ebi.atlas.model.experiment.sample.AssayGroup; -import uk.ac.ebi.atlas.model.experiment.ExperimentDesign; -import uk.ac.ebi.atlas.model.experiment.sample.Contrast; -import uk.ac.ebi.atlas.model.experiment.summary.ContrastProperty; -import uk.ac.ebi.atlas.model.experiment.summary.ContrastPropertyType; -import uk.ac.ebi.atlas.model.experiment.summary.ContrastSummary; -import uk.ac.ebi.atlas.model.experiment.summary.ContrastSummaryBuilder; -import uk.ac.ebi.atlas.testutils.AssayGroupFactory; - -import java.util.Iterator; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -public class ContrastSummaryBuilderTest { - private static final String REF_ASSAY = "REF_ASSAY"; - private static final String FACTOR_HEADER = "factor header"; - private static final String FACTOR_VALUE = "factor value"; - - private static final String TEST_ASSAY = "TEST_ASSAY"; - private static final String FACTOR_HEADER2 = "factor header 2"; - private static final String FACTOR_VALUE2 = "factor value 2"; - - private static final String SAMPLE_HEADER = "sample header"; - private static final String SAMPLE_VALUE1 = "sample value 1"; - private static final String SAMPLE_VALUE2 = "sample value 2"; - private static final String EXPERIMENT_DESCRIPTION = "experiment description"; - private static final String CONTRAST_ID = "contrast id"; - private static final String ARRAY_DESIGN_ACCESSION = "array design accession"; - private static final AssayGroup REF_ASSAY_GROUP = AssayGroupFactory.create("ref_id", REF_ASSAY); - private static final AssayGroup TEST_ASSAY_GROUP = AssayGroupFactory.create("test_id", TEST_ASSAY); - private static final String CONTRAST_NAME = "contrast description"; - private static final Contrast CONTRAST = - new Contrast(CONTRAST_ID, CONTRAST_NAME, REF_ASSAY_GROUP, TEST_ASSAY_GROUP, ARRAY_DESIGN_ACCESSION); - private static final String FACTOR_VALUE3 = "factor value 3"; - private static final String FACTOR_VALUE4 = "factor value 4"; - - @Test - public void build() { - ExperimentDesign experimentDesign = new ExperimentDesign(); - experimentDesign.putFactor(REF_ASSAY, FACTOR_HEADER, FACTOR_VALUE); - experimentDesign.putFactor(REF_ASSAY, FACTOR_HEADER2, FACTOR_VALUE2); - - experimentDesign.putFactor(TEST_ASSAY, FACTOR_HEADER, FACTOR_VALUE3); - experimentDesign.putFactor(TEST_ASSAY, FACTOR_HEADER2, FACTOR_VALUE4); - - experimentDesign.putSampleCharacteristic(REF_ASSAY, SAMPLE_HEADER, SAMPLE_VALUE1); - experimentDesign.putSampleCharacteristic(TEST_ASSAY, SAMPLE_HEADER, SAMPLE_VALUE2); - - ContrastSummaryBuilder subject = new ContrastSummaryBuilder(). - withExperimentDescription(EXPERIMENT_DESCRIPTION).forContrast(CONTRAST). - withExperimentDesign(experimentDesign); - - ContrastSummary contrastSummary = subject.build(); - - Iterator contrastSummaryIterator = contrastSummary.iterator(); - - ContrastProperty contrastProperty1 = contrastSummaryIterator.next(); - ContrastProperty contrastProperty2 = contrastSummaryIterator.next(); - ContrastProperty contrastProperty3 = contrastSummaryIterator.next(); - ContrastProperty contrastProperty4 = contrastSummaryIterator.next(); - assertThat(contrastSummaryIterator.hasNext(), is(false)); - - assertThat( - contrastProperty1, - is(new ContrastProperty(FACTOR_HEADER, FACTOR_VALUE3, FACTOR_VALUE, ContrastPropertyType.FACTOR))); - - assertThat( - contrastProperty2, - is(new ContrastProperty(FACTOR_HEADER2, FACTOR_VALUE4, FACTOR_VALUE2, ContrastPropertyType.FACTOR))); - - assertThat( - contrastProperty3, - is(new ContrastProperty("array design", "", "", ContrastPropertyType.SAMPLE))); - - assertThat( - contrastProperty4, - is(new ContrastProperty(SAMPLE_HEADER, SAMPLE_VALUE2, SAMPLE_VALUE1, ContrastPropertyType.SAMPLE))); - } -} diff --git a/app/src/test/resources/fixtures/experiment-fixture.sql b/app/src/test/resources/fixtures/experiment-fixture.sql index 64817539..f41bfb08 100644 --- a/app/src/test/resources/fixtures/experiment-fixture.sql +++ b/app/src/test/resources/fixtures/experiment-fixture.sql @@ -17,9 +17,9 @@ INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, do INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-451', 'RNASEQ_MRNA_BASELINE', 'dc592ad1-d3d5-4284-bf98-6cbfe4472dfe', '2021-01-22 21:08:47.237945', '22253936', '10.1371/journal.pntd.0001455', false, 'Schistosoma mansoni', '2021-01-22 21:08:47.237945'); INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-4559', 'RNASEQ_MRNA_DIFFERENTIAL', '4d48908f-ef54-44b3-9405-35cdf6acd08e', '2021-01-22 21:08:47.259809', null, null, false, 'Zea mays subsp. mays', '2021-01-22 21:08:47.259809'); INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5128', 'MICROARRAY_1COLOUR_MRNA_DIFFERENTIAL', '5550f8a9-2055-419b-864c-ab2c9d91d9d2', '2021-01-22 21:08:47.284917', null, null, false, 'Saccharomyces cerevisiae', '2021-01-22 21:08:47.284917'); -INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5200', 'RNASEQ_MRNA_BASELINE', '2e13c434-4da1-40fb-b9e3-af92fed0c959', '2021-01-22 21:08:48.092384', null, '10.1038/s41586-020-1969-6', false, 'Homo sapiens', '2021-01-22 21:08:48.092384'); +-- INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5200', 'RNASEQ_MRNA_BASELINE', '2e13c434-4da1-40fb-b9e3-af92fed0c959', '2021-01-22 21:08:48.092384', null, '10.1038/s41586-020-1969-6', false, 'Homo sapiens', '2021-01-22 21:08:48.092384'); INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5422', 'RNASEQ_MRNA_DIFFERENTIAL', '2ec933ae-3344-4601-9279-dc58633cb940', '2021-01-22 21:08:51.054533', null, null, false, 'Zea mays', '2021-01-22 21:08:51.054533'); -INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5423', 'RNASEQ_MRNA_BASELINE', '3129b93d-cf5e-49b3-aebb-e7da1af3fe44', '2021-01-22 21:09:02.204925', null, null, false, 'Homo sapiens', '2021-01-22 21:09:02.204925'); +-- INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5423', 'RNASEQ_MRNA_BASELINE', '3129b93d-cf5e-49b3-aebb-e7da1af3fe44', '2021-01-22 21:09:02.204925', null, null, false, 'Homo sapiens', '2021-01-22 21:09:02.204925'); INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5633', 'RNASEQ_MRNA_DIFFERENTIAL', 'f4beab2e-9657-44c0-b29d-51b0a7bcf6e7', '2021-01-22 21:09:02.258675', null, null, false, 'Mus musculus domesticus', '2021-01-22 21:09:02.258675'); INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-MTAB-5941', 'RNASEQ_MRNA_DIFFERENTIAL', 'b94045cd-6ebf-4eba-8880-0c7bf60ac5a3', '2021-01-22 21:09:02.279798', null, null, false, 'Oryza sativa Japonica Group', '2021-01-22 21:09:02.279798'); INSERT INTO experiment (accession, type, access_key, last_update, pubmed_ids, dois, private, species, load_date) VALUES ('E-TABM-713', 'MICROARRAY_1COLOUR_MICRORNA_DIFFERENTIAL', '577cb0f0-77da-41a6-8a40-e2d87d26462f', '2021-01-22 21:09:02.324879', '19823581', '10.1371/journal.pone.0007405', false, 'Homo sapiens', '2021-01-22 21:09:02.324879'); diff --git a/atlas-web-core b/atlas-web-core index b6e5ad6d..b0cc17fb 160000 --- a/atlas-web-core +++ b/atlas-web-core @@ -1 +1 @@ -Subproject commit b6e5ad6d3f11afea1239d311607b4f82af746f67 +Subproject commit b0cc17fbe1d765568826414b52b785eaaf482e01 diff --git a/buildSrc/src/main/groovy/atlas-web-app.java-conventions.gradle b/buildSrc/src/main/groovy/atlas-web-app.java-conventions.gradle index 7ff37170..55330537 100644 --- a/buildSrc/src/main/groovy/atlas-web-app.java-conventions.gradle +++ b/buildSrc/src/main/groovy/atlas-web-app.java-conventions.gradle @@ -64,8 +64,8 @@ dependencies { testImplementation 'org.springframework:spring-test:5.1.5.RELEASE' testImplementation 'com.h2database:h2:1.4.199' testImplementation 'org.junit.jupiter:junit-jupiter-params:5.4.1' - testImplementation 'org.mockito:mockito-core:2.25.1' - testImplementation 'org.mockito:mockito-junit-jupiter:2.25.1' + testImplementation 'org.mockito:mockito-core:5.2.0' + testImplementation 'org.mockito:mockito-junit-jupiter:5.2.0' testImplementation 'org.assertj:assertj-core:3.11.1' testImplementation 'org.assertj:assertj-guava:3.2.0' testImplementation 'org.hamcrest:java-hamcrest:2.0.0.0' @@ -121,19 +121,19 @@ test { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' } + if (project.hasProperty('excludeTests')) { + exclude project.property('excludeTests') + } + if (project.hasProperty('testResultsPath')) { - reporting.baseDir = "${buildDir}/reports/" + project.property('testResultsPath') - testResultsDirName = "${buildDir}/test-results/" + project.property('testResultsPath') + testReportDirName = project.property('testResultsPath') + testResultsDirName = project.property('testResultsPath') jacoco { destinationFile = file("${buildDir}/jacoco/${project.property('testResultsPath')}.exec") } } - if (project.hasProperty('excludeTests')) { - exclude project.property('excludeTests') - } - testLogging { outputs.upToDateWhen {false} showStandardStreams = true @@ -148,7 +148,7 @@ jacocoTestReport { executionData.setFrom fileTree(dir: "${buildDir}/jacoco", include: '*.exec' ) reports { - html.destination file("${buildDir}/jacoco/html") + html.outputLocation = layout.buildDirectory.dir("reports/jacoco") } } diff --git a/docker/dev.env b/docker/dev.env index c7e69d88..261bd0f8 100644 --- a/docker/dev.env +++ b/docker/dev.env @@ -3,10 +3,13 @@ PROJECT_NAME=gxa GRADLE_WRAPPER_DISTS_VOL_NAME=gradle-wrapper-dists GRADLE_RO_DEP_CACHE_VOL_NAME=gradle-ro-dep-cache +WEBAPP_PROPERTIES_VOL_NAME=webapp-properties +ATLAS_DATA_GXA_VOL_NAME=gxa-data +ATLAS_DATA_GXA_EXPDESIGN_VOL_NAME=atlas-data-expdesign ATLAS_DATA_BIOENTITY_PROPERTIES_VOL_NAME=atlas-data-bioentity-properties ATLAS_DATA_EXP_VOL_NAME=atlas-data-exp - ATLAS_DATA_EXPDESIGN_VOL_NAME=atlas-data-expdesign + POSTGRES_HOST=gxa-postgres POSTGRES_DB=gxpgxadev POSTGRES_USER=atlasdev diff --git a/docker/prepare-dev-environment/solr/docker-compose.yml b/docker/prepare-dev-environment/solr/docker-compose.yml index a96a98af..22a22d79 100644 --- a/docker/prepare-dev-environment/solr/docker-compose.yml +++ b/docker/prepare-dev-environment/solr/docker-compose.yml @@ -63,20 +63,17 @@ services: ./gradlew -PdataFilesLocation=/root \ -PexperimentFilesLocation=/atlas-data/exp \ -PexperimentDesignLocation=/atlas-data/expdesign \ - -PzkHosts=$${ZK_HOSTS} \ - -PsolrHosts="" \ -PjdbcUrl=jdbc:postgresql://${POSTGRES_HOST}:5432/${POSTGRES_DB} \ -PjdbcUsername=${POSTGRES_USER} \ -PjdbcPassword=${POSTGRES_PASSWORD} \ -PzkHosts=$${ZK_HOSTS} \ -PsolrHosts=$${SOLR_HOSTS} \ - :cli:bootRun --args="bulk-analytics-json --output=/root/experiments-jsonl -e $(echo $${EXP_IDS} | sed -e "s/ /,/g")" + :cli:bootRun --args="bulk-analytics-json --output=/root/experiments-jsonl -e $(echo ${EXP_IDS} | sed -e "s/ /,/g")" cd /root/solr-bulk/bin ./create-bulk-analytics-collection.sh ./create-bulk-analytics-schema.sh - cd /root/index-bioentities/bin export SOLR_COLLECTION=$${SOLR_COLLECTION_BULK_ANALYTICS} export SCHEMA_VERSION=$${SOLR_COLLECTION_BULK_ANALYTICS_SCHEMA_VERSION} export SOLR_PROCESSORS=dedupe diff --git a/jenkins-k8s-pod.yaml b/jenkins-k8s-pod.yaml new file mode 100644 index 00000000..9ba40b1b --- /dev/null +++ b/jenkins-k8s-pod.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: Pod +metadata: + namespace: jenkins-gene-expression +spec: + serviceAccountName: admin-jenkins-gene-expression + # If we set workspaceVolume dynamicPVC() or workspaceVolume persistentVolumeClaimWorkspaceVolume() we need to add a + # fsGroup 1000 to make the /home/jenkins/agent directory writable. Otherwise, we get the following exception in the + # jnlp container: + # Exception in thread "main" java.io.IOException: The specified working directory should be fully accessible to the remoting executable (RWX): /home/jenkins/agent + # The securityContext in the pod is inherited, as it becomes the default pod template + securityContext: + fsGroup: 1000 + containers: + - name: "kubectl" + image: bitnami/kubectl + command: [ "/bin/sh" ] + args: [ "-c", "kubectl scale --replicas=4 solrcloud gxa && sleep 2h" ] + resources: + requests: + memory: "256Mi" + limits: + memory: "256Mi" + - name: jnlp + resources: + requests: + memory: "512Mi" + limits: + memory: "512Mi" + - name: openjdk + image: openjdk:11 + resources: + requests: + memory: "3.5Gi" + ephemeral-storage: "6Gi" + limits: + memory: "5Gi" + env: + - name: GRADLE_OPTS + value: "-Dorg.gradle.daemon=false" + - name: GRADLE_RO_DEP_CACHE + value: "/root/gradle-ro-dep-cache" + command: [ "/bin/sh" ] + args: [ "-c", "mkdir -p /root/expdesign-rw && cp /test-data/expdesign/* /root/expdesign-rw ; sleep 2h" ] + volumeMounts: + - mountPath: /test-data/gxa + name: gxa-data + readOnly: true + - mountPath: /test-data/expdesign + name: gxa-expdesign + readOnly: true + - mountPath: /test-data/ontology + name: gxa-data-ontology + readOnly: true + - mountPath: /test-data/bioentity_properties + name: bioentity-properties + readOnly: true + - mountPath: /root/gradle-ro-dep-cache + name: gradle-ro-dep-cache + readOnly: true + - name: postgres + image: postgres:11-alpine + resources: + requests: + memory: "100Mi" + limits: + memory: "200Mi" + env: + - name: POSTGRES_DB + value: "postgres" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + value: "postgres" + volumes: + - name: gxa-data + persistentVolumeClaim: + claimName: gxa-data-rox + readOnly: true + - name: gxa-expdesign + persistentVolumeClaim: + claimName: gxa-expdesign-rox + readOnly: true + - name: gxa-data-ontology + persistentVolumeClaim: + claimName: gxa-data-ontology-rox + readOnly: true + - name: bioentity-properties + persistentVolumeClaim: + claimName: bioentity-properties-rox + readOnly: true + - name: gradle-ro-dep-cache + persistentVolumeClaim: + claimName: gradle-7.0-ro-dep-cache-rox + readOnly: true