Merge pull request #89 from fscheel/AddQualityMetricCalculation

Add quality metric calculation

curation-swagger-spec.yaml

@@ -163,6 +163,8 @@ paths:
     post:
       description: |
         Storing of a new Haplotype Frequency set.
+        The service will silently ignore any quality metrics sent for submission and calculate the supported
+        ones itself.
       parameters:
         - name: HFCurationRequest
           in: body

src/main/java/org/nmdp/hfcus/configuration/SwaggerDocumentationConfig.java

@@ -3,6 +3,7 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import org.springframework.transaction.annotation.EnableTransactionManagement;
 import springfox.documentation.builders.ApiInfoBuilder;
 import springfox.documentation.builders.RequestHandlerSelectors;
 import springfox.documentation.service.ApiInfo;
@@ -12,6 +13,7 @@
 
 
 @Configuration
+@EnableTransactionManagement
 public class SwaggerDocumentationConfig {
 
     private ApiInfo apiInfo() {

src/main/java/org/nmdp/hfcus/controller/HFCurationApiController.java

@@ -18,6 +18,7 @@
 import org.nmdp.hfcus.dao.RepositoryContainer;
 import org.nmdp.hfcus.dao.ScopeListRepository;
 import org.nmdp.hfcus.model.*;
+import org.nmdp.hfcus.quality.QualityService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
@@ -32,6 +33,7 @@
 public class HFCurationApiController implements HfcApi {
 
     private final RepositoryContainer repositoryContainer;
+    private final QualityService qualityService;
 
     @Autowired
     public HFCurationApiController(
@@ -42,7 +44,8 @@ public HFCurationApiController(
             LabelSetRepository labelSetRepository,
             HaplotypeFrequencySetRepository haplotypeFrequencySetRepository,
             AccessRepository accessRepository,
-            ScopeListRepository scopeListRepository
+            ScopeListRepository scopeListRepository,
+            QualityService qualityService
     ) {
         this.repositoryContainer = new RepositoryContainer();
         repositoryContainer.setCurationRepository(curationRepository);
@@ -53,6 +56,12 @@ public HFCurationApiController(
         repositoryContainer.setHaplotypeFrequencySetRepository(haplotypeFrequencySetRepository);
         repositoryContainer.setAccessRepository(accessRepository);
         repositoryContainer.setScopeListRepository(scopeListRepository);
+        this.qualityService = qualityService;
+        try {
+            qualityService.run();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
     }
 
     private <ModelType, ResponseType> ResponseEntity<ResponseType> RetrieveSubdataFromDatabase(
@@ -93,6 +102,12 @@ public ResponseEntity<HFCurationResponse> hfcPost(@Valid @RequestBody HFCuration
         if (hfCurationRequest != null) {
             HFCuration curation = new HFCuration(repositoryContainer, hfCurationRequest);
             curation = repositoryContainer.getCurationRepository().save(curation);
+            try {
+                qualityService.addToQueue(curation);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            }
             return ResponseEntity.ok(curation.toSwaggerObject());
         }
 

src/main/java/org/nmdp/hfcus/dao/HFCurationRepository.java

@@ -1,8 +1,14 @@
 package org.nmdp.hfcus.dao;
 
 import org.nmdp.hfcus.model.HFCuration;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.CrudRepository;
 
+import java.util.stream.Stream;
+
 public interface HFCurationRepository  extends CrudRepository<HFCuration, Long> {
     Iterable<HFCuration> findBypopulationDataId(Long populationId);
+
+    @Query("select h from HFCuration h")
+    Stream<HFCuration> findAllStreamable();
 }

src/main/java/org/nmdp/hfcus/model/HaplotypeFrequencySet.java

@@ -16,38 +16,33 @@
 @Data
 @NoArgsConstructor
 public class HaplotypeFrequencySet implements ICurationDataModel<io.swagger.model.HaplotypeFrequencyData> {
+    public HaplotypeFrequencySet() {
+        //intentionally left empty
+    }
 
-    HaplotypeFrequencySet(HaplotypeFrequencyData swaggerObject){
+    public HaplotypeFrequencySet(HaplotypeFrequencyData swaggerObject) {
         license = swaggerObject.getLicense().getTypeOfLicense();
-        if (license == null){
+        if (license == null) {
             throw new RequiredFieldInvalidException("requires a license");
         }
         if (swaggerObject.getResolutionData() != null) {
             resolutionList = new ArrayList<>();
-            for (ResolutionInfo resolutionInfo :swaggerObject.getResolutionData()) {
+            for (ResolutionInfo resolutionInfo : swaggerObject.getResolutionData()) {
                 resolutionList.add(new Resolution(resolutionInfo));
             }
         }
         List<io.swagger.model.HaplotypeFrequency> swaggerFrequencyList = swaggerObject.getHaplotypeFrequencyList();
         if (swaggerFrequencyList != null) {
             frequencyList = new ArrayList<>();
-            for (io.swagger.model.HaplotypeFrequency frequency: swaggerFrequencyList) {
+            for (io.swagger.model.HaplotypeFrequency frequency : swaggerFrequencyList) {
                 frequencyList.add(new HaplotypeFrequency(frequency));
             }
-            if (frequencyList.size() == 0){
+            if (frequencyList.size() == 0) {
                 throw new RequiredFieldInvalidException("frequency list must not be empty");
             }
-        }
-        else
-        {
+        } else {
             throw new RequiredFieldInvalidException("requires a frequency list");
         }
-        if (swaggerObject.getQualityList() != null){
-            qualityList = new ArrayList<>();
-            for (io.swagger.model.Quality quality: swaggerObject.getQualityList()) {
-                qualityList.add(new Quality(quality));
-            }
-        }
     }
 
     @Id
@@ -61,28 +56,86 @@ public class HaplotypeFrequencySet implements ICurationDataModel<io.swagger.mode
     private List<HaplotypeFrequency> frequencyList;
     @OneToMany(cascade = CascadeType.ALL)
     private List<Quality> qualityList;
+    private int numberOfCalculatedQualities = 0;
+
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public List<Resolution> getResolutionList() {
+        return resolutionList;
+    }
+
+    public void setResolutionList(List<Resolution> resolutionList) {
+        this.resolutionList = resolutionList;
+    }
+
+    public License.TypeOfLicenseEnum getLicense() {
+        return license;
+    }
+
+    public void setLicense(License.TypeOfLicenseEnum license) {
+        this.license = license;
+    }
+
+    public List<Resolution> getResolution() {
+        return resolutionList;
+    }
+
+    public void setResolution(List<Resolution> resolutionList) {
+        this.resolutionList = resolutionList;
+    }
+
+    public List<HaplotypeFrequency> getFrequencyList() {
+        return frequencyList;
+    }
+
+    public void setFrequencyList(List<HaplotypeFrequency> frequencyList) {
+        this.frequencyList = frequencyList;
+    }
+
+    public List<Quality> getQualityList() {
+        return qualityList;
+    }
+
+    public void setQualityList(List<Quality> qualityList) {
+        this.qualityList = qualityList;
+    }
+
+    public int getNumberOfCalculatedQualities() {
+        return numberOfCalculatedQualities;
+    }
+
+    public void setNumberOfCalculatedQualities(int numberOfCalculatedQualities) {
+        this.numberOfCalculatedQualities = numberOfCalculatedQualities;
+    }
 
     @Override
-    public HaplotypeFrequencyData toSwaggerObject(){
+    public HaplotypeFrequencyData toSwaggerObject() {
         HaplotypeFrequencyData data = new HaplotypeFrequencyData();
         License license = new License();
         license.setTypeOfLicense(this.license);
         data.setLicense(license);
-        if (resolutionList != null){
+        if (resolutionList != null) {
             ResolutionData resolutions = new ResolutionData();
-            for (Resolution resolution: resolutionList) {
+            for (Resolution resolution : resolutionList) {
                 resolutions.add(resolution.toSwaggerObject());
             }
             data.setResolutionData(resolutions);
         }
         List<io.swagger.model.HaplotypeFrequency> frequencies = new ArrayList<>();
-        for (HaplotypeFrequency frequency: frequencyList){
+        for (HaplotypeFrequency frequency : frequencyList) {
             frequencies.add(frequency.toSwaggerObject());
         }
         data.setHaplotypeFrequencyList(frequencies);
-        if (qualityList != null){
+        if (qualityList != null) {
             List<io.swagger.model.Quality> qualities = new ArrayList<>();
-            for (Quality quality: qualityList){
+            for (Quality quality : qualityList) {
                 qualities.add(quality.toSwaggerObject());
             }
             data.setQualityList(qualities);

src/main/java/org/nmdp/hfcus/quality/Div50Calculator.java

@@ -0,0 +1,49 @@
+package org.nmdp.hfcus.quality;
+
+import org.nmdp.hfcus.model.HFCuration;
+import org.nmdp.hfcus.model.HaplotypeFrequency;
+import org.nmdp.hfcus.model.Quality;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+@Component
+public class Div50Calculator implements IQualityMetricCalculator {
+    @Override
+    public boolean calculationNeeded(HFCuration frequency) {
+        List<Quality> qualityList = frequency.getHaplotypeFrequencyData().getQualityList();
+        if (qualityList == null){
+            return true;
+        }
+        return qualityList
+                .stream()
+                .noneMatch(quality -> quality.getTypeOfQuality() == io.swagger.model.Quality.TypeOfQualityEnum.DIV_50);
+    }
+
+    @Override
+    public void calculateMetric(HFCuration curation) {
+        List<HaplotypeFrequency> frequencyList = curation.getHaplotypeFrequencyData().getFrequencyList();
+        AtomicInteger result = new AtomicInteger();
+        AtomicReference<Double> frequencies = new AtomicReference<>((double) 0);
+        frequencyList
+                .stream()
+                .sorted(Comparator.comparing(HaplotypeFrequency::getFrequency).reversed())
+                .forEachOrdered(frequency -> {
+                    if (frequencies.get() < 0.5){
+                        frequencies.updateAndGet(currentValue -> currentValue + frequency.getFrequency());
+                        result.getAndIncrement();
+                    }
+                });
+        List<Quality> qualities = curation.getHaplotypeFrequencyData().getQualityList();
+        if (qualities == null){
+            qualities = new LinkedList<Quality>();
+            curation.getHaplotypeFrequencyData().setQualityList(qualities);
+        }
+        Quality newQuality = new Quality();
+        newQuality.setValue(result.doubleValue());
+        newQuality.setTypeOfQuality(io.swagger.model.Quality.TypeOfQualityEnum.DIV_50);
+        qualities.add(newQuality);
+    }
+}

src/main/java/org/nmdp/hfcus/quality/IQualityMetricCalculator.java

@@ -0,0 +1,10 @@
+package org.nmdp.hfcus.quality;
+
+import org.nmdp.hfcus.model.HFCuration;
+import org.springframework.stereotype.Component;
+
+@Component
+public interface IQualityMetricCalculator {
+    boolean calculationNeeded(HFCuration frequency);
+    void calculateMetric(HFCuration frequency);
+}

src/main/java/org/nmdp/hfcus/quality/QualityService.java

@@ -0,0 +1,68 @@
+package org.nmdp.hfcus.quality;
+
+import org.nmdp.hfcus.dao.HFCurationRepository;
+import org.nmdp.hfcus.model.HFCuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.stream.Stream;
+
+@Service
+@EnableAsync
+public class QualityService {
+
+    private static final LinkedBlockingQueue<Long> queue = new LinkedBlockingQueue<>();
+    private static Logger logger = LoggerFactory.getLogger(QualityService.class);
+    private List<IQualityMetricCalculator> metricCalculators;
+    private HFCurationRepository curationRepository;
+    private QualityWorker qualityWorker;
+
+    @Autowired
+    public QualityService(
+            List<IQualityMetricCalculator> metricCalculators,
+            HFCurationRepository curationRepository,
+            QualityWorker qualityWorker
+    ) {
+        this.metricCalculators = metricCalculators;
+        this.curationRepository = curationRepository;
+        this.qualityWorker = qualityWorker;
+    }
+
+    @Async
+    public void run() throws InterruptedException {
+        try (Stream<HFCuration> stream = curationRepository.findAllStreamable()) {
+            stream
+                    .filter(
+                            hfCuration ->
+                                    hfCuration.getHaplotypeFrequencyData().getNumberOfCalculatedQualities() < metricCalculators.size()
+                    )
+                    .forEach(hfCuration -> {
+                        try {
+                            queue.put(hfCuration.getId());
+                        } catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
+                        }
+                    });
+        }
+        if (Thread.currentThread().isInterrupted()) {
+            throw new InterruptedException();
+        }
+        while (true) {
+            Long nextUp = queue.take();
+            for (IQualityMetricCalculator calculator : metricCalculators) {
+                qualityWorker.handleSingleMetric(nextUp, calculator);
+            }
+            Thread.sleep(5000);
+        }
+    }
+
+    public void addToQueue(HFCuration value) throws InterruptedException {
+        queue.put(value.getId());
+    }
+}

src/main/java/org/nmdp/hfcus/quality/QualityWorker.java

@@ -0,0 +1,35 @@
+package org.nmdp.hfcus.quality;
+
+import org.nmdp.hfcus.dao.HFCurationRepository;
+import org.nmdp.hfcus.model.HFCuration;
+import org.nmdp.hfcus.model.HaplotypeFrequencySet;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.List;
+
+@Component
+public class QualityWorker {
+
+
+    private HFCurationRepository curationRepository;
+
+    @Autowired
+    public QualityWorker(
+            HFCurationRepository curationRepository
+    ) {
+        this.curationRepository = curationRepository;
+    }
+
+    @Transactional
+    public void handleSingleMetric(Long nextUp, IQualityMetricCalculator calculator) {
+        HFCuration curation = curationRepository.findOne(nextUp);
+        if (calculator.calculationNeeded(curation)) {
+            calculator.calculateMetric(curation);
+            HaplotypeFrequencySet freqData = curation.getHaplotypeFrequencyData();
+            freqData.setNumberOfCalculatedQualities(freqData.getNumberOfCalculatedQualities() + 1);
+            curationRepository.save(curation);
+        }
+    }
+}