Add value annotation, add validation to integration test

hms-dbmi · Oct 24, 2024 · 1ae1adc · 1ae1adc
1 parent efff3fd
commit 1ae1adc
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 8 deletions.
diff --git a/...src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/MultiValueQueryProcessor.java b/...src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/MultiValueQueryProcessor.java
@@ -8,6 +8,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.util.*;
@@ -18,16 +19,16 @@
 public class MultiValueQueryProcessor implements HpdsProcessor {
 
     public static final String PATIENT_ID_FIELD_NAME = "patient_id";
-    private final int ID_BATCH_SIZE;
+    private final int idBatchSize;
     private final AbstractProcessor abstractProcessor;
 
     private Logger log = LoggerFactory.getLogger(MultiValueQueryProcessor.class);
 
 
     @Autowired
-    public MultiValueQueryProcessor(AbstractProcessor abstractProcessor) {
+    public MultiValueQueryProcessor(AbstractProcessor abstractProcessor, @Value("${ID_BATCH_SIZE:0}") int idBatchSize) {
         this.abstractProcessor = abstractProcessor;
-        ID_BATCH_SIZE = Integer.parseInt(System.getProperty("ID_BATCH_SIZE", "1000"));
+        this.idBatchSize = idBatchSize;
     }
 
     @Override
@@ -42,7 +43,7 @@ public String[] getHeaderRow(Query query) {
     public void runQuery(Query query, AsyncResult result) {
         Set<Integer> idList = abstractProcessor.getPatientSubsetForQuery(query);
         log.info("Processing " + idList.size() + " rows for result " + result.getId());
-        Lists.partition(new ArrayList<>(idList), ID_BATCH_SIZE).stream()
+        Lists.partition(new ArrayList<>(idList), idBatchSize).stream()
                 .forEach(patientIds -> {
                     Map<String, Map<Integer, List<String>>> pathToPatientToValueMap = buildResult(result, query, new TreeSet<>(patientIds));
                     List<List<List<String>>> fieldValuesPerPatient = patientIds.stream().map(patientId -> {

diff --git a/service/src/main/resources/application-integration-test.properties b/service/src/main/resources/application-integration-test.properties
@@ -1,6 +1,7 @@
 SMALL_JOB_LIMIT = 100
 SMALL_TASK_THREADS = 1
 LARGE_TASK_THREADS = 1
+ID_BATCH_SIZE=1000
 VCF_EXCERPT_ENABLED=true
 
 hpds.genomicProcessor.impl=local

diff --git a/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryServiceTest.java b/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryServiceTest.java
@@ -1,5 +1,7 @@
 package edu.harvard.hms.dbmi.avillach.hpds.service;
 
+import de.siegmar.fastcsv.reader.CsvContainer;
+import de.siegmar.fastcsv.reader.CsvReader;
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query;
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult;
@@ -49,11 +51,17 @@ public void dataframeMulti() throws IOException, InterruptedException {
 
         AsyncResult asyncResult = queryService.runQuery(query);
 
-        Thread.sleep(1000);
+        int retries = 0;
+        while ((AsyncResult.Status.RUNNING.equals(asyncResult.getStatus()) || AsyncResult.Status.PENDING.equals(asyncResult.getStatus())) && retries < 10) {
+            retries++;
+            Thread.sleep(200);
+        }
 
-        System.out.println(asyncResult.getStatus());
-        System.out.println(IOUtils.toString(new FileInputStream(asyncResult.getFile()), StandardCharsets.UTF_8));
-        ;
+        assertEquals(AsyncResult.Status.SUCCESS, asyncResult.getStatus());
+        CsvReader csvReader = new CsvReader();
+        CsvContainer csvContainer = csvReader.read(asyncResult.getFile(), StandardCharsets.UTF_8);
+        // 22 plus header
+        assertEquals(23, csvContainer.getRows().size());
     }
 
 }