[ALS-7809] DRS URI: Data Dictionary API (#56)

* Add new endpoint that excepts a list of concepts
Returns a list of concepts with their metadata included.

dictonaryReqeust.http

@@ -14,3 +14,10 @@ Content-Type: application/json
 
 {"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"throat sore acute #8","includedTags":[],
   "excludedTags":[],"returnTags":"true","offset":0,"limit":100000},"resourceUUID":null}
+
+###
+
+POST http://localhost:80/concepts/detail
+Content-Type: application/json
+
+["\\phs000993\\pht005015\\phv00253191\\BODY_SITE\\", "\\phs002913\\W2Q_COV_REINFEC_2_OTH\\"]

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java

@@ -58,6 +58,11 @@ public ResponseEntity<Concept> conceptDetail(@PathVariable(name = "dataset") Str
         return conceptService.conceptDetail(dataset, conceptPath).map(ResponseEntity::ok).orElse(ResponseEntity.notFound().build());
     }
 
+    @PostMapping(path = "/concepts/detail")
+    public ResponseEntity<List<Concept>> conceptsDetail(@RequestBody() List<String> conceptPaths) {
+        return ResponseEntity.ok(conceptService.conceptsWithDetail(conceptPaths));
+    }
+
     @PostMapping(path = "/concepts/tree/{dataset}")
     public ResponseEntity<Concept> conceptTree(
         @PathVariable(name = "dataset") String dataset, @RequestBody() String conceptPath,

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java

@@ -3,7 +3,6 @@
 import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
 import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
 import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair;
-import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper;
 import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -27,19 +26,21 @@ public class ConceptRepository {
     private final ConceptFilterQueryGenerator filterGen;
     private final ConceptMetaExtractor conceptMetaExtractor;
     private final ConceptResultSetExtractor conceptResultSetExtractor;
+    private final ConceptRowWithMetaMapper conceptRowWithMetaMapper;
     private final List<String> disallowedMetaFields;
 
     @Autowired
     public ConceptRepository(
         NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen,
         ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor,
-        @Value("${filtering.unfilterable_concepts}") List<String> disallowedMetaFields
+        ConceptRowWithMetaMapper conceptRowWithMetaMapper, @Value("${filtering.unfilterable_concepts}") List<String> disallowedMetaFields
     ) {
         this.template = template;
         this.mapper = mapper;
         this.filterGen = filterGen;
         this.conceptMetaExtractor = conceptMetaExtractor;
         this.conceptResultSetExtractor = conceptResultSetExtractor;
+        this.conceptRowWithMetaMapper = conceptRowWithMetaMapper;
         this.disallowedMetaFields = disallowedMetaFields;
     }
 
@@ -230,4 +231,52 @@ WITH RECURSIVE nodes AS (
     }
 
 
+    public List<Concept> getConceptsByPathWithMetadata(List<String> conceptPaths) {
+        String sql = ALLOW_FILTERING_Q + ", "
+            + """
+                filtered_concepts AS (
+                     SELECT
+                        concept_node.*
+                     FROM
+                         concept_node
+                     WHERE
+                         concept_path IN (:conceptPaths)
+                 ),
+                 aggregated_meta AS (
+                     SELECT
+                         concept_node_meta.concept_node_id,
+                         json_agg(json_build_object('key', concept_node_meta.key, 'value', concept_node_meta.value)) AS metadata
+                     FROM
+                         concept_node_meta
+                     WHERE
+                         concept_node_meta.concept_node_id IN (
+                             SELECT concept_node_id FROM filtered_concepts
+                         )
+                     GROUP BY
+                         concept_node_meta.concept_node_id
+                 )
+                 SELECT
+                     concept_node.*,
+                     ds.REF as dataset,
+                     ds.abbreviation AS studyAcronym,
+                     continuous_min.VALUE as min, continuous_max.VALUE as max,
+                     categorical_values.VALUE as values,
+                     allow_filtering.allowFiltering AS allowFiltering,
+                     meta_description.VALUE AS description,
+                     aggregated_meta.metadata AS metadata
+                 FROM
+                     filtered_concepts as concept_node
+                     LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id
+                     LEFT JOIN concept_node_meta AS meta_description ON concept_node.concept_node_id = meta_description.concept_node_id AND meta_description.KEY = 'description'
+                     LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
+                     LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
+                     LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
+                     LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id
+                     LEFT JOIN aggregated_meta ON concept_node.concept_node_id = aggregated_meta.concept_node_id
+                """;
+
+        MapSqlParameterSource params =
+            new MapSqlParameterSource().addValue("conceptPaths", conceptPaths).addValue("disallowed_meta_keys", disallowedMetaFields);
+        return template.query(sql, params, conceptRowWithMetaMapper);
+    }
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java

@@ -3,42 +3,56 @@
 import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept;
 import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.List;
+import java.util.Map;
 
 @Component
 public class ConceptResultSetUtil {
 
-    private static final Logger log = LoggerFactory.getLogger(ConceptResultSetUtil.class);
     private final JsonBlobParser jsonBlobParser;
 
     @Autowired
     public ConceptResultSetUtil(JsonBlobParser jsonBlobParser) {
         this.jsonBlobParser = jsonBlobParser;
     }
 
-    public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
+    public CategoricalConcept mapCategoricalWithMetadata(ResultSet rs) throws SQLException {
+        Map<String, String> metadata = jsonBlobParser.parseMetaData(rs.getString("metadata"));
+        return new CategoricalConcept(getCategoricalConcept(rs), metadata);
+    }
+
+    private CategoricalConcept getCategoricalConcept(ResultSet rs) throws SQLException {
         return new CategoricalConcept(
             rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"),
             rs.getString("description"), rs.getString("values") == null ? List.of() : jsonBlobParser.parseValues(rs.getString("values")),
             rs.getBoolean("allowFiltering"), rs.getString("studyAcronym"), null, null
         );
     }
 
-    public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
+    public ContinuousConcept mapContinuousWithMetadata(ResultSet rs) throws SQLException {
+        Map<String, String> metadata = jsonBlobParser.parseMetaData(rs.getString("metadata"));
+        return new ContinuousConcept(getContinuousConcept(rs), metadata);
+    }
+
+    private ContinuousConcept getContinuousConcept(ResultSet rs) throws SQLException {
         return new ContinuousConcept(
             rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"),
             rs.getString("description"), rs.getBoolean("allowFiltering"), jsonBlobParser.parseMin(rs.getString("values")),
             jsonBlobParser.parseMax(rs.getString("values")), rs.getString("studyAcronym"), null
         );
     }
 
+    public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException {
+        return getContinuousConcept(rs);
+    }
 
+    public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException {
+        return getCategoricalConcept(rs);
+    }
 
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowWithMetaMapper.java

@@ -0,0 +1,30 @@
+package edu.harvard.dbmi.avillach.dictionary.concept;
+
+import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.stereotype.Component;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+@Component
+public class ConceptRowWithMetaMapper implements RowMapper<Concept> {
+
+    private final ConceptResultSetUtil conceptResultSetUtil;
+
+    @Autowired
+    public ConceptRowWithMetaMapper(ConceptResultSetUtil conceptResultSetUtil) {
+        this.conceptResultSetUtil = conceptResultSetUtil;
+    }
+
+    @Override
+    public Concept mapRow(ResultSet rs, int rowNum) throws SQLException {
+        return switch (ConceptType.toConcept(rs.getString("concept_type"))) {
+            case Categorical -> conceptResultSetUtil.mapCategoricalWithMetadata(rs);
+            case Continuous -> conceptResultSetUtil.mapContinuousWithMetadata(rs);
+        };
+    }
+
+}

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java

@@ -71,4 +71,7 @@ public Optional<Concept> conceptDetailWithoutAncestors(String dataset, String co
         return getConcept(dataset, conceptPath, false);
     }
 
+    public List<Concept> conceptsWithDetail(List<String> conceptPaths) {
+        return this.conceptRepository.getConceptsByPathWithMetadata(conceptPaths);
+    }
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java

@@ -1,21 +1,31 @@
 package edu.harvard.dbmi.avillach.dictionary.util;
 
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 @Component
 public class JsonBlobParser {
 
     private final static Logger log = LoggerFactory.getLogger(JsonBlobParser.class);
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
+    public JsonBlobParser() {}
 
     public List<String> parseValues(String valuesArr) {
         try {
@@ -62,4 +72,25 @@ public Float parseMax(String valuesArr) {
         return parseFromIndex(valuesArr, 1);
     }
 
+    public Map<String, String> parseMetaData(String jsonMetaData) {
+        Map<String, String> metadata;
+
+        try {
+            List<Map<String, String>> maps = objectMapper.readValue(jsonMetaData, new TypeReference<List<Map<String, String>>>() {});
+            // convert the list to a flat map
+            Map<String, String> map = new HashMap<>();
+            for (Map<String, String> entry : maps) {
+                if (map.put(entry.get("key"), entry.get("value")) != null) {
+                    throw new IllegalStateException(
+                        "parseMetaData() Duplicate key found in metadata. Key: " + entry.get("key") + " Value: " + entry.get("value")
+                    );
+                }
+            }
+            metadata = map;
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+
+        return metadata;
+    }
 }

src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java

@@ -148,4 +148,18 @@ void shouldDumpConcepts() {
         Assertions.assertEquals(concepts, actual.getBody().getContent());
         Assertions.assertEquals(HttpStatus.OK, actual.getStatusCode());
     }
+
+    @Test
+    void shouldReturnConceptsWithMeta() {
+        CategoricalConcept fooBar = new CategoricalConcept(
+            "/foo//bar", "bar", "Bar", "my_dataset", "foo!", List.of("a", "b"), true, "", List.of(), Map.of("key", "value")
+        );
+        Concept fooBaz = new ContinuousConcept("/foo//baz", "baz", "Baz", "my_dataset", "foo!", true, 0F, 100F, "", Map.of("key", "value"));
+        List<Concept> concepts = List.of(fooBar, fooBaz);
+        List<String> conceptPaths = List.of("/foo//bar", "/foo//bar");
+        Mockito.when(conceptService.conceptsWithDetail(conceptPaths)).thenReturn(concepts);
+        ResponseEntity<List<Concept>> listResponseEntity = subject.conceptsDetail(conceptPaths);
+        Assertions.assertEquals(HttpStatus.OK, listResponseEntity.getStatusCode());
+        Assertions.assertEquals(concepts, listResponseEntity.getBody());
+    }
 }

src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java

@@ -324,5 +324,31 @@ void shouldGetContConceptWithDecimalNotation() {
         Assertions.assertEquals(6.77f, concept.max());
     }
 
+    @Test
+    void shouldGetConceptsByConceptPath() {
+        List<String> conceptPaths = List.of(
+            "\\phs002385\\TXNUM\\", "\\phs000284\\pht001902\\phv00122507\\age\\", "\\phs000007\\pht000022" + "\\phv00004260\\FM219\\",
+            "\\NHANES\\examination\\physical fitness\\Stage 1 heart rate (per min)", "\\phs000007\\pht000021" + "\\phv00003844\\FL200\\",
+            "\\phs002715\\age\\"
+        );
+        List<Concept> conceptsByPath = subject.getConceptsByPathWithMetadata(conceptPaths);
+        Assertions.assertFalse(conceptsByPath.isEmpty());
+        Assertions.assertEquals(6, conceptsByPath.size());
+    }
+
+    @Test
+    void shouldGetSameConceptMetaAsConceptDetails() {
+        List<String> conceptPaths = List.of("\\phs002385\\TXNUM\\", "\\phs000284\\pht001902\\phv00122507\\age\\");
+        List<Concept> conceptsByPath = subject.getConceptsByPathWithMetadata(conceptPaths);
+        Assertions.assertFalse(conceptsByPath.isEmpty());
+
+        // Verify the meta data is correctly retrieve by comparing against known good query.
+        Concept concept = conceptsByPath.getFirst();
+        Map<String, String> expectedMeta = subject.getConceptMeta(concept.dataset(), concept.conceptPath());
+
+        // compare the maps to each other.
+        Map<String, String> actualMeta = concept.meta();
+        Assertions.assertEquals(actualMeta, expectedMeta);
+    }
 
 }