From 09f9c245c2c2bc0f74c6e5046d466ed9bc3a39ad Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:15:12 -0500 Subject: [PATCH 1/2] [ALS-7809] DRS URI: Data Dictionary API (#56) * Add new endpoint that excepts a list of concepts Returns a list of concepts with their metadata included. --- dictonaryReqeust.http | 7 +++ .../dictionary/concept/ConceptController.java | 5 ++ .../dictionary/concept/ConceptRepository.java | 53 ++++++++++++++++++- .../concept/ConceptResultSetUtil.java | 24 +++++++-- .../concept/ConceptRowWithMetaMapper.java | 30 +++++++++++ .../dictionary/concept/ConceptService.java | 3 ++ .../dictionary/util/JsonBlobParser.java | 31 +++++++++++ .../concept/ConceptControllerTest.java | 14 +++++ .../concept/ConceptRepositoryTest.java | 26 +++++++++ 9 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowWithMetaMapper.java diff --git a/dictonaryReqeust.http b/dictonaryReqeust.http index ae53188..34a30bb 100644 --- a/dictonaryReqeust.http +++ b/dictonaryReqeust.http @@ -14,3 +14,10 @@ Content-Type: application/json {"@type":"GeneralQueryRequest","resourceCredentials":{},"query":{"searchTerm":"throat sore acute #8","includedTags":[], "excludedTags":[],"returnTags":"true","offset":0,"limit":100000},"resourceUUID":null} + +### + +POST http://localhost:80/concepts/detail +Content-Type: application/json + +["\\phs000993\\pht005015\\phv00253191\\BODY_SITE\\", "\\phs002913\\W2Q_COV_REINFEC_2_OTH\\"] \ No newline at end of file diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java index d2c5c1c..9563a47 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java @@ -58,6 +58,11 @@ public ResponseEntity conceptDetail(@PathVariable(name = "dataset") Str return conceptService.conceptDetail(dataset, conceptPath).map(ResponseEntity::ok).orElse(ResponseEntity.notFound().build()); } + @PostMapping(path = "/concepts/detail") + public ResponseEntity> conceptsDetail(@RequestBody() List conceptPaths) { + return ResponseEntity.ok(conceptService.conceptsWithDetail(conceptPaths)); + } + @PostMapping(path = "/concepts/tree/{dataset}") public ResponseEntity conceptTree( @PathVariable(name = "dataset") String dataset, @RequestBody() String conceptPath, diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java index 7f449bb..a25dc2f 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java @@ -3,7 +3,6 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; import edu.harvard.dbmi.avillach.dictionary.filter.Filter; import edu.harvard.dbmi.avillach.dictionary.filter.QueryParamPair; -import edu.harvard.dbmi.avillach.dictionary.legacysearch.SearchResultRowMapper; import edu.harvard.dbmi.avillach.dictionary.util.MapExtractor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -27,19 +26,21 @@ public class ConceptRepository { private final ConceptFilterQueryGenerator filterGen; private final ConceptMetaExtractor conceptMetaExtractor; private final ConceptResultSetExtractor conceptResultSetExtractor; + private final ConceptRowWithMetaMapper conceptRowWithMetaMapper; private final List disallowedMetaFields; @Autowired public ConceptRepository( NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen, ConceptMetaExtractor conceptMetaExtractor, ConceptResultSetExtractor conceptResultSetExtractor, - @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields + ConceptRowWithMetaMapper conceptRowWithMetaMapper, @Value("${filtering.unfilterable_concepts}") List disallowedMetaFields ) { this.template = template; this.mapper = mapper; this.filterGen = filterGen; this.conceptMetaExtractor = conceptMetaExtractor; this.conceptResultSetExtractor = conceptResultSetExtractor; + this.conceptRowWithMetaMapper = conceptRowWithMetaMapper; this.disallowedMetaFields = disallowedMetaFields; } @@ -230,4 +231,52 @@ WITH RECURSIVE nodes AS ( } + public List getConceptsByPathWithMetadata(List conceptPaths) { + String sql = ALLOW_FILTERING_Q + ", " + + """ + filtered_concepts AS ( + SELECT + concept_node.* + FROM + concept_node + WHERE + concept_path IN (:conceptPaths) + ), + aggregated_meta AS ( + SELECT + concept_node_meta.concept_node_id, + json_agg(json_build_object('key', concept_node_meta.key, 'value', concept_node_meta.value)) AS metadata + FROM + concept_node_meta + WHERE + concept_node_meta.concept_node_id IN ( + SELECT concept_node_id FROM filtered_concepts + ) + GROUP BY + concept_node_meta.concept_node_id + ) + SELECT + concept_node.*, + ds.REF as dataset, + ds.abbreviation AS studyAcronym, + continuous_min.VALUE as min, continuous_max.VALUE as max, + categorical_values.VALUE as values, + allow_filtering.allowFiltering AS allowFiltering, + meta_description.VALUE AS description, + aggregated_meta.metadata AS metadata + FROM + filtered_concepts as concept_node + LEFT JOIN dataset AS ds ON concept_node.dataset_id = ds.dataset_id + LEFT JOIN concept_node_meta AS meta_description ON concept_node.concept_node_id = meta_description.concept_node_id AND meta_description.KEY = 'description' + LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min' + LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max' + LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values' + LEFT JOIN allow_filtering ON concept_node.concept_node_id = allow_filtering.concept_node_id + LEFT JOIN aggregated_meta ON concept_node.concept_node_id = aggregated_meta.concept_node_id + """; + + MapSqlParameterSource params = + new MapSqlParameterSource().addValue("conceptPaths", conceptPaths).addValue("disallowed_meta_keys", disallowedMetaFields); + return template.query(sql, params, conceptRowWithMetaMapper); + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java index 0201691..60c1418 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptResultSetUtil.java @@ -3,19 +3,17 @@ import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept; import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept; import edu.harvard.dbmi.avillach.dictionary.util.JsonBlobParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.Map; @Component public class ConceptResultSetUtil { - private static final Logger log = LoggerFactory.getLogger(ConceptResultSetUtil.class); private final JsonBlobParser jsonBlobParser; @Autowired @@ -23,7 +21,12 @@ public ConceptResultSetUtil(JsonBlobParser jsonBlobParser) { this.jsonBlobParser = jsonBlobParser; } - public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { + public CategoricalConcept mapCategoricalWithMetadata(ResultSet rs) throws SQLException { + Map metadata = jsonBlobParser.parseMetaData(rs.getString("metadata")); + return new CategoricalConcept(getCategoricalConcept(rs), metadata); + } + + private CategoricalConcept getCategoricalConcept(ResultSet rs) throws SQLException { return new CategoricalConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), rs.getString("description"), rs.getString("values") == null ? List.of() : jsonBlobParser.parseValues(rs.getString("values")), @@ -31,7 +34,12 @@ public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { ); } - public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { + public ContinuousConcept mapContinuousWithMetadata(ResultSet rs) throws SQLException { + Map metadata = jsonBlobParser.parseMetaData(rs.getString("metadata")); + return new ContinuousConcept(getContinuousConcept(rs), metadata); + } + + private ContinuousConcept getContinuousConcept(ResultSet rs) throws SQLException { return new ContinuousConcept( rs.getString("concept_path"), rs.getString("name"), rs.getString("display"), rs.getString("dataset"), rs.getString("description"), rs.getBoolean("allowFiltering"), jsonBlobParser.parseMin(rs.getString("values")), @@ -39,6 +47,12 @@ public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { ); } + public ContinuousConcept mapContinuous(ResultSet rs) throws SQLException { + return getContinuousConcept(rs); + } + public CategoricalConcept mapCategorical(ResultSet rs) throws SQLException { + return getCategoricalConcept(rs); + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowWithMetaMapper.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowWithMetaMapper.java new file mode 100644 index 0000000..c12a02d --- /dev/null +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRowWithMetaMapper.java @@ -0,0 +1,30 @@ +package edu.harvard.dbmi.avillach.dictionary.concept; + +import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept; +import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; + +@Component +public class ConceptRowWithMetaMapper implements RowMapper { + + private final ConceptResultSetUtil conceptResultSetUtil; + + @Autowired + public ConceptRowWithMetaMapper(ConceptResultSetUtil conceptResultSetUtil) { + this.conceptResultSetUtil = conceptResultSetUtil; + } + + @Override + public Concept mapRow(ResultSet rs, int rowNum) throws SQLException { + return switch (ConceptType.toConcept(rs.getString("concept_type"))) { + case Categorical -> conceptResultSetUtil.mapCategoricalWithMetadata(rs); + case Continuous -> conceptResultSetUtil.mapContinuousWithMetadata(rs); + }; + } + +} diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java index dc35254..668c51c 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java @@ -71,4 +71,7 @@ public Optional conceptDetailWithoutAncestors(String dataset, String co return getConcept(dataset, conceptPath, false); } + public List conceptsWithDetail(List conceptPaths) { + return this.conceptRepository.getConceptsByPathWithMetadata(conceptPaths); + } } diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java index f976e7c..c9a9a0a 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/util/JsonBlobParser.java @@ -1,21 +1,31 @@ package edu.harvard.dbmi.avillach.dictionary.util; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import org.json.JSONArray; import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; @Component public class JsonBlobParser { private final static Logger log = LoggerFactory.getLogger(JsonBlobParser.class); + private final ObjectMapper objectMapper = new ObjectMapper(); + + public JsonBlobParser() {} public List parseValues(String valuesArr) { try { @@ -62,4 +72,25 @@ public Float parseMax(String valuesArr) { return parseFromIndex(valuesArr, 1); } + public Map parseMetaData(String jsonMetaData) { + Map metadata; + + try { + List> maps = objectMapper.readValue(jsonMetaData, new TypeReference>>() {}); + // convert the list to a flat map + Map map = new HashMap<>(); + for (Map entry : maps) { + if (map.put(entry.get("key"), entry.get("value")) != null) { + throw new IllegalStateException( + "parseMetaData() Duplicate key found in metadata. Key: " + entry.get("key") + " Value: " + entry.get("value") + ); + } + } + metadata = map; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return metadata; + } } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java index 363c180..d6dcb9a 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java @@ -148,4 +148,18 @@ void shouldDumpConcepts() { Assertions.assertEquals(concepts, actual.getBody().getContent()); Assertions.assertEquals(HttpStatus.OK, actual.getStatusCode()); } + + @Test + void shouldReturnConceptsWithMeta() { + CategoricalConcept fooBar = new CategoricalConcept( + "/foo//bar", "bar", "Bar", "my_dataset", "foo!", List.of("a", "b"), true, "", List.of(), Map.of("key", "value") + ); + Concept fooBaz = new ContinuousConcept("/foo//baz", "baz", "Baz", "my_dataset", "foo!", true, 0F, 100F, "", Map.of("key", "value")); + List concepts = List.of(fooBar, fooBaz); + List conceptPaths = List.of("/foo//bar", "/foo//bar"); + Mockito.when(conceptService.conceptsWithDetail(conceptPaths)).thenReturn(concepts); + ResponseEntity> listResponseEntity = subject.conceptsDetail(conceptPaths); + Assertions.assertEquals(HttpStatus.OK, listResponseEntity.getStatusCode()); + Assertions.assertEquals(concepts, listResponseEntity.getBody()); + } } diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java index 8d20516..c1beed9 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java @@ -324,5 +324,31 @@ void shouldGetContConceptWithDecimalNotation() { Assertions.assertEquals(6.77f, concept.max()); } + @Test + void shouldGetConceptsByConceptPath() { + List conceptPaths = List.of( + "\\phs002385\\TXNUM\\", "\\phs000284\\pht001902\\phv00122507\\age\\", "\\phs000007\\pht000022" + "\\phv00004260\\FM219\\", + "\\NHANES\\examination\\physical fitness\\Stage 1 heart rate (per min)", "\\phs000007\\pht000021" + "\\phv00003844\\FL200\\", + "\\phs002715\\age\\" + ); + List conceptsByPath = subject.getConceptsByPathWithMetadata(conceptPaths); + Assertions.assertFalse(conceptsByPath.isEmpty()); + Assertions.assertEquals(6, conceptsByPath.size()); + } + + @Test + void shouldGetSameConceptMetaAsConceptDetails() { + List conceptPaths = List.of("\\phs002385\\TXNUM\\", "\\phs000284\\pht001902\\phv00122507\\age\\"); + List conceptsByPath = subject.getConceptsByPathWithMetadata(conceptPaths); + Assertions.assertFalse(conceptsByPath.isEmpty()); + + // Verify the meta data is correctly retrieve by comparing against known good query. + Concept concept = conceptsByPath.getFirst(); + Map expectedMeta = subject.getConceptMeta(concept.dataset(), concept.conceptPath()); + + // compare the maps to each other. + Map actualMeta = concept.meta(); + Assertions.assertEquals(actualMeta, expectedMeta); + } } From f00ad2637ae41829780d502a0f05dfc5f91249e2 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Wed, 20 Nov 2024 11:38:26 -0500 Subject: [PATCH 2/2] [CHORE] Fix duplicate type fields (no one told me) --- .../avillach/dictionary/concept/model/CategoricalConcept.java | 1 - .../avillach/dictionary/concept/model/ContinuousConcept.java | 1 - .../dbmi/avillach/dictionary/concept/model/ConceptTest.java | 3 +-- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java index 135a43a..de0cf29 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java @@ -42,7 +42,6 @@ public CategoricalConcept(String conceptPath, String dataset) { } - @JsonProperty("type") @Override public ConceptType type() { return ConceptType.Categorical; diff --git a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java index 021cb4f..7299d3e 100644 --- a/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java +++ b/src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java @@ -43,7 +43,6 @@ public ContinuousConcept( this(conceptPath, name, display, dataset, description, allowFiltering, min, max, studyAcronym, meta, null); } - @JsonProperty("type") @Override public ConceptType type() { return ConceptType.Continuous; diff --git a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java index b469338..35ed853 100644 --- a/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java +++ b/src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptTest.java @@ -78,8 +78,7 @@ void shouldIncludeTypeInList() throws JsonProcessingException { String actual = new ObjectMapper().writeValueAsString(concepts); String expected = - "[{\"conceptPath\":\"/foo//baz\",\"name\":\"baz\",\"display\":\"Baz\",\"dataset\":\"study_a\",\"description\":null,\"allowFiltering\":true,\"min\":0.0,\"max\":1.0,\"studyAcronym\":\"\",\"meta\":{},\"children\":null,\"table\":null,\"study\":null,\"type\":\"Continuous\"},{\"conceptPath\":\"/foo//bar\",\"name\":\"bar\",\"display\":\"Bar\",\"dataset\":\"study_a\",\"description\":null,\"values\":[\"a\",\"b\"],\"allowFiltering\":true,\"studyAcronym\":\"\",\"children\":null,\"meta\":{},\"table\":null,\"study\":null,\"type\":\"Categorical\"}]"; - + "[{\"conceptPath\":\"/foo//baz\",\"name\":\"baz\",\"display\":\"Baz\",\"dataset\":\"study_a\",\"description\":null,\"allowFiltering\":true,\"min\":0.0,\"max\":1.0,\"studyAcronym\":\"\",\"meta\":{},\"children\":null,\"table\":null,\"study\":null},{\"conceptPath\":\"/foo//bar\",\"name\":\"bar\",\"display\":\"Bar\",\"dataset\":\"study_a\",\"description\":null,\"values\":[\"a\",\"b\"],\"allowFiltering\":true,\"studyAcronym\":\"\",\"children\":null,\"meta\":{},\"table\":null,\"study\":null}]"; Assertions.assertEquals(expected, actual); } }