Skip to content

Commit

Permalink
I got it
Browse files Browse the repository at this point in the history
  • Loading branch information
Luke Sikina committed Jul 18, 2024
1 parent eb0213f commit fe80364
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 85 deletions.
119 changes: 109 additions & 10 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,6 @@ public String createFacetSQLAndPopulateParams(Filter filter, MapSqlParameterSour
}
}

private String createMultiCategorySQLWithSearch(Map<String, List<Facet>> facets, String search, MapSqlParameterSource params) {
return "";
}

private Map<String, String> createSQLSafeCategoryKeys(List<String> categories) {
HashMap<String, String> keys = new HashMap<>();
for (int i = 0; i < categories.size(); i++) {
Expand All @@ -52,6 +48,99 @@ private Map<String, String> createSQLSafeCategoryKeys(List<String> categories) {
return keys;
}

private String createMultiCategorySQLWithSearch(Map<String, List<Facet>> facets, String search, MapSqlParameterSource params) {
Map<String, String> categoryKeys = createSQLSafeCategoryKeys(facets.keySet().stream().toList());
params.addValue("search", search);

/*
for each category of facet present in the filter, create a query that gives all the concept IDs for those
facets
*/
String conceptsQuery = "WITH " + facets.keySet().stream().map(category -> {
List<String[]> selectedFacetsNotInCategory = facets.entrySet().stream()
.filter(e -> !e.getKey().equals(category))
.flatMap(e -> e.getValue().stream())
.map(facet -> new String[]{facet.category(), facet.name()})
.toList();
params.addValue("facets_not_in_cat_" + categoryKeys.get(category), selectedFacetsNotInCategory);
params.addValue("facet_category_" + categoryKeys.get(category), category);
return """
facet_category_%s_concepts AS (
SELECT
concept_node.concept_node_id
FROM
facet
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id
JOIN concept_node ON concept_node.concept_node_id = fcn.concept_node_id
LEFT JOIN concept_node_meta AS continuous_min ON concept_node.concept_node_id = continuous_min.concept_node_id AND continuous_min.KEY = 'min'
LEFT JOIN concept_node_meta AS continuous_max ON concept_node.concept_node_id = continuous_max.concept_node_id AND continuous_max.KEY = 'max'
LEFT JOIN concept_node_meta AS categorical_values ON concept_node.concept_node_id = categorical_values.concept_node_id AND categorical_values.KEY = 'values'
WHERE
(fc.name, facet.name) IN (:facets_not_in_cat_%s)
AND concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery
AND (
continuous_min.value <> '' OR
continuous_max.value <> '' OR
categorical_values.value <> ''
)
)
""".formatted(categoryKeys.get(category), categoryKeys.get(category));
}).collect(Collectors.joining(",\n"));
/*
Categories with no selected facets contribute no concepts, so ignore them for now.
Now, for each category with selected facets, take all the concepts from all other categories with selections
and INTERSECT them. This creates the concepts for this category
*/
String selectedFacetsQuery = facets.keySet().stream().map(category -> {
String allConceptsForCategory = categoryKeys.values().stream()
.filter(key -> !categoryKeys.get(category).equals(key))
.map(key -> "SELECT * FROM facet_category_" + key + "_concepts")
.collect(Collectors.joining(" UNION "));
params.addValue("", "");
return """
(
SELECT
facet.facet_id, count(*)
FROM
facet
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
WHERE
fcn.concept_node_id IN (%s)
GROUP BY
facet.facet_id
)
""".formatted(allConceptsForCategory);
})
.collect(Collectors.joining("\n\tUNION\n"));

/*
For categories with no selected facets, take all the concepts from all facets, and use them for the counts
*/
params.addValue("all_selected_facet_categories", facets.keySet());
String allConceptsForUnselectedCategories = categoryKeys.values().stream()
.map(key -> "SELECT * FROM facet_category_" + key + "_concepts")
.collect(Collectors.joining(" UNION "));
String unselectedFacetsQuery = """
UNION
(
SELECT
facet.facet_id, count(*)
FROM
facet
JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
WHERE
fc.name NOT IN (:all_selected_facet_categories)
AND fcn.concept_node_id IN (%s)
GROUP BY
facet.facet_id
)
""".formatted(allConceptsForUnselectedCategories);

return conceptsQuery + selectedFacetsQuery + unselectedFacetsQuery;
}

private String createMultiCategorySQLNoSearch(Map<String, List<Facet>> facets, MapSqlParameterSource params) {
Map<String, String> categoryKeys = createSQLSafeCategoryKeys(facets.keySet().stream().toList());

Expand Down Expand Up @@ -175,7 +264,8 @@ facet.facet_id, count(*)
continuous_max.value <> '' OR
categorical_values.value <> ''
)
GROUP BY facet.facet_id
GROUP BY
facet.facet_id
)
UNION
(
Expand Down Expand Up @@ -204,10 +294,12 @@ facet.facet_id, count(*)
FROM
facet
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id
JOIN matching_concepts ON fcn.concept_node_id = matching_concepts.concept_node_id
WHERE
fc.name = :facet_category_name
GROUP BY facet.facet_id
fc.name <> :facet_category_name
GROUP BY
facet.facet_id
)
""";
}
Expand Down Expand Up @@ -237,7 +329,8 @@ facet.facet_id, count(*)
continuous_max.value <> '' OR
categorical_values.value <> ''
)
GROUP BY facet.facet_id
GROUP BY
facet.facet_id
)
UNION
(
Expand Down Expand Up @@ -266,10 +359,12 @@ facet.facet_id, count(*)
FROM
facet
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id
JOIN matching_concepts ON fcn.concept_node_id = matching_concepts.concept_node_id
WHERE
fc.name = :facet_category_name
GROUP BY facet.facet_id
fc.name <> :facet_category_name
GROUP BY
facet.facet_id
)
""";
}
Expand Down Expand Up @@ -297,6 +392,8 @@ facet.facet_id, count(*)
continuous_max.value <> '' OR
categorical_values.value <> ''
)
GROUP BY
facet.facet_id
""";

}
Expand All @@ -319,6 +416,8 @@ facet.facet_id, count(*)
continuous_min.value <> '' OR
continuous_max.value <> '' OR
categorical_values.value <> ''
GROUP BY
facet.facet_id
""";
}
}
86 changes: 11 additions & 75 deletions src/main/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,27 @@ public class FacetRepository {

private final NamedParameterJdbcTemplate template;

private final FilterQueryGenerator generator;

private final FacetMapper mapper;

private final FacetQueryGenerator generator;

@Autowired
public FacetRepository(
NamedParameterJdbcTemplate template, FilterQueryGenerator generator, FacetMapper mapper
NamedParameterJdbcTemplate template, FacetQueryGenerator generator, FacetMapper mapper
) {
this.template = template;
this.generator = generator;
this.mapper = mapper;
}

public List<FacetCategory> getFacets(Filter filter) {
QueryParamPair pair = generator.generateFilterQuery(filter, Pageable.unpaged());
MapSqlParameterSource params = new MapSqlParameterSource();
String innerSQL = generator.createFacetSQLAndPopulateParams(filter, params);
// return a list of facets and the number of concepts associated with them
String sql = """
WITH facet_counts_q AS (
%s
)
SELECT
facet_category.name AS category_name,
parent_facet.name AS parent_name,
Expand All @@ -48,81 +52,13 @@ public List<FacetCategory> getFacets(Filter filter) {
facet.name, facet.display, facet.description
FROM
facet
LEFT JOIN facet_counts_q ON facet.facet_id = facet_counts_q.facet_id
LEFT JOIN facet_category ON facet_category.facet_category_id = facet.facet_category_id
LEFT JOIN facet as parent_facet ON facet.parent_id = parent_facet.facet_id
""".formatted(pair.query());

return template.query(sql, pair.params(), new FacetCategoryExtractor());
}


private String createFacetSQL(Filter filter, MapSqlParameterSource params) {
String searchSQL;
String orderSQL;
if (StringUtils.hasLength(filter.search())) {
searchSQL = """
concept_node.searchable_fields @@ (phraseto_tsquery(:search)::text || ':*')::tsquery AND
(
continuous_min.value <> '' OR
continuous_max.value <> '' OR
categorical_values.value <> ''
)
""";
}
if (CollectionUtils.isEmpty(filter.facets())) {
return "";
}
Map<String, List<Facet>> categories = filter.facets().stream().collect(Collectors.groupingBy(Facet::category));
if (categories.size() == 1) {
return """
(
SELECT
facet.facet_id, count(*)
FROM facet
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
JOIN concept_node ON concept_node.concept_node_id = facet__concept_node.concept_node
JOIN facet_category fc on fc.facet_category_id = facet.facet_category_id
WHERE fc.name = :facet_category_name
GROUP BY facet.facet_id
)
""";
}
return categories.keySet().stream().map(category -> {
// get all the selected facets NOT IN this category
// get all the concepts for those facets
// get all the facets that match those concepts
List<List<String>> facetsNotInCategory = filter.facets().stream()
.filter(c -> !c.category().equals(category))
.map(facet -> List.of(facet.name(), facet.category()))
.toList();
params.addValue("facets_not_in_" + category, facetsNotInCategory);
params.addValue("category_" + category, category);
""".formatted(innerSQL);

return """
(
WITH study_ids_dataset_ids_query AS (
SELECT
distinct(concept_node_id) AS concept_node_id
FROM
facet
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
LEFT join facet_category fc ON fc.facet_category_id = facet.facet_category_id
WHERE
(facet.name, fc.name) IN (:facets_not_in_%s)
)
SELECT facet.name, count(*)
FROM
facet
JOIN facet__concept_node fcn ON fcn.facet_id = facet.facet_id
JOIN study_ids_dataset_ids_query ON study_ids_dataset_ids_query.concept_node_id = fcn.concept_node_id
LEFT JOIN facet_category fc ON fc.facet_category_id = facet.facet_category_id
WHERE
fc.name = :category_%s
GROUP BY facet.name
)
""".formatted(category, category);
}).collect(Collectors.joining("\n\tUNION\n"));
return template.query(sql, params, new FacetCategoryExtractor());
}

public Optional<Facet> getFacet(String facetCategory, String facet) {
Expand Down
16 changes: 16 additions & 0 deletions src/test/java/edu/harvard/dbmi/avillach/dictionary/facet/FacetQueryGeneratorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,20 @@ void shouldaaa() {
String query = generator.createFacetSQLAndPopulateParams(jim, new MapSqlParameterSource());
System.out.println(query);
}

@Test
void shouldaaa2() {
Filter jim = new Filter(List.of(
new Facet("phs000007", "", "", null, null, "study_ids_dataset_ids", null)
), "");
String query = generator.createFacetSQLAndPopulateParams(jim, new MapSqlParameterSource());
System.out.println(query);
}

@Test
void shoudlbrungus() {
Filter jim = new Filter(List.of(), "");
String query = generator.createFacetSQLAndPopulateParams(jim, new MapSqlParameterSource());
System.out.println(query);
}
}

0 comments on commit fe80364

Please sign in to comment.