From 100eb771be43280103044cbcf66aa041a264c594 Mon Sep 17 00:00:00 2001 From: Giuseppe Nespolino Date: Wed, 26 Jun 2024 17:09:57 +0200 Subject: [PATCH] feat: Improve TEI Performance [DHIS2-14884] (#14039) * style: formatting * feat: peer review comments [DHIS2-14884] * test: fixing tests [DHIS2-14884] * test: fixing tests [DHIS2-14884] * feat: more fields extracted from json [DHIS2-14884] * feat: more fields extracted from json [DHIS2-14884] * feat: fixing warning [DHIS2-14884] * fix: e2e tests after date fields renamed * test: merge from master [DHIS2-13779] * fix: compilation problem [DHIS2-13779] * fix: fix code after merge from master [DHIS2-14884] * fix: fix e2e tests after merge [DHIS2-14884] * fix: fix sonar warning [DHIS2-14884] * fix: fixed more tests [DHIS2-14884] * fix: fixed more tests [DHIS2-14884] * fix: merged from master [DHIS2-14884] * refactor: cleaner design [DHIS2-14884] * refactor: rollback param name [DHIS2-14884] * fix: unit tests [DHIS2-14884] fix: unit tests [DHIS2-14884] feat: orderBy using subqueries [DHIS2-14884] * fix: peer review [DHIS2-14884] * fix: peer review [DHIS2-14884] * fix: peer review [DHIS2-14884] * feat: rowcontext from json [DHIS2-14884] * feat: rowcontext from json [DHIS2-14884] * feat: rowcontext from json [DHIS2-14884] * feat: conditions are now exists subquery, left joins removed [DHIS2-14884] * feat: conditions are now exists subquery, left joins removed [DHIS2-14884] * fix: fixes some small issues (dates and SCHEDULE)[DHIS2-14884] * test: makes some e2e more consistent [DHIS2-14884] * fix: fixes some small issues (dates and SCHEDULE)[DHIS2-14884] * test: makes some e2e more consistent [DHIS2-14884] * test: fixes unit test [DHIS2-14884] * test: adds date formatter test [DHIS2-14884] * fix: missing import after merge [DHIS2-14884] * fix: missing import after merge [DHIS2-14884] * fix: fixes NPE [DHIS2-17577] --------- Co-authored-by: Maikel Arabori <51713408+maikelarabori@users.noreply.github.com> --- .../common/collection/CollectionUtils.java | 17 + .../java/org/hisp/dhis/util/DateUtils.java | 13 + .../dhis/analytics/common/QueryCreator.java | 35 ++ .../hisp/dhis/analytics/common/SqlQuery.java | 44 ++- .../analytics/common/SqlQueryExecutor.java | 6 +- .../dhis/analytics/common/TeiListGrid.java | 75 ++-- .../analytics/common/ValueTypeMapping.java | 22 +- .../params/dimension/DimensionIdentifier.java | 4 + .../dimension/DimensionIdentifierHelper.java | 12 + .../params/dimension/DimensionParam.java | 5 + .../dhis/analytics/common/query/Field.java | 21 ++ .../jsonextractor/EnrollmentExtractor.java | 79 +++++ .../query/jsonextractor/EventExtractor.java | 65 ++++ .../query/jsonextractor/JsonEnrollment.java | 81 +++++ .../jsonextractor/JsonExtractorUtils.java | 65 ++++ .../jsonextractor/SqlRowSetDelegator.java | 41 +++ .../SqlRowSetJsonExtractorDelegator.java | 329 ++++++++++++++++++ .../tei/TeiAnalyticsQueryService.java | 1 + .../tei/query/DataElementCondition.java | 6 +- .../querybuilder/DataElementQueryBuilder.java | 104 ++---- .../querybuilder/LeftJoinsQueryBuilder.java | 151 -------- .../context/querybuilder/OffsetHelper.java | 81 +++++ .../querybuilder/OrgUnitQueryBuilder.java | 9 +- .../querybuilder/PeriodQueryBuilder.java | 40 ++- .../context/querybuilder/SqlQueryHelper.java | 282 +++++++++++---- .../querybuilder/StatusQueryBuilder.java | 31 +- .../context/querybuilder/TeiQueryBuilder.java | 45 ++- .../query/context/sql/RenderableSqlQuery.java | 6 +- .../query/context/sql/SqlQueryCreator.java | 13 +- .../analytics/common/GridAdaptorTest.java | 10 +- .../jsonextractor/JsonExtractorUtilsTest.java | 56 +++ .../tei/query/RenderableDataValueTest.java | 7 +- .../analytics/tei/query/TeiSqlQueryTest.java | 14 +- .../DataElementQueryBuilderTest.java | 2 +- .../querybuilder/OffsetHelperTest.java | 67 ++++ .../querybuilder/SqlQueryHelperTest.java | 320 ++++++++++------- .../org/hisp/dhis/system/grid/ListGrid.java | 4 - 37 files changed, 1602 insertions(+), 561 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/QueryCreator.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EnrollmentExtractor.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EventExtractor.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonEnrollment.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtils.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetDelegator.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetJsonExtractorDelegator.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelper.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtilsTest.java create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelperTest.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/collection/CollectionUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/collection/CollectionUtils.java index b099fefe0ebb..b0180aa14011 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/collection/CollectionUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/collection/CollectionUtils.java @@ -31,6 +31,7 @@ import static java.util.stream.Collectors.toUnmodifiableSet; import static lombok.AccessLevel.PRIVATE; +import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -385,4 +386,20 @@ public static boolean containsUid( } return false; } + + /** + * Merges the given maps into a single map. The order of the maps is important, as the maps are + * merged in the order they are provided, meaning that the last map will override any duplicate + * keys from the previous maps. + * + * @param maps the maps to merge + * @param the type of the keys and values in the maps + * @return the merged map + */ + @SafeVarargs + public static Map merge(Map... maps) { + Map result = new HashMap<>(); + Stream.of(maps).forEach(result::putAll); + return ImmutableMap.copyOf(result); + } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/DateUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/DateUtils.java index 97bdec837f2f..1070c6dc67ff 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/DateUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/DateUtils.java @@ -145,6 +145,9 @@ public class DateUtils { private static final DateTimeFormatter LONG_DATE_FORMAT_WITH_MILLIS = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS"); + private static final DateTimeFormatter LONG_DATE_FORMAT_NO_T = + DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private static final DateTimeFormatter HTTP_DATE_FORMAT = DateTimeFormat.forPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.ENGLISH); @@ -206,6 +209,16 @@ public static String toLongDate(Date date) { return date != null ? LONG_DATE_FORMAT.print(new DateTime(date)) : null; } + /** + * Formats a Date to the format yyyy-MM-dd HH:mm:ss.S + * + * @param date the Date to parse. + * @return A formatted date string. + */ + public static String toLongDateNoT(Date date) { + return date != null ? LONG_DATE_FORMAT_NO_T.print(new DateTime(date)) : null; + } + /** * Formats a Date to the format yyyy-MM-dd HH:mm:ss. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/QueryCreator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/QueryCreator.java new file mode 100644 index 000000000000..6b8b6438827c --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/QueryCreator.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common; + +public interface QueryCreator { + + Query createForSelect(); + + Query createForCount(); +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQuery.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQuery.java index ec7d7f7fda41..8bc2d6a077de 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQuery.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQuery.java @@ -30,10 +30,16 @@ import static org.springframework.util.Assert.hasText; import static org.springframework.util.Assert.notNull; +import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.tei.query.context.sql.QueryContext; /** * @see org.hisp.dhis.analytics.common.Query @@ -41,23 +47,47 @@ */ @EqualsAndHashCode @Slf4j -@Getter public class SqlQuery implements Query { - private final String statement; - private final Map params; + @Getter private final String statement; + private final SqlQueryContext sqlQueryContext; /** * @throws IllegalArgumentException if statement or params are null/empty/blank. */ - public SqlQuery(String statement, Map params) { + private SqlQuery(String statement, SqlQueryContext sqlQueryContext) { hasText(statement, "The 'statement' must not be null/empty/blank"); - notNull(params, "The 'params' must not be null"); + notNull(sqlQueryContext, "The 'params' must not be null"); this.statement = statement; - this.params = params; + this.sqlQueryContext = sqlQueryContext; log.debug("STATEMENT: " + statement); - log.debug("PARAMS: " + params); + log.debug("PARAMS: " + sqlQueryContext); + } + + public static SqlQuery of(String render, QueryContext queryContext) { + return new SqlQuery( + render, + new SqlQueryContext( + queryContext.getParametersPlaceHolder(), + queryContext.getTeiQueryParams().getCommonParams().streamDimensions().toList())); + } + + @Nonnull + @Override + public Map getParams() { + return sqlQueryContext.getParams(); + } + + public List> getDimensionIdentifiers() { + return sqlQueryContext.getDimensionIdentifiers(); + } + + @RequiredArgsConstructor + @Getter + private static class SqlQueryContext { + private final Map params; + private final List> dimensionIdentifiers; } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java index 8444c1488744..98b8fbb16359 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/SqlQueryExecutor.java @@ -31,6 +31,7 @@ import java.util.Optional; import javax.annotation.Nonnull; +import org.hisp.dhis.analytics.common.query.jsonextractor.SqlRowSetJsonExtractorDelegator; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; @@ -54,14 +55,15 @@ public SqlQueryExecutor(@Qualifier("analyticsReadOnlyJdbcTemplate") JdbcTemplate * @throws IllegalArgumentException if the query argument is null. */ @Override - public SqlQueryResult find(SqlQuery query) { + public SqlQueryResult find(@Nonnull SqlQuery query) { notNull(query, "The 'query' must not be null"); SqlRowSet rowSet = namedParameterJdbcTemplate.queryForRowSet( query.getStatement(), new MapSqlParameterSource().addValues(query.getParams())); - return new SqlQueryResult(rowSet); + return new SqlQueryResult( + new SqlRowSetJsonExtractorDelegator(rowSet, query.getDimensionIdentifiers())); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TeiListGrid.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TeiListGrid.java index 021ce6d74896..fb98beff7a30 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TeiListGrid.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/TeiListGrid.java @@ -27,30 +27,30 @@ */ package org.hisp.dhis.analytics.common; +import static java.lang.String.format; import static java.util.Objects.isNull; import com.fasterxml.jackson.annotation.JsonIgnore; import java.sql.ResultSet; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.commons.lang3.StringUtils; +import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.query.jsonextractor.SqlRowSetJsonExtractorDelegator; import org.hisp.dhis.analytics.tei.TeiQueryParams; import org.hisp.dhis.common.Grid; import org.hisp.dhis.common.GridHeader; -import org.hisp.dhis.common.ValueStatus; import org.hisp.dhis.common.ValueType; -import org.hisp.dhis.event.EventStatus; import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.system.util.MathUtils; import org.springframework.jdbc.support.rowset.SqlRowSet; +@Slf4j public class TeiListGrid extends ListGrid { private static final List ROUNDABLE_TYPES = @@ -112,7 +112,8 @@ public Grid addNamedRows(SqlRowSet rs) { addValue(value); headersSet.add(columnLabel); - rowContextItems.putAll(getRowContextItem(rs, cols[i], i)); + rowContextItems.putAll( + ((SqlRowSetJsonExtractorDelegator) rs).getRowContextItem(cols[i], i)); } } if (!rowContextItems.isEmpty()) { @@ -130,17 +131,26 @@ public Grid addNamedRows(SqlRowSet rs) { private Object getValueAndRoundIfNecessary( SqlRowSet rs, String columnLabel, boolean skipRounding) { ValueType valueType = getValueType(columnLabel); + Object value = rs.getObject(columnLabel); if (skipRounding || isNotRoundableType(valueType)) { - return rs.getObject(columnLabel); + return value; + } + // if roundable type we try to parse from string into double and round it + try { + return roundIfNecessary(value); + } catch (Exception e) { + log.warn( + format("Failed to parse value as double: %s for column: %s ", value, columnLabel), e); + // as a fallback we return the value as is + return value; } - return roundIfNecessary(rs, columnLabel); } - private Double roundIfNecessary(SqlRowSet rs, String columnLabel) { - if (isNull(rs.getObject(columnLabel))) { + private Double roundIfNecessary(Object value) { + if (isNull(value)) { return null; } - double doubleValue = rs.getDouble(columnLabel); + double doubleValue = Double.parseDouble(value.toString()); if (teiQueryParams.getCommonParams().isSkipRounding()) { return doubleValue; } @@ -161,49 +171,4 @@ private ValueType getValueType(String col) { .map(DimensionParam::getValueType) .orElse(ValueType.TEXT); } - - /** - * The method retrieves row context content that describes the origin of the data value, - * indicating whether it is set, not set, or undefined. The column index is used as the map key, - * and the corresponding value contains information about the origin, also known as the value - * status. - * - * @param rs the {@link ResultSet}, - * @param columnName the {@link String}, grid row column name - * @return Map of column index and value status - */ - private Map getRowContextItem(SqlRowSet rs, String columnName, int rowIndex) { - Map rowContextItem = new HashMap<>(); - String existIndicatorColumnLabel = columnName + EXISTS; - String statusIndicatorColumnLabel = columnName + STATUS; - String hasValueIndicatorColumnLabel = columnName + HAS_VALUE; - - if (Arrays.stream(rs.getMetaData().getColumnNames()) - .anyMatch(n -> n.equalsIgnoreCase(existIndicatorColumnLabel))) { - - boolean isDefined = rs.getBoolean(existIndicatorColumnLabel); - boolean isSet = rs.getBoolean(hasValueIndicatorColumnLabel); - boolean isScheduled = - StringUtils.equalsIgnoreCase( - rs.getString(statusIndicatorColumnLabel), EventStatus.SCHEDULE.toString()); - - ValueStatus valueStatus = ValueStatus.SET; - - if (!isDefined) { - valueStatus = ValueStatus.NOT_DEFINED; - } else if (isScheduled) { - valueStatus = ValueStatus.SCHEDULED; - } else if (!isSet) { - valueStatus = ValueStatus.NOT_SET; - } - - if (valueStatus != ValueStatus.SET) { - Map valueStatusMap = new HashMap<>(); - valueStatusMap.put("valueStatus", valueStatus.getValue()); - rowContextItem.put(Integer.toString(rowIndex), valueStatusMap); - } - } - - return rowContextItem; - } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ValueTypeMapping.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ValueTypeMapping.java index c5c2c6c541b1..bac5c446b1f2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ValueTypeMapping.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/ValueTypeMapping.java @@ -60,17 +60,11 @@ public enum ValueTypeMapping { DATE(ValueTypeMapping::dateConverter, LocalDate.class, LocalDateTime.class), TIME(s -> s, ValueType.TIME, s -> s.replace(".", ":"), "varchar"), BOOLEAN( - ValueTypeMapping::booleanConverter, - ValueTypeMapping::booleanSelectTransformer, - Boolean.class); - - private static final UnaryOperator BOOLEAN_SELECT_TRANSFORMER = - columnName -> - "case when " - + columnName - + " = 'true' then 1 when " - + columnName - + " = 'false' then 0 end"; + ValueTypeMapping::booleanConverter, ValueTypeMapping::booleanJsonExtractor, Boolean.class); + + private static final UnaryOperator BOOLEAN_JSON_EXTRACTOR = + value -> value.equalsIgnoreCase("true") ? "1" : "0"; + private final Function converter; @Getter private final UnaryOperator selectTransformer; private final ValueType[] valueTypes; @@ -115,7 +109,7 @@ private static boolean isAssignableFrom(Class[] classes, ValueType valueType) { Class... classes) { this.converter = converter; this.valueTypes = fromClasses(classes); - this.selectTransformer = selectTransformer; + this.selectTransformer = s -> Objects.isNull(s) ? null : selectTransformer.apply(s); this.argumentTransformer = UnaryOperator.identity(); this.postgresCast = name(); } @@ -148,8 +142,8 @@ private static boolean isTrue(String value) { return "1".equals(value) || "true".equalsIgnoreCase(value); } - private static String booleanSelectTransformer(String columnName) { - return BOOLEAN_SELECT_TRANSFORMER.apply(columnName); + private static String booleanJsonExtractor(String value) { + return BOOLEAN_JSON_EXTRACTOR.apply(value); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifier.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifier.java index cb55865e484c..fe0994bc663a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifier.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifier.java @@ -105,6 +105,10 @@ public boolean isEnrollmentDimension() { return hasProgram() && !hasProgramStage(); } + public boolean isTeDimension() { + return !hasProgram() && !hasProgramStage(); + } + public boolean isEventDimension() { return hasProgram() && hasProgramStage(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelper.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelper.java index 328965242174..e5e996576977 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelper.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionIdentifierHelper.java @@ -36,8 +36,10 @@ import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension.INCIDENTDATE; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension.OCCURREDDATE; import static org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension.OUNAME; +import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.DATA_ELEMENT; import static org.hisp.dhis.analytics.common.params.dimension.ElementWithOffset.emptyElementWithOffset; import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.TEI_ALIAS; +import static org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryBuilders.isOfType; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_IDENTIFIER_SEP; import static org.hisp.dhis.commons.util.TextUtils.doubleQuote; @@ -280,4 +282,14 @@ private static String getStaticDimensionDisplayName( return null; } } + + /** + * Checks if the given dimension identifier is of type data element. + * + * @param dimensionIdentifier the dimension identifier to check. + * @return true if the dimension identifier is of type data element, false otherwise. + */ + public static boolean isDataElement(DimensionIdentifier dimensionIdentifier) { + return isOfType(dimensionIdentifier, DATA_ELEMENT) && dimensionIdentifier.isEventDimension(); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java index 691677313219..d4de3ecd2f9c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/params/dimension/DimensionParam.java @@ -59,6 +59,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.common.ValueTypeMapping; import org.hisp.dhis.analytics.tei.query.context.TeiHeaderProvider; import org.hisp.dhis.analytics.tei.query.context.TeiStaticField; import org.hisp.dhis.common.DimensionalObject; @@ -241,6 +242,10 @@ public String getName() { return staticDimension.name(); } + public String transformValue(String value) { + return ValueTypeMapping.fromValueType(getValueType()).getSelectTransformer().apply(value); + } + @RequiredArgsConstructor public enum StaticDimension implements TeiHeaderProvider { TRACKEDENTITYINSTANCEUID("Tracked entity instance UID", TEXT, STATIC, TRACKED_ENTITY_INSTANCE), diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/Field.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/Field.java index f6e9f82ff65c..ca13252bc71c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/Field.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/Field.java @@ -35,7 +35,9 @@ import static org.hisp.dhis.commons.util.TextUtils.doubleQuote; import lombok.AccessLevel; +import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.With; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; @@ -55,6 +57,16 @@ public class Field extends BaseRenderable { private final Boolean quotingNeeded; + /** a flag to indicate whether the field will be used in the headers */ + @With @Getter private final boolean usedInHeaders; + + /** virtual fields won't be added to the select clause */ + @With @Getter private final boolean virtual; + + public Field asVirtual() { + return withVirtual(true); + } + /** * Static constructor for a field which double quote "name" when rendered. * @@ -67,6 +79,15 @@ public static Field of(String tableAlias, Renderable name, String fieldAlias) { return of(tableAlias, name, fieldAlias, DimensionIdentifier.EMPTY, true); } + private static Field of( + String tableAlias, + Renderable name, + String alias, + DimensionIdentifier dimensionIdentifier, + boolean quotingNeeded) { + return of(tableAlias, name, alias, dimensionIdentifier, quotingNeeded, true, false); + } + public static Field ofDimensionIdentifier( DimensionIdentifier dimensionIdentifier) { return ofRenamedDimensionIdentifier( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EnrollmentExtractor.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EnrollmentExtractor.java new file mode 100644 index 000000000000..4cf75e829f9b --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EnrollmentExtractor.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common.query.jsonextractor; + +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension; + +/** + * This enum represents a mapping between static dimensions and their corresponding getter method + */ +@RequiredArgsConstructor +enum EnrollmentExtractor { + ENROLLMENTDATE( + DimensionParam.StaticDimension.ENROLLMENTDATE, + a -> JsonExtractorUtils.getFormattedDate(a.getEnrollmentDate())), + INCIDENTDATE( + DimensionParam.StaticDimension.INCIDENTDATE, + a -> JsonExtractorUtils.getFormattedDate(a.getIncidentDate())), + OUNAME(DimensionParam.StaticDimension.OUNAME, JsonEnrollment::getOrgUnitName), + OUCODE(DimensionParam.StaticDimension.OUCODE, JsonEnrollment::getOrgUnitCode), + OUNAMEHIERARCHY( + DimensionParam.StaticDimension.OUNAMEHIERARCHY, JsonEnrollment::getOrgUnitNameHierarchy), + ENROLLMENT_STATUS( + List.of(StaticDimension.ENROLLMENT_STATUS, StaticDimension.PROGRAM_STATUS), + JsonEnrollment::getEnrollmentStatus); + + // The static dimensions that this extractor is responsible for + private final List dimensions; + + // The function that extracts the value of the dimension from the enrollment + @Getter private final Function extractor; + + EnrollmentExtractor( + StaticDimension staticDimension, Function getOrgUnitNameHierarchy) { + this(List.of(staticDimension), getOrgUnitNameHierarchy); + } + + static EnrollmentExtractor byDimension(DimensionParam.StaticDimension dimension) { + return Arrays.stream(values()) + .filter( + enrollmentExtractor -> + enrollmentExtractor.dimensions.stream().anyMatch(dimension::equals)) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "No enrollment extractor is defined for static dimension " + dimension)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EventExtractor.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EventExtractor.java new file mode 100644 index 000000000000..e0ec67c26aad --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/EventExtractor.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common.query.jsonextractor; + +import java.util.Arrays; +import java.util.function.Function; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam.StaticDimension; + +/** + * This enum maps static dimensions to functions that extract the corresponding values from an + * JsonEvent + */ +@RequiredArgsConstructor +enum EventExtractor { + OCCURREDDATE( + StaticDimension.OCCURREDDATE, a -> JsonExtractorUtils.getFormattedDate(a.getOccurredDate())), + OUNAME(DimensionParam.StaticDimension.OUNAME, JsonEnrollment.JsonEvent::getOrgUnitName), + OUCODE(DimensionParam.StaticDimension.OUCODE, JsonEnrollment.JsonEvent::getOrgUnitCode), + OUNAMEHIERARCHY( + DimensionParam.StaticDimension.OUNAMEHIERARCHY, + JsonEnrollment.JsonEvent::getOrgUnitNameHierarchy), + EVENT_STATUS(StaticDimension.EVENT_STATUS, JsonEnrollment.JsonEvent::getEventStatus); + + private final DimensionParam.StaticDimension dimension; + + @Getter private final Function extractor; + + static EventExtractor byDimension(DimensionParam.StaticDimension dimension) { + return Arrays.stream(values()) + .filter(eventExtractor -> eventExtractor.dimension.equals(dimension)) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "No event extractor is defined for static dimension " + dimension)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonEnrollment.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonEnrollment.java new file mode 100644 index 000000000000..295c4089cdff --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonEnrollment.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common.query.jsonextractor; + +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.Map; +import lombok.Data; + +@Data +class JsonEnrollment { + private String programUid; + + private String enrollmentUid; + + private LocalDateTime enrollmentDate; + + private LocalDateTime incidentDate; + + private LocalDateTime endDate; + + private String orgUnitUid; + + private String orgUnitName; + + private String orgUnitCode; + + private String orgUnitNameHierarchy; + + private String enrollmentStatus; + + private Collection events; + + @Data + static class JsonEvent { + private String programStageUid; + + private String eventUid; + + private LocalDateTime occurredDate; + + private LocalDateTime dueDate; + + private String orgUnitUid; + + private String orgUnitName; + + private String orgUnitCode; + + private String orgUnitNameHierarchy; + + private String eventStatus; + + private Map eventDataValues; + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtils.java new file mode 100644 index 000000000000..fbcd680ad146 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtils.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common.query.jsonextractor; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.regex.Pattern; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.hisp.dhis.util.DateUtils; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class JsonExtractorUtils { + + private static final Pattern TRAILING_ZEROES = Pattern.compile("0*$"); + private static final Pattern ENDING_WITH_DOT = Pattern.compile("\\.$"); + + public static String getFormattedDate(LocalDateTime date) { + if (date == null) { + return null; + } + + return withOutTrailingZeroes( + DateUtils.toLongDateNoT(Date.from(date.atZone(ZoneId.systemDefault()).toInstant()))); + } + + /** + * Removes trailing zeroes from the date string Examples: - "2020-01-01T00:00:00.000" -> + * "2020-01-01T00:00:00.0" - "2020-01-01T00:00:00.100" -> "2020-01-01T00:00:00.1" - + * "2020-01-01T00:00:00.010" -> "2020-01-01T00:00:00.01" - "2020-01-01T00:00:00.001" -> + * "2020-01-01T00:00:00.001" + * + * @param date date string + * @return date string without trailing zeroes + */ + private static String withOutTrailingZeroes(String date) { + return ENDING_WITH_DOT.matcher(TRAILING_ZEROES.matcher(date).replaceAll("")).replaceAll(".0"); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetDelegator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetDelegator.java new file mode 100644 index 000000000000..f43b35e3be03 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetDelegator.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common.query.jsonextractor; + +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; +import org.springframework.jdbc.support.rowset.SqlRowSet; + +/** + * This class is a simple SqlRowSet wrapper that delegates all calls to the wrapped SqlRowSet. It is + * used to simplify the implementation of the {@link SqlRowSetJsonExtractorDelegator} class. + */ +@RequiredArgsConstructor +class SqlRowSetDelegator implements SqlRowSet { + @Delegate private final SqlRowSet sqlRowSet; +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetJsonExtractorDelegator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetJsonExtractorDelegator.java new file mode 100644 index 000000000000..726a1d76a95d --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/jsonextractor/SqlRowSetJsonExtractorDelegator.java @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common.query.jsonextractor; + +import static java.util.Comparator.comparing; +import static java.util.Comparator.naturalOrder; +import static java.util.Comparator.nullsFirst; +import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; +import static org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifierHelper.isDataElement; +import static org.hisp.dhis.analytics.tei.query.context.querybuilder.OffsetHelper.getItemBasedOnOffset; +import static org.hisp.dhis.common.ValueType.ORGANISATION_UNIT; +import static org.hisp.dhis.feedback.ErrorCode.E7250; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import lombok.SneakyThrows; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType; +import org.hisp.dhis.analytics.common.query.jsonextractor.JsonEnrollment.JsonEvent; +import org.hisp.dhis.common.BaseIdentifiableObject; +import org.hisp.dhis.common.IdScheme; +import org.hisp.dhis.common.IllegalQueryException; +import org.hisp.dhis.common.ValueStatus; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.event.EventStatus; +import org.hisp.dhis.legend.LegendSet; +import org.springframework.jdbc.InvalidResultSetAccessException; +import org.springframework.jdbc.support.rowset.SqlRowSet; + +/** + * A {@link SqlRowSet} implementation that extracts values from a JSON string column in the wrapped + * {@link SqlRowSet} and returns them as if they were columns in the row set. + */ +public class SqlRowSetJsonExtractorDelegator extends SqlRowSetDelegator { + + private static final ObjectMapper OBJECT_MAPPER; + + private static final Map DEFAULT_ID_SCHEMES_BY_VALUE_TYPE = + Map.of(ORGANISATION_UNIT, IdScheme.NAME); + + private static final Map SUFFIX_BY_ID_SCHEME = + Map.of( + IdScheme.NAME, "_name", + IdScheme.CODE, "_code"); + + static { + OBJECT_MAPPER = new ObjectMapper(); + OBJECT_MAPPER.findAndRegisterModules(); + } + + @SneakyThrows + private List parseEnrollmentsFromJson(String json) { + return OBJECT_MAPPER.readValue(json, new TypeReference<>() {}); + } + + private static final Comparator ENR_ENROLLMENT_DATE_COMPARATOR = + comparing(JsonEnrollment::getEnrollmentDate, nullsFirst(naturalOrder())).reversed(); + + private static final Comparator EVT_OCCURRED_DATE_COMPARATOR = + comparing(JsonEnrollment.JsonEvent::getOccurredDate, nullsFirst(naturalOrder())).reversed(); + + private final transient Map> dimIdByKey; + + private final List existingColumnsInRowSet; + + public SqlRowSetJsonExtractorDelegator( + SqlRowSet sqlRowSet, List> dimensionIdentifiers) { + super(sqlRowSet); + this.dimIdByKey = new HashMap<>(); + for (DimensionIdentifier dimensionIdentifier : dimensionIdentifiers) { + if (!dimIdByKey.containsKey(dimensionIdentifier.getKey())) { + dimIdByKey.put(dimensionIdentifier.getKey(), dimensionIdentifier); + } + } + // we need to know which columns are in the sqlrowset, so that when a column is not present, we + // can check if it is present in the json string + this.existingColumnsInRowSet = Arrays.asList(sqlRowSet.getMetaData().getColumnNames()); + } + + @Override + @SneakyThrows + public Object getObject(String columnLabel) throws InvalidResultSetAccessException { + // if the column is present in the rowset, we invoke the default behavior + if (existingColumnsInRowSet.contains(columnLabel)) { + return super.getObject(columnLabel); + } + // if the column is not present in the rowset, we check if it is present in the json string + List enrollments = parseEnrollmentsFromJson(super.getString("enrollments")); + + DimensionIdentifier dimensionIdentifier = dimIdByKey.get(columnLabel); + + if (dimensionIdentifier.isEnrollmentDimension()) { + return getObjectForEnrollments(enrollments, dimensionIdentifier); + } + + if (dimensionIdentifier.isEventDimension()) { + return getObjectForEvents(enrollments, dimensionIdentifier); + } + throw new IllegalQueryException(E7250, dimensionIdentifier); + } + + private Object getObjectForEvents( + List enrollments, DimensionIdentifier dimensionIdentifier) { + return getJsonEnrollment(enrollments, dimensionIdentifier) + .map(jEnr -> getJsonEvent(dimensionIdentifier, jEnr)) + .map(jEvt -> getEventExtractor(dimensionIdentifier.getDimension()).apply(jEvt)) + .orElse(null); + } + + private static Optional getJsonEnrollment( + List enrollments, DimensionIdentifier dimensionIdentifier) { + return getItemBasedOnOffset( + enrollments.stream() + // gets only enrollments whose program is the same as specified in the dimension + .filter( + jEnr -> + jEnr.getProgramUid() + .equals(dimensionIdentifier.getProgram().getElement().getUid())), + ENR_ENROLLMENT_DATE_COMPARATOR, + dimensionIdentifier.getProgram().getOffsetWithDefault()); + } + + private Object getObjectForEnrollments( + List enrollments, DimensionIdentifier dimensionIdentifier) { + return getJsonEnrollment(enrollments, dimensionIdentifier) + .map(jEnr -> getEnrollmentExtractor(dimensionIdentifier.getDimension()).apply(jEnr)) + .orElse(null); + } + + private static JsonEvent getJsonEvent( + DimensionIdentifier dimensionIdentifier, JsonEnrollment jsonEnrollment) { + + if (isNull(jsonEnrollment)) { + return null; + } + + return getItemBasedOnOffset( + CollectionUtils.emptyIfNull(jsonEnrollment.getEvents()).stream() + // gets only events whose program stage is the same as specified in the + // dimension + .filter( + jEvt -> + jEvt.getProgramStageUid() + .equals(dimensionIdentifier.getProgramStage().getElement().getUid())), + EVT_OCCURRED_DATE_COMPARATOR, + dimensionIdentifier.getProgramStage().getOffsetWithDefault()) + .orElse(null); + } + + /** + * Returns a function that extracts the value of the dimension from the event. + * + * @param dimension the dimension + * @return the function to extract the value of the dimension from the event + */ + @SuppressWarnings("unchecked") + private Function getEventExtractor(DimensionParam dimension) { + if (dimension.isStaticDimension()) { + return EventExtractor.byDimension(dimension.getStaticDimension()).getExtractor(); + } + if (dimension.getDimensionParamObjectType().equals(DimensionParamObjectType.DATA_ELEMENT)) { + // it is a data element dimension here + return jsonEvent -> { + String rawValue = + Optional.of(jsonEvent) + .map(JsonEvent::getEventDataValues) + .map(map -> map.get(dimension.getQueryItem().getItemId())) + .map(o -> (Map) o) + .map(map -> map.get(getValueFieldName(dimension))) + .map(Objects::toString) + .orElse(null); + + if (isNull(rawValue)) { + return null; + } + + // apply legendSet mapping if present + if (dimension.getQueryItem().hasLegendSet()) { + return mapByLegendSet(dimension.getQueryItem().getLegendSet(), rawValue); + } + return dimension.transformValue(rawValue); + }; + } + if (dimension + .getDimensionParamObjectType() + .equals(DimensionParamObjectType.ORGANISATION_UNIT)) { + return JsonEvent::getOrgUnitUid; + } + throw new IllegalStateException("Unknown dimension identifier " + dimension); + } + + private Object mapByLegendSet(LegendSet legendSet, String rawValue) { + // RawValue should be a double + double value = Double.parseDouble(rawValue); + return legendSet.getLegends().stream() + .filter(legend -> value >= legend.getStartValue() && value < legend.getEndValue()) + .findFirst() + .map(BaseIdentifiableObject::getDisplayName) + .orElse(null); + } + + private String getValueFieldName(DimensionParam dimension) { + ValueType valueType = dimension.getValueType(); + IdScheme idScheme = + Optional.of(dimension) + .map(DimensionParam::getIdScheme) + .orElse(nonNull(valueType) ? DEFAULT_ID_SCHEMES_BY_VALUE_TYPE.get(valueType) : null); + if (nonNull(valueType) + && nonNull(idScheme) + && DEFAULT_ID_SCHEMES_BY_VALUE_TYPE.containsKey(valueType) + && SUFFIX_BY_ID_SCHEME.containsKey(idScheme)) { + return "value" + SUFFIX_BY_ID_SCHEME.get(idScheme); + } + return "value"; + } + + /** + * Returns a function that extracts the value of the dimension from the enrollment. + * + * @param dimension the dimension + * @return the function to extract the value of the dimension from the enrollment + */ + private Function getEnrollmentExtractor(DimensionParam dimension) { + if (dimension.isStaticDimension()) { + return EnrollmentExtractor.byDimension(dimension.getStaticDimension()).getExtractor(); + } + if (dimension + .getDimensionParamObjectType() + .equals(DimensionParamObjectType.ORGANISATION_UNIT)) { + return JsonEnrollment::getOrgUnitUid; + } + throw new IllegalQueryException(E7250, dimension.toString()); + } + + /** + * The method retrieves row context content that describes the origin of the data value, + * indicating whether it is set, not set, or undefined. The column index is used as the map key, + * and the corresponding value contains information about the origin, also known as the value + * status. + * + * @param columnName the {@link String}, grid row column name + * @return Map of column index and value status + */ + public Map getRowContextItem(String columnName, int rowIndex) { + + DimensionIdentifier dimensionIdentifier = dimIdByKey.get(columnName); + + // RowContext makes sense for data element dimensions only + if (isNull(dimensionIdentifier) + || !dimensionIdentifier.isEventDimension() + || !isDataElement(dimensionIdentifier)) { + return Collections.emptyMap(); + } + + JsonEvent event = + getJsonEnrollment( + parseEnrollmentsFromJson(super.getString("enrollments")), dimensionIdentifier) + .map(jEnr -> getJsonEvent(dimensionIdentifier, jEnr)) + .orElse(null); + + Map rowContextItem = new HashMap<>(); + + boolean isStageDefined = event != null; + boolean isSet = + event != null + && Objects.nonNull(event.getEventDataValues()) + && event.getEventDataValues().containsKey(dimensionIdentifier.getDimension().getUid()); + boolean isScheduled = + event != null + && StringUtils.equalsIgnoreCase( + event.getEventStatus(), EventStatus.SCHEDULE.toString()); + + ValueStatus valueStatus = ValueStatus.SET; + + if (!isStageDefined) { + valueStatus = ValueStatus.NOT_DEFINED; + } else if (isScheduled) { + valueStatus = ValueStatus.SCHEDULED; + } else if (!isSet) { + valueStatus = ValueStatus.NOT_SET; + } + + if (valueStatus != ValueStatus.SET) { + Map valueStatusMap = new HashMap<>(); + valueStatusMap.put("valueStatus", valueStatus.getValue()); + rowContextItem.put(Integer.toString(rowIndex), valueStatusMap); + } + + return rowContextItem; + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java index d968ced07dae..5ba08785147f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/TeiAnalyticsQueryService.java @@ -62,6 +62,7 @@ @Service @RequiredArgsConstructor public class TeiAnalyticsQueryService { + private final QueryExecutor queryExecutor; private final GridAdaptor gridAdaptor; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/DataElementCondition.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/DataElementCondition.java index 4f94d3c291ae..14179cf3dae8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/DataElementCondition.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/DataElementCondition.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.analytics.tei.query; -import static org.hisp.dhis.commons.util.TextUtils.doubleQuote; +import static org.hisp.dhis.commons.util.TextUtils.EMPTY; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.common.ValueTypeMapping; @@ -52,8 +52,6 @@ public String render() { static RenderableDataValue getDataValueRenderable( DimensionIdentifier dimensionIdentifier, ValueTypeMapping valueTypeMapping) { return RenderableDataValue.of( - doubleQuote(dimensionIdentifier.getPrefix()), - dimensionIdentifier.getDimension().getUid(), - valueTypeMapping); + EMPTY, dimensionIdentifier.getDimension().getUid(), valueTypeMapping); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java index 6e2d349ae9ea..7f36989cb3fc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilder.java @@ -30,27 +30,22 @@ import static java.util.Objects.nonNull; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.common.ValueTypeMapping.fromValueType; -import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.DATA_ELEMENT; import static org.hisp.dhis.analytics.common.query.Field.ofUnquoted; -import static org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryBuilders.isOfType; import static org.hisp.dhis.common.ValueType.ORGANISATION_UNIT; import static org.hisp.dhis.commons.util.TextUtils.doubleQuote; -import static org.hisp.dhis.system.grid.ListGrid.EXISTS; -import static org.hisp.dhis.system.grid.ListGrid.HAS_VALUE; import static org.hisp.dhis.system.grid.ListGrid.LEGEND; -import static org.hisp.dhis.system.grid.ListGrid.STATUS; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.Getter; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; +import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifierHelper; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; import org.hisp.dhis.analytics.common.query.Field; import org.hisp.dhis.analytics.common.query.GroupableCondition; @@ -59,7 +54,6 @@ import org.hisp.dhis.analytics.common.query.Renderable; import org.hisp.dhis.analytics.tei.query.DataElementCondition; import org.hisp.dhis.analytics.tei.query.RenderableDataValue; -import org.hisp.dhis.analytics.tei.query.StageExistsRenderable; import org.hisp.dhis.analytics.tei.query.SuffixedRenderableDataValue; import org.hisp.dhis.analytics.tei.query.context.sql.QueryContext; import org.hisp.dhis.analytics.tei.query.context.sql.RenderableSqlQuery; @@ -76,10 +70,10 @@ public class DataElementQueryBuilder implements SqlQueryBuilder { private final List>> headerFilters = - List.of(DataElementQueryBuilder::isDataElement); + List.of(DimensionIdentifierHelper::isDataElement); private final List>> dimensionFilters = - List.of(DataElementQueryBuilder::isDataElement); + List.of(DimensionIdentifierHelper::isDataElement); private final List> sortingFilters = List.of(DataElementQueryBuilder::isDataElementOrder); @@ -104,17 +98,7 @@ public RenderableSqlQuery buildSqlQuery( GroupedDimensions groupedDimensions = getGroupedDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams); - Stream.of( - // Fields holding the value of data elements - getValueFields(groupedDimensions), - // Fields holding the "exists" flag of the stages - getExistsFields(groupedDimensions), - // Fields holding the status of the stages (SCHEDULED, COMPLETE, etc) - getStatusFields(groupedDimensions), - // Fields holding the "hasValue" flag of the data elements - getHasValueFields(groupedDimensions)) - .flatMap(Function.identity()) - .forEach(builder::selectField); + getValueFields(groupedDimensions).forEach(builder::selectField); // Groupable conditions comes from dimensions acceptedDimensions.stream() @@ -122,7 +106,9 @@ public RenderableSqlQuery buildSqlQuery( .map( dimId -> GroupableCondition.of( - dimId.getGroupId(), DataElementCondition.of(queryContext, dimId))) + dimId.getGroupId(), + SqlQueryHelper.buildExistsValueSubquery( + dimId, DataElementCondition.of(queryContext, dimId)))) .forEach(builder::groupableCondition); // Order clause comes from sorting params @@ -132,62 +118,21 @@ public RenderableSqlQuery buildSqlQuery( IndexedOrder.of( analyticsSortingParams.getIndex(), Order.of( - Field.of(analyticsSortingParams.getOrderBy().toString()), + SqlQueryHelper.buildOrderSubQuery( + analyticsSortingParams.getOrderBy(), + RenderableDataValue.of( + EMPTY, + analyticsSortingParams.getOrderBy().getDimension().getUid(), + fromValueType( + analyticsSortingParams + .getOrderBy() + .getDimension() + .getValueType()))), analyticsSortingParams.getSortDirection())))); return builder.build(); } - /** - * Returns the fields holding the "hasValue" flag of the data elements. - * - * @param groupedDimensions the groupedDimensions. - * @return the stream of fields holding the "hasValue" flag of the data elements. - */ - private Stream getHasValueFields(GroupedDimensions groupedDimensions) { - return groupedDimensions - .streamOfFirstDimensionInEachGroup() - .map( - dimensionIdentifier -> - ofUnquoted( - EMPTY, - () -> - doubleQuote(dimensionIdentifier.getPrefix()) - + ".eventdatavalues :: jsonb ?? '" - + dimensionIdentifier.getDimension().getUid() - + "'", - dimensionIdentifier + HAS_VALUE)); - } - - /** - * Returns the fields holding the status of the stages. - * - * @param groupedDimensions the groupedDimensions. - * @return the stream of fields holding the status of the stages. - */ - private Stream getStatusFields(GroupedDimensions groupedDimensions) { - return groupedDimensions - .streamOfFirstDimensionInEachGroup() - .map( - dimensionIdentifier -> - ofUnquoted( - EMPTY, - () -> doubleQuote(dimensionIdentifier.getPrefix()) + STATUS, - dimensionIdentifier.toString() + STATUS)); - } - - /** - * Returns the fields holding the "exists" flag of the stages. - * - * @param groupedDimensions the groupedDimensions. - * @return the stream of fields holding the "exists" flag of the stages. - */ - private Stream getExistsFields(GroupedDimensions groupedDimensions) { - return groupedDimensions - .streamOfFirstDimensionInEachGroup() - .map(dim -> ofUnquoted(EMPTY, StageExistsRenderable.of(dim), dim.getKey() + EXISTS)); - } - /** * Returns the fields holding the value of data elements. * @@ -199,7 +144,8 @@ private static Stream getValueFields(GroupedDimensions groupedDimensions) return groupedDimensions.getGroupsByKey().stream() .map(DimensionGroup::dimensions) .flatMap(Collection::stream) - .map(DataElementQueryBuilder::toField); + .map(DataElementQueryBuilder::toField) + .map(Field::asVirtual); } /** @@ -277,16 +223,6 @@ private static Renderable withMaybeLegendSet( * @return true if the sorting parameter is of type data element, false otherwise. */ private static boolean isDataElementOrder(AnalyticsSortingParams analyticsSortingParams) { - return isDataElement(analyticsSortingParams.getOrderBy()); - } - - /** - * Checks if the given dimension identifier is of type data element. - * - * @param dimensionIdentifier the dimension identifier to check. - * @return true if the dimension identifier is of type data element, false otherwise. - */ - private static boolean isDataElement(DimensionIdentifier dimensionIdentifier) { - return isOfType(dimensionIdentifier, DATA_ELEMENT) && dimensionIdentifier.isEventDimension(); + return DimensionIdentifierHelper.isDataElement(analyticsSortingParams.getOrderBy()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java deleted file mode 100644 index 8fafbfb191f6..000000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/LeftJoinsQueryBuilder.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2004-2023, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.analytics.tei.query.context.querybuilder; - -import static org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifierHelper.DIMENSION_SEPARATOR; -import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.PROGRAM_ATTRIBUTE; -import static org.hisp.dhis.analytics.common.params.dimension.DimensionParamObjectType.PROGRAM_INDICATOR; -import static org.hisp.dhis.analytics.common.query.BinaryConditionRenderer.fieldsEqual; -import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.PI_UID; -import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.TEI_ALIAS; -import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.TEI_UID; -import static org.hisp.dhis.analytics.tei.query.context.querybuilder.SqlQueryHelper.enrollmentSelect; -import static org.hisp.dhis.analytics.tei.query.context.querybuilder.SqlQueryHelper.eventSelect; -import static org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryBuilders.isOfType; -import static org.hisp.dhis.commons.util.TextUtils.doubleQuote; - -import java.util.List; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import org.apache.commons.lang3.tuple.Pair; -import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; -import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; -import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; -import org.hisp.dhis.analytics.common.params.dimension.ElementWithOffset; -import org.hisp.dhis.analytics.common.query.LeftJoin; -import org.hisp.dhis.analytics.tei.query.context.sql.QueryContext; -import org.hisp.dhis.analytics.tei.query.context.sql.RenderableSqlQuery; -import org.hisp.dhis.analytics.tei.query.context.sql.RenderableSqlQuery.RenderableSqlQueryBuilder; -import org.hisp.dhis.analytics.tei.query.context.sql.SqlParameterManager; -import org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryBuilder; -import org.hisp.dhis.program.Program; -import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.trackedentity.TrackedEntityType; -import org.springframework.stereotype.Service; - -/** This class is responsible for building the SQL statement for the main TEI table. */ -@Service -public class LeftJoinsQueryBuilder implements SqlQueryBuilder { - - @Nonnull - @Override - public List>> getDimensionFilters() { - return List.of( - dimensionIdentifier -> - dimensionIdentifier.isEventDimension() || dimensionIdentifier.isEnrollmentDimension(), - dimensionIdentifier -> !isOfType(dimensionIdentifier, PROGRAM_INDICATOR)); - } - - @Nonnull - @Override - public List> getSortingFilters() { - return List.of( - sortingParams -> - sortingParams.getOrderBy().isEventDimension() - || sortingParams.getOrderBy().isEnrollmentDimension(), - sortingParams -> !isOfType(sortingParams.getOrderBy(), PROGRAM_INDICATOR)); - } - - @Override - public RenderableSqlQuery buildSqlQuery( - QueryContext queryContext, - List> headerIdentifiers, - List> dimensionIdentifiers, - List analyticsSortingParams) { - RenderableSqlQueryBuilder renderableSqlQuery = RenderableSqlQuery.builder(); - - List> allDimensions = - streamDimensions(headerIdentifiers, dimensionIdentifiers, analyticsSortingParams) - .filter(dimensionIdentifier -> !isOfType(dimensionIdentifier, PROGRAM_ATTRIBUTE)) - .collect(Collectors.toList()); - - Set> allDeclaredPrograms = - allDimensions.stream().map(DimensionIdentifier::getProgram).collect(Collectors.toSet()); - - Set, ElementWithOffset>> - allDeclaredProgramStages = - allDimensions.stream() - .filter(DimensionIdentifier::isEventDimension) - .map( - dimensionIdentifier -> - Pair.of( - dimensionIdentifier.getProgram(), - dimensionIdentifier.getProgramStage())) - .collect(Collectors.toSet()); - - TrackedEntityType trackedEntityType = queryContext.getTeiQueryParams().getTrackedEntityType(); - SqlParameterManager sqlParameterManager = queryContext.getSqlParameterManager(); - - for (ElementWithOffset program : allDeclaredPrograms) { - String enrollmentAlias = doubleQuote(program.toString()); - renderableSqlQuery.leftJoin( - LeftJoin.of( - () -> - "(" - + enrollmentSelect(program, trackedEntityType, sqlParameterManager) - + ") as " - + enrollmentAlias, - fieldsEqual(TEI_ALIAS, TEI_UID, enrollmentAlias, TEI_UID))); - } - - for (Pair, ElementWithOffset> programStage : - allDeclaredProgramStages) { - String enrollmentAlias = doubleQuote(programStage.getLeft().toString()); - String eventAlias = - doubleQuote( - programStage.getLeft().toString() - + DIMENSION_SEPARATOR - + programStage.getRight().toString()); - renderableSqlQuery.leftJoin( - LeftJoin.of( - () -> - "(" - + eventSelect( - programStage.getLeft(), - programStage.getRight(), - trackedEntityType, - sqlParameterManager) - + ") as " - + eventAlias, - fieldsEqual(enrollmentAlias, PI_UID, eventAlias, PI_UID))); - } - return renderableSqlQuery.build(); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelper.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelper.java new file mode 100644 index 000000000000..898417c7bccc --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelper.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.tei.query.context.querybuilder; + +import static java.lang.Math.abs; +import static lombok.AccessLevel.PRIVATE; + +import java.util.Comparator; +import java.util.Optional; +import java.util.stream.Stream; +import lombok.NoArgsConstructor; + +/** Utility class for handling offsets. */ +@NoArgsConstructor(access = PRIVATE) +public class OffsetHelper { + + /** + * Given a stream of objects, a comparator and an offset, returns the object at the specified + * offset. + * + * @param stream the stream of objects + * @param comparator the comparator to sort the objects + * @param offset the offset + * @param the type of the objects + * @return the object at the specified offset + */ + public static Optional getItemBasedOnOffset( + Stream stream, Comparator comparator, int offset) { + if (offset > 0) { // 1 first (--> skip 0), 2 second (--> skip 1), 3 third (--> skip 2), etc. + return stream + // positive offset means sort by ascending date + .sorted(comparator.reversed()) + .skip(offset - 1L) + .findFirst(); + } + // 0 latest, -1 second latest (--> skip 1), -2 third latest (--> skip 2), etc. + return stream.sorted(comparator).skip(-offset).findFirst(); + } + + /** + * Returns the offset as a string. If the offset is negative, the absolute value is incremented by + * 1. This is due to the fact that the row_number() function in SQL starts at 1. + * + * @param offset the offset + * @return the offset as a string + */ + public static Offset getOffset(Integer offset) { + if (offset > 0) { + return new Offset(String.valueOf(offset), "asc"); + } + // this is due to the fact that the row_number() function in SQL starts at 1 + return new Offset(String.valueOf(abs(offset) + 1), "desc"); + } + + public record Offset(String offset, String direction) {} +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java index 487c1ff75246..18ee9b9258be 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OrgUnitQueryBuilder.java @@ -83,6 +83,7 @@ public RenderableSqlQuery buildSqlQuery( getPrefix(dimensionIdentifier), () -> dimensionIdentifier.getDimension().getUid(), dimensionIdentifier.toString())) + .map(Field::asVirtual) .forEach(builder::selectField); acceptedDimensions.stream() @@ -90,7 +91,9 @@ public RenderableSqlQuery buildSqlQuery( .map( dimId -> GroupableCondition.of( - dimId.getGroupId(), OrganisationUnitCondition.of(dimId, queryContext))) + dimId.getGroupId(), + SqlQueryHelper.buildExistsValueSubquery( + dimId, OrganisationUnitCondition.of(dimId, queryContext)))) .forEach(builder::groupableCondition); acceptedSortingParams.forEach( @@ -99,7 +102,9 @@ public RenderableSqlQuery buildSqlQuery( IndexedOrder.of( sortingParam.getIndex(), Order.of( - Field.ofDimensionIdentifier(sortingParam.getOrderBy()), + SqlQueryHelper.buildOrderSubQuery( + sortingParam.getOrderBy(), + () -> sortingParam.getOrderBy().getDimension().getUid()), sortingParam.getSortDirection())))); return builder.build(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java index 1044640e901a..05eab4c998b8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/PeriodQueryBuilder.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.analytics.tei.query.context.querybuilder; +import static java.util.function.Predicate.not; import static org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifierHelper.DIMENSION_SEPARATOR; import static org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifierHelper.getPrefix; import static org.hisp.dhis.commons.util.TextUtils.doubleQuote; @@ -36,7 +37,6 @@ import java.util.function.Function; import java.util.function.Predicate; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.TimeField; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; @@ -79,23 +79,24 @@ public RenderableSqlQuery buildSqlQuery( RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder(); streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) - .map( - dimensionIdentifier -> { - String field = getTimeField(dimensionIdentifier, StaticDimension::getColumnName); - String alias = getTimeField(dimensionIdentifier, StaticDimension::getHeaderName); - - String prefix = getPrefix(dimensionIdentifier, false); + .filter(DimensionIdentifier::isTeDimension) + .map(PeriodQueryBuilder::asField) + .forEach(builder::selectField); - return Field.ofUnquoted( - doubleQuote(prefix), () -> field, prefix + DIMENSION_SEPARATOR + alias); - }) + streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) + .filter(not(DimensionIdentifier::isTeDimension)) + .map(PeriodQueryBuilder::asField) + // non TEI periods are virtual fields, since those will be extracted from JSON + .map(Field::asVirtual) .forEach(builder::selectField); acceptedDimensions.stream() .map( dimensionIdentifier -> GroupableCondition.of( - getGroupId(dimensionIdentifier), PeriodCondition.of(dimensionIdentifier, ctx))) + getGroupId(dimensionIdentifier), + SqlQueryHelper.buildExistsValueSubquery( + dimensionIdentifier, PeriodCondition.of(dimensionIdentifier, ctx)))) .forEach(builder::groupableCondition); acceptedSortingParams.forEach( @@ -103,12 +104,12 @@ public RenderableSqlQuery buildSqlQuery( DimensionIdentifier dimensionIdentifier = sortingParam.getOrderBy(); String fieldName = getTimeField(dimensionIdentifier, StaticDimension::getColumnName); - Field field = - Field.ofUnquoted( - getPrefix(sortingParam.getOrderBy()), () -> fieldName, StringUtils.EMPTY); builder.orderClause( IndexedOrder.of( - sortingParam.getIndex(), Order.of(field, sortingParam.getSortDirection()))); + sortingParam.getIndex(), + Order.of( + SqlQueryHelper.buildOrderSubQuery(sortingParam.getOrderBy(), () -> fieldName), + sortingParam.getSortDirection()))); }); return builder.build(); @@ -125,6 +126,15 @@ private String getGroupId(DimensionIdentifier dimensionIdentifie return dimensionIdentifier.getGroupId() + ":" + getTimeField(dimensionIdentifier, Enum::name); } + private static Field asField(DimensionIdentifier dimensionIdentifier) { + String field = getTimeField(dimensionIdentifier, StaticDimension::getColumnName); + String alias = getTimeField(dimensionIdentifier, StaticDimension::getHeaderName); + + String prefix = getPrefix(dimensionIdentifier, false); + + return Field.ofUnquoted(doubleQuote(prefix), () -> field, prefix + DIMENSION_SEPARATOR + alias); + } + /** * Extracts the time field from the dimension identifier. If the dimension identifier is a period * dimension, the time field is extracted from the period object. If the dimension identifier is a diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelper.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelper.java index fbb68910fe76..5b5f93395e81 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelper.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelper.java @@ -27,20 +27,20 @@ */ package org.hisp.dhis.analytics.tei.query.context.querybuilder; -import static java.lang.Math.abs; import static lombok.AccessLevel.PRIVATE; -import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.ANALYTICS_TEI_ENR; -import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.ANALYTICS_TEI_EVT; -import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.PS_UID; -import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.P_UID; +import static org.apache.commons.text.StringSubstitutor.replace; +import static org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifierHelper.isDataElement; +import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.TEI_ALIAS; +import static org.hisp.dhis.common.collection.CollectionUtils.merge; +import java.util.Map; import lombok.NoArgsConstructor; -import org.hisp.dhis.analytics.common.params.dimension.ElementWithOffset; -import org.hisp.dhis.analytics.tei.query.context.sql.SqlParameterManager; -import org.hisp.dhis.event.EventStatus; -import org.hisp.dhis.program.Program; -import org.hisp.dhis.program.ProgramStage; -import org.hisp.dhis.trackedentity.TrackedEntityType; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; +import org.hisp.dhis.analytics.common.query.Field; +import org.hisp.dhis.analytics.common.query.Renderable; +import org.hisp.dhis.analytics.tei.query.context.querybuilder.OffsetHelper.Offset; /** * Helper class that contains methods used along with query generation. It's mainly referenced in @@ -48,61 +48,213 @@ */ @NoArgsConstructor(access = PRIVATE) class SqlQueryHelper { - static String enrollmentSelect( - ElementWithOffset program, - TrackedEntityType trackedEntityType, - SqlParameterManager sqlParameterManager) { - int offset = program.getOffsetWithDefault(); - - return "select innermost_enr.*" - + " from (select *," - + " row_number() over (partition by trackedentityinstanceuid order by enrollmentdate " - + (offset > 0 ? "asc" : "desc") - + ") as rn " - + " from " - + ANALYTICS_TEI_ENR - + trackedEntityType.getUid().toLowerCase() - + " where " - + P_UID - + " = " - + sqlParameterManager.bindParamAndGetIndex(program.getElement().getUid()) - + ") innermost_enr" - + " where innermost_enr.rn = " - // This logic is needed because of the row_number(), which starts in 1. - + (offset > 0 ? offset : abs(offset) + 1); + + private static final String ENROLLMENT_ORDER_BY_SUBQUERY = + """ + (select ${selectedEnrollmentField} + from (select *, + row_number() over ( partition by trackedentityinstanceuid + order by enrollmentdate ${programOffsetDirection} ) as rn + from analytics_tei_enrollments_${trackedEntityTypeUid} + where programuid = '${programUid}' + and t_1.trackedentityinstanceuid = trackedentityinstanceuid) en + where en.rn = ${programOffset})"""; + + private static final String EVENT_ORDER_BY_SUBQUERY = + """ + (select ${selectedEventField} + from (select *, + row_number() over ( partition by programinstanceuid + order by occurreddate ${programStageOffsetDirection} ) as rn + from analytics_tei_events_${trackedEntityTypeUid} events + where programstageuid = '${programStageUid}' + and programinstanceuid = %s + and status != 'SCHEDULE') ev + where ev.rn = ${programStageOffset})""" + .formatted(ENROLLMENT_ORDER_BY_SUBQUERY); + + private static final String DATA_VALUES_ORDER_BY_SUBQUERY = + """ + (select ${dataElementField} + from analytics_tei_events_${trackedEntityTypeUid} + where programstageinstanceuid = %s)""" + .formatted(EVENT_ORDER_BY_SUBQUERY); + + private static final String ENROLLMENT_EXISTS_SUBQUERY = + """ + exists(select 1 + from (select * + from (select *, row_number() over (partition by trackedentityinstanceuid order by enrollmentdate ${programOffsetDirection}) as rn + from analytics_tei_enrollments_${trackedEntityTypeUid} + where programuid = '${programUid}' + and trackedentityinstanceuid = t_1.trackedentityinstanceuid) en + where en.rn = 1) as "${enrollmentSubqueryAlias}" + where ${enrollmentCondition})"""; + + private static final String EVENT_EXISTS_SUBQUERY = + replace( + ENROLLMENT_EXISTS_SUBQUERY, + Map.of( + "enrollmentCondition", + """ + exists(select 1 + from (select * + from (select *, row_number() over ( partition by programinstanceuid order by occurreddate ${programStageOffsetDirection} ) as rn + from analytics_tei_events_${trackedEntityTypeUid} + where "${enrollmentSubqueryAlias}".programinstanceuid = programinstanceuid + and programstageuid = '${programStageUid}' + and status != 'SCHEDULE') ev + where ev.rn = 1) as "${eventSubqueryAlias}" + where ${eventCondition})""")); + + private static final String DATA_VALUES_EXISTS_SUBQUERY = + replace( + EVENT_EXISTS_SUBQUERY, + Map.of( + "eventCondition", + """ + exists(select 1 + from analytics_tei_events_${trackedEntityTypeUid} + where "${eventSubqueryAlias}".programstageinstanceuid = programstageinstanceuid + and ${eventDataValueCondition})""")); + + /** + * Builds the order by sub-query for the given dimension identifier and field. + * + * @param dimId the dimension identifier + * @param field the renderable field on which to eventually sort by + * @return the renderable order by sub-query + */ + static Renderable buildOrderSubQuery( + DimensionIdentifier dimId, Renderable field) { + if (isDataElement(dimId)) { + return () -> + replace( + DATA_VALUES_ORDER_BY_SUBQUERY, + merge( + getEnrollmentPlaceholders(dimId), + getEventPlaceholders(dimId), + Map.of( + "selectedEnrollmentField", "programInstanceUid", + "selectedEventField", "programStageInstanceUid", + "dataElementField", field.render()))); + } + if (dimId.isEventDimension() && !isDataElement(dimId)) { + return () -> + replace( + EVENT_ORDER_BY_SUBQUERY, + merge( + getEnrollmentPlaceholders(dimId), + getEventPlaceholders(dimId), + Map.of( + "selectedEnrollmentField", + "programInstanceUid", + "selectedEventField", + field.render()))); + } + if (dimId.isEnrollmentDimension()) { + return () -> + replace( + ENROLLMENT_ORDER_BY_SUBQUERY, + merge( + getEnrollmentPlaceholders(dimId), + Map.of("selectedEnrollmentField", field.render()))); + } + if (dimId.isTeDimension()) { + return Field.of(TEI_ALIAS, field, StringUtils.EMPTY); + } + throw new IllegalArgumentException("Unsupported dimension type: " + dimId); + } + + /** + * Builds the exists value sub-query for the given dimension identifier and condition. + * + * @param dimId the dimension identifier + * @param condition the condition to apply + * @return the renderable exists value sub-query + */ + public static Renderable buildExistsValueSubquery( + DimensionIdentifier dimId, Renderable condition) { + if (isDataElement(dimId)) { + return () -> + replace( + DATA_VALUES_EXISTS_SUBQUERY, + merge( + getEnrollmentPlaceholders(dimId), + getEventPlaceholders(dimId), + Map.of( + "eventSubqueryAlias", dimId.getPrefix(), + "enrollmentSubqueryAlias", "enrollmentSubqueryAlias", + "eventDataValueCondition", condition.render()))); + } + if (dimId.isEventDimension() && !isDataElement(dimId)) { + return () -> + replace( + EVENT_EXISTS_SUBQUERY, + merge( + getEnrollmentPlaceholders(dimId), + getEventPlaceholders(dimId), + Map.of( + "eventSubqueryAlias", dimId.getPrefix(), + "enrollmentSubqueryAlias", "enrollmentSubqueryAlias", + "eventCondition", condition.render()))); + } + if (dimId.isEnrollmentDimension()) { + return () -> + replace( + ENROLLMENT_EXISTS_SUBQUERY, + merge( + getEnrollmentPlaceholders(dimId), + Map.of( + "enrollmentSubqueryAlias", dimId.getPrefix(), + "enrollmentCondition", condition.render()))); + } + if (dimId.isTeDimension()) { + return condition; + } + throw new IllegalArgumentException("Unsupported dimension type: " + dimId); + } + + /** + * Returns the placeholders for the event. + * + * @param dimId the dimension identifier + * @return the placeholders + */ + private static Map getEventPlaceholders( + DimensionIdentifier dimId) { + + String programStageUid = dimId.getProgramStage().getElement().getUid(); + Offset programStageOffset = + OffsetHelper.getOffset(dimId.getProgramStage().getOffsetWithDefault()); + + return Map.of( + "programStageUid", + programStageUid, + "programStageOffset", + programStageOffset.offset(), + "programStageOffsetDirection", + programStageOffset.direction()); } - static String eventSelect( - ElementWithOffset program, - ElementWithOffset programStage, - TrackedEntityType trackedEntityType, - SqlParameterManager sqlParameterManager) { - int offset = programStage.getOffsetWithDefault(); - String orderByDirection = offset > 0 ? "asc" : "desc"; - - return "select innermost_evt.*" - + " from (select *," - + " row_number() over (partition by programinstanceuid order by occurreddate " - + orderByDirection - + ", created " - + orderByDirection - + " ) as rn" - + " from " - + ANALYTICS_TEI_EVT - + trackedEntityType.getUid().toLowerCase() - + " where status != '" - + EventStatus.SCHEDULE - + "' and " - + P_UID - + " = " - + sqlParameterManager.bindParamAndGetIndex(program.getElement().getUid()) - + " and " - + PS_UID - + " = " - + sqlParameterManager.bindParamAndGetIndex(programStage.getElement().getUid()) - + ") innermost_evt" - + " where innermost_evt.rn = " - // This logic is needed because of the row_number(), which starts in 1. - + (offset > 0 ? offset : abs(offset) + 1); + /** + * Returns the placeholders for the enrollment. + * + * @param dimId the dimension identifier + * @return the placeholders + */ + private static Map getEnrollmentPlaceholders( + DimensionIdentifier dimId) { + + String trackedEntityTypeUid = dimId.getProgram().getElement().getTrackedEntityType().getUid(); + + String programUid = dimId.getProgram().getElement().getUid(); + Offset programOffset = OffsetHelper.getOffset(dimId.getProgram().getOffsetWithDefault()); + + return Map.of( + "trackedEntityTypeUid", StringUtils.lowerCase(trackedEntityTypeUid), + "programUid", programUid, + "programOffset", programOffset.offset(), + "programOffsetDirection", programOffset.direction()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java index efeaf87d7c39..3c0bb518475a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/StatusQueryBuilder.java @@ -38,7 +38,6 @@ import java.util.Optional; import java.util.function.Predicate; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.common.params.AnalyticsSortingParams; import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; @@ -86,6 +85,7 @@ public RenderableSqlQuery buildSqlQuery( RenderableSqlQuery.RenderableSqlQueryBuilder builder = RenderableSqlQuery.builder(); streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) + .filter(DimensionIdentifier::isTeDimension) .map( dimensionIdentifier -> { StaticDimension staticDimension = @@ -99,12 +99,31 @@ public RenderableSqlQuery buildSqlQuery( }) .forEach(builder::selectField); + streamDimensions(acceptedHeaders, acceptedDimensions, acceptedSortingParams) + .filter(Predicate.not(DimensionIdentifier::isTeDimension)) + .map( + dimensionIdentifier -> { + StaticDimension staticDimension = + dimensionIdentifier.getDimension().getStaticDimension(); + String prefix = getPrefix(dimensionIdentifier, false); + + return Field.ofUnquoted( + doubleQuote(prefix), + staticDimension::getColumnName, + prefix + DIMENSION_SEPARATOR + staticDimension.getHeaderName()); + }) + // Fields that are not TE specific, are Virtual since they will be extracted from the JSON + .map(Field::asVirtual) + .forEach(builder::selectField); + acceptedDimensions.stream() .filter(SqlQueryBuilders::hasRestrictions) .map( dimensionIdentifier -> GroupableCondition.of( - dimensionIdentifier.getGroupId(), StatusCondition.of(dimensionIdentifier, ctx))) + dimensionIdentifier.getGroupId(), + SqlQueryHelper.buildExistsValueSubquery( + dimensionIdentifier, StatusCondition.of(dimensionIdentifier, ctx)))) .forEach(builder::groupableCondition); acceptedSortingParams.forEach( @@ -113,12 +132,12 @@ public RenderableSqlQuery buildSqlQuery( String fieldName = dimensionIdentifier.getDimension().getStaticDimension().getColumnName(); - Field field = - Field.ofUnquoted( - getPrefix(sortingParam.getOrderBy()), () -> fieldName, StringUtils.EMPTY); builder.orderClause( IndexedOrder.of( - sortingParam.getIndex(), Order.of(field, sortingParam.getSortDirection()))); + sortingParam.getIndex(), + Order.of( + SqlQueryHelper.buildOrderSubQuery(sortingParam.getOrderBy(), () -> fieldName), + sortingParam.getSortDirection()))); }); return builder.build(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/TeiQueryBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/TeiQueryBuilder.java index 4cf1a902bad6..4166229018d0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/TeiQueryBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/TeiQueryBuilder.java @@ -32,6 +32,7 @@ import static org.hisp.dhis.analytics.tei.query.context.QueryContextConstants.TEI_ALIAS; import static org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryBuilders.hasRestrictions; import static org.hisp.dhis.analytics.tei.query.context.sql.SqlQueryBuilders.isOfType; +import static org.hisp.dhis.commons.util.TextUtils.EMPTY; import java.util.List; import java.util.function.Function; @@ -64,6 +65,37 @@ @RequiredArgsConstructor @org.springframework.core.annotation.Order(1) public class TeiQueryBuilder extends SqlQueryBuilderAdaptor { + private static final String JSON_AGGREGATION_QUERY = + """ + coalesce( (select json_agg(json_build_object('programUid', pr.uid, + 'enrollmentUid', en.programinstanceuid, + 'enrollmentDate', en.enrollmentdate, + 'incidentDate', en.incidentdate, + 'endDate', en.enddate, + 'orgUnitUid', en.ou, + 'orgUnitName', en.ouname, + 'orgUnitCode', en.oucode, + 'orgUnitNameHierarchy', en.ounamehierarchy, + 'enrollmentStatus', en.enrollmentstatus, + 'events', + coalesce( (select json_agg(json_build_object('programStageUid', ps.uid, + 'eventUid', ev.programstageuid, + 'occurredDate', ev.occurreddate, + 'dueDate', ev.scheduleddate, + 'orgUnitUid', ev.ou, + 'orgUnitName', ev.ouname, + 'orgUnitCode', ev.oucode, + 'orgUnitNameHierarchy', ev.ounamehierarchy, + 'eventStatus', ev.status, + 'eventDataValues', ev.eventdatavalues)) + from analytics_tei_events_%s ev, + programstage ps + where ev.programinstanceuid = en.programinstanceuid + and ps.uid = ev.programstageuid), '[]'::json))) + from analytics_tei_enrollments_%s en, + program pr + where en.trackedentityinstanceuid = t_1.trackedentityinstanceuid + and pr.uid = en.programuid), '[]'::json)"""; private final IdentifiableObjectManager identifiableObjectManager; @@ -88,6 +120,13 @@ public boolean alwaysRun() { @Override protected Stream getSelect(QueryContext queryContext) { + String aggregationQuery = + JSON_AGGREGATION_QUERY.formatted( + queryContext.getTetTableSuffix(), queryContext.getTetTableSuffix()); + + Field aggregatedEnrollments = + Field.ofUnquoted(EMPTY, () -> aggregationQuery, "enrollments").withUsedInHeaders(false); + return Stream.of( // Organisation unit group set columns. identifiableObjectManager @@ -95,12 +134,12 @@ protected Stream getSelect(QueryContext queryContext) { .stream() .map(OrganisationUnitGroupSet::getUid) .map(attr -> Field.of(TEI_ALIAS, () -> attr, attr)), - // Static fields column. TeiFields.getStaticFields(), - // Tei/Program attributes. - TeiFields.getDimensionFields(queryContext.getTeiQueryParams())) + TeiFields.getDimensionFields(queryContext.getTeiQueryParams()), + // Enrollments + Stream.of(aggregatedEnrollments)) .flatMap(Function.identity()); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/RenderableSqlQuery.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/RenderableSqlQuery.java index b235c0a5931a..9944c5efc481 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/RenderableSqlQuery.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/RenderableSqlQuery.java @@ -137,7 +137,11 @@ private String from() { } private String select() { - return getIfPresentOrElse(SELECT, () -> Select.of(selectFields).render()); + return getIfPresentOrElse(SELECT, () -> Select.of(nonVirtualSelectFields()).render()); + } + + private List nonVirtualSelectFields() { + return selectFields.stream().filter(f -> !f.isVirtual()).toList(); } private String getIfPresentOrElse(String key, Supplier supplier) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreator.java index e7815ce88cf1..2f099e2edf53 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/tei/query/context/sql/SqlQueryCreator.java @@ -29,6 +29,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.hisp.dhis.analytics.common.QueryCreator; import org.hisp.dhis.analytics.common.SqlQuery; /** @@ -36,18 +37,18 @@ * QueryContext} to get the parameter placeholders. Supports both select and count queries. A select * query can be converted to a count query by calling {@link #createForCount()}. */ +@Getter @RequiredArgsConstructor(staticName = "of") -public class SqlQueryCreator { - private final QueryContext queryContext; +public class SqlQueryCreator implements QueryCreator { - @Getter private final RenderableSqlQuery renderableSqlQuery; + private final QueryContext queryContext; + private final RenderableSqlQuery renderableSqlQuery; public SqlQuery createForSelect() { - return new SqlQuery(renderableSqlQuery.render(), queryContext.getParametersPlaceHolder()); + return SqlQuery.of(renderableSqlQuery.render(), queryContext); } public SqlQuery createForCount() { - return new SqlQuery( - renderableSqlQuery.forCount().render(), queryContext.getParametersPlaceHolder()); + return SqlQuery.of(renderableSqlQuery.forCount().render(), queryContext); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/GridAdaptorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/GridAdaptorTest.java index f31be111ffb0..536d235c10d2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/GridAdaptorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/GridAdaptorTest.java @@ -48,6 +48,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import javax.sql.rowset.RowSetMetaDataImpl; @@ -60,6 +61,7 @@ import org.hisp.dhis.analytics.common.processing.HeaderParamsHandler; import org.hisp.dhis.analytics.common.processing.MetadataParamsHandler; import org.hisp.dhis.analytics.common.query.Field; +import org.hisp.dhis.analytics.common.query.jsonextractor.SqlRowSetJsonExtractorDelegator; import org.hisp.dhis.analytics.data.handler.SchemeIdResponseMapper; import org.hisp.dhis.analytics.tei.TeiQueryParams; import org.hisp.dhis.common.BaseDimensionalItemObject; @@ -108,7 +110,7 @@ void testCreateGridWithFields() throws SQLException { RowSetMetaDataImpl metaData = new RowSetMetaDataImpl(); metaData.setColumnCount(2); metaData.setColumnName(1, "anyFakeCol-1"); - metaData.setColumnName(2, "anyFakeCol-2"); + metaData.setColumnName(2, "oucode"); TeiQueryParams teiQueryParams = TeiQueryParams.builder() @@ -122,7 +124,8 @@ void testCreateGridWithFields() throws SQLException { when(resultSet.getMetaData()).thenReturn(metaData); SqlRowSet sqlRowSet = new ResultSetWrappingSqlRowSet(resultSet); - SqlQueryResult mockSqlResult = new SqlQueryResult(sqlRowSet); + SqlQueryResult mockSqlResult = + new SqlQueryResult(new SqlRowSetJsonExtractorDelegator(sqlRowSet, Collections.emptyList())); long anyCount = 0; // When @@ -159,7 +162,8 @@ void testCreateGridWithEmptyField() throws SQLException { when(resultSet.getMetaData()).thenReturn(metaData); SqlRowSet sqlRowSet = new ResultSetWrappingSqlRowSet(resultSet); - SqlQueryResult mockSqlResult = new SqlQueryResult(sqlRowSet); + SqlQueryResult mockSqlResult = + new SqlQueryResult(new SqlRowSetJsonExtractorDelegator(sqlRowSet, Collections.emptyList())); long anyCount = 0; // When diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtilsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtilsTest.java new file mode 100644 index 000000000000..14b6e4232e33 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/jsonextractor/JsonExtractorUtilsTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.common.query.jsonextractor; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.LocalDateTime; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class JsonExtractorUtilsTest { + + private final LocalDateTime aDate = LocalDateTime.of(2022, 1, 1, 0, 0, 0); + + @ParameterizedTest + @MethodSource("provideDataForTest") + void testGetFormattedDate(Integer millis, String expected) { + LocalDateTime testedDate = aDate.withNano(millis * 1000 * 1000); + assertEquals(expected, JsonExtractorUtils.getFormattedDate(testedDate)); + } + + private static Stream provideDataForTest() { + return Stream.of( + Arguments.of(100, "2022-01-01 00:00:00.1"), + Arguments.of(110, "2022-01-01 00:00:00.11"), + Arguments.of(111, "2022-01-01 00:00:00.111"), + Arguments.of(0, "2022-01-01 00:00:00.0")); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/RenderableDataValueTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/RenderableDataValueTest.java index 140b93cf437f..e9431fc44098 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/RenderableDataValueTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/RenderableDataValueTest.java @@ -53,12 +53,7 @@ void testRenderBoolean() { RenderableDataValue renderableDataValue = RenderableDataValue.of("alias", "dataValue", ValueTypeMapping.BOOLEAN); String result = renderableDataValue.transformedIfNecessary().render(); - assertEquals( - "case when" - + " (alias.\"eventdatavalues\" -> 'dataValue' ->> 'value')::BOOLEAN = 'true' then 1" - + " when (alias.\"eventdatavalues\" -> 'dataValue' ->> 'value')::BOOLEAN = 'false' then 0" - + " end", - result); + assertEquals("0", result); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/TeiSqlQueryTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/TeiSqlQueryTest.java index 0fe0fceeb651..e2952f55e8fb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/TeiSqlQueryTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/TeiSqlQueryTest.java @@ -61,6 +61,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.program.ProgramStage; +import org.hisp.dhis.trackedentity.TrackedEntityType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -111,10 +112,14 @@ void testSqlQueryRenderingWithCommonDimensionalObject() { // when DimensionalObject dimensionalObject = new BaseDimensionalObject("abc"); + TrackedEntityType trackedEntityType = createTrackedEntityType('A'); + Program program = createProgram('A'); + program.setTrackedEntityType(trackedEntityType); + TeiQueryParams teiQueryParams = TeiQueryParams.builder() - .trackedEntityType(createTrackedEntityType('A')) - .commonParams(stubSortingCommonParams(createProgram('A'), 1, dimensionalObject)) + .trackedEntityType(trackedEntityType) + .commonParams(stubSortingCommonParams(program, 1, dimensionalObject)) .build(); // when @@ -122,10 +127,7 @@ void testSqlQueryRenderingWithCommonDimensionalObject() { sqlQueryCreatorService.getSqlQueryCreator(teiQueryParams).createForSelect().getStatement(); // then - assertTrue(sql.contains(" order by \"prabcdefghA[1].pgabcdefghS[1].abc\" desc nulls last")); - assertTrue( - sql.contains( - "(\"prabcdefghA[1].pgabcdefghS[1]\".\"eventdatavalues\" -> 'abc' ->> 'value')::TEXT as \"prabcdefghA[1].pgabcdefghS[1].abc\"")); + assertTrue(sql.contains(" order by (select (\"eventdatavalues\" -> 'abc' ->> 'value')::TEXT")); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilderTest.java index 636ffb551ffc..7cf6d8c33d0a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilderTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/DataElementQueryBuilderTest.java @@ -81,6 +81,6 @@ void testBuildSqlQuery() { // 3. The "exists" field // 4. the "status" field // 5. the "hasValue" field - assertEquals(5, renderableSqlQuery.getSelectFields().size()); + assertEquals(2, renderableSqlQuery.getSelectFields().size()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelperTest.java new file mode 100644 index 000000000000..82166c513f41 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/OffsetHelperTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.tei.query.context.querybuilder; + +import java.util.Comparator; +import java.util.Optional; +import java.util.stream.Stream; +import org.hisp.dhis.analytics.tei.query.context.querybuilder.OffsetHelper.Offset; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +/** Tests for {@link OffsetHelper}. */ +class OffsetHelperTest { + + @ParameterizedTest(name = "testGetItemBasedOnOffset - {index}") + @CsvSource({"2,d", "1,e", "0,a", "-1,b", "-2,c"}) + void testGetItemBasedOnOffset(String offsetParam, String expectedResponse) { + // Given + Stream stream = Stream.of("a", "b", "c", "d", "e"); + Comparator comparator = Comparator.naturalOrder(); + int offset = Integer.parseInt(offsetParam); + + // When + Optional result = OffsetHelper.getItemBasedOnOffset(stream, comparator, offset); + + // Then + Assertions.assertTrue(result.isPresent()); + Assertions.assertEquals(expectedResponse, result.get()); + } + + @ParameterizedTest + @CsvSource({"1,1,asc", "2,2,asc", "0,1,desc", "-1,2,desc", "-2,3,desc"}) + void testGetOffset(String offsetParam, String expectedOffset, String expectedDirection) { + // When + Offset offset = OffsetHelper.getOffset(Integer.parseInt(offsetParam)); + + // Then + Assertions.assertEquals(expectedOffset, offset.offset()); + Assertions.assertEquals(expectedDirection, offset.direction()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelperTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelperTest.java index 68b7c81e8101..9cefb24f1617 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelperTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/tei/query/context/querybuilder/SqlQueryHelperTest.java @@ -27,10 +27,17 @@ */ package org.hisp.dhis.analytics.tei.query.context.querybuilder; -import static org.junit.jupiter.api.Assertions.*; - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.function.Consumer; +import org.hisp.dhis.analytics.common.params.dimension.DimensionIdentifier; +import org.hisp.dhis.analytics.common.params.dimension.DimensionParam; import org.hisp.dhis.analytics.common.params.dimension.ElementWithOffset; -import org.hisp.dhis.analytics.tei.query.context.sql.SqlParameterManager; +import org.hisp.dhis.common.UidObject; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; import org.hisp.dhis.trackedentity.TrackedEntityType; @@ -40,170 +47,231 @@ class SqlQueryHelperTest { @Test - void testEnrollmentSelectPositiveOffset() { - // Given - int positiveOffset = 1; - - Program program = new Program(); - program.setUid("uid1"); - - ElementWithOffset programElement = ElementWithOffset.of(program, positiveOffset); - - TrackedEntityType trackedEntityType = new TrackedEntityType(); - trackedEntityType.setUid("uid2"); - - SqlParameterManager sqlParameterManager = new SqlParameterManager(); - - // When - String statement = - SqlQueryHelper.enrollmentSelect(programElement, trackedEntityType, sqlParameterManager); - - // Then - assertEquals( - "select innermost_enr.* from (select *, row_number() over (partition by trackedentityinstanceuid order by enrollmentdate asc) as rn from analytics_tei_enrollments_uid2 where programuid = :1) innermost_enr where innermost_enr.rn = 1", - statement); + void test_throws_when_undetected_type() { + DimensionParam dimensionParam = mock(DimensionParam.class); + DimensionIdentifier testedDimension = mock(DimensionIdentifier.class); + when(testedDimension.getDimension()).thenReturn(dimensionParam); + + assertThrows( + IllegalArgumentException.class, + () -> SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field")); + + assertThrows( + IllegalArgumentException.class, + () -> SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field")); } @Test - void testEnrollmentSelectNegativeOffset() { - // Given - int negativeOffset = -1; - - Program program = new Program(); - program.setUid("uid1"); - - ElementWithOffset programElement = ElementWithOffset.of(program, negativeOffset); + void test_subQuery_TE() { + DimensionParam dimensionParam = mock(DimensionParam.class); + DimensionIdentifier testedDimension = mock(DimensionIdentifier.class); + when(testedDimension.getDimension()).thenReturn(dimensionParam); - TrackedEntityType trackedEntityType = new TrackedEntityType(); - trackedEntityType.setUid("uid2"); + when(testedDimension.isTeDimension()).thenReturn(true); - SqlParameterManager sqlParameterManager = new SqlParameterManager(); - - // When - String statement = - SqlQueryHelper.enrollmentSelect(programElement, trackedEntityType, sqlParameterManager); + assertEquals( + "t_1.\"field\"", + SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field").render()); - // Then assertEquals( - "select innermost_enr.* from (select *, row_number() over (partition by trackedentityinstanceuid order by enrollmentdate desc) as rn from analytics_tei_enrollments_uid2 where programuid = :1) innermost_enr where innermost_enr.rn = 2", - statement); + "field", SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field").render()); } @Test - void testEnrollmentSelectZeroOffset() { - // Given - int zeroOffset = 0; + void test_subQuery_enrollment() { + DimensionParam dimensionParam = mock(DimensionParam.class); + DimensionIdentifier testedDimension = mock(DimensionIdentifier.class); + when(testedDimension.getDimension()).thenReturn(dimensionParam); + when(testedDimension.getPrefix()).thenReturn("prefix"); - Program program = new Program(); - program.setUid("uid1"); + TrackedEntityType trackedEntityType = mock(TrackedEntityType.class); + when(trackedEntityType.getUid()).thenReturn("trackedEntityType"); - ElementWithOffset programElement = ElementWithOffset.of(program, zeroOffset); + ElementWithOffset program = + mockElementWithOffset( + Program.class, + "programUid", + p -> when(p.getTrackedEntityType()).thenReturn(trackedEntityType)); - TrackedEntityType trackedEntityType = new TrackedEntityType(); - trackedEntityType.setUid("uid2"); + when(testedDimension.getProgram()).thenReturn(program); - SqlParameterManager sqlParameterManager = new SqlParameterManager(); + when(testedDimension.isEnrollmentDimension()).thenReturn(true); - // When - String statement = - SqlQueryHelper.enrollmentSelect(programElement, trackedEntityType, sqlParameterManager); + assertEquals( + """ + (select field + from (select *, + row_number() over ( partition by trackedentityinstanceuid + order by enrollmentdate desc ) as rn + from analytics_tei_enrollments_trackedentitytype + where programuid = 'programUid' + and t_1.trackedentityinstanceuid = trackedentityinstanceuid) en + where en.rn = 1)""", + SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field").render()); - // Then assertEquals( - "select innermost_enr.* from (select *, row_number() over (partition by trackedentityinstanceuid order by enrollmentdate desc) as rn from analytics_tei_enrollments_uid2 where programuid = :1) innermost_enr where innermost_enr.rn = 1", - statement); + """ + exists(select 1 + from (select * + from (select *, row_number() over (partition by trackedentityinstanceuid order by enrollmentdate desc) as rn + from analytics_tei_enrollments_trackedentitytype + where programuid = 'programUid' + and trackedentityinstanceuid = t_1.trackedentityinstanceuid) en + where en.rn = 1) as "prefix" + where field)""", + SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field").render()); } @Test - void testEventSelectZeroOffset() { - // Given - int zeroOffset = 0; + void test_subQuery_event() { + DimensionParam dimensionParam = mock(DimensionParam.class); + DimensionIdentifier testedDimension = mock(DimensionIdentifier.class); + when(testedDimension.getDimension()).thenReturn(dimensionParam); + when(testedDimension.getPrefix()).thenReturn("prefix"); - Program program = new Program(); - program.setUid("uid1"); + TrackedEntityType trackedEntityType = mock(TrackedEntityType.class); + when(trackedEntityType.getUid()).thenReturn("trackedEntityType"); - ProgramStage programStage = new ProgramStage(); - programStage.setProgram(program); + ElementWithOffset program = + mockElementWithOffset( + Program.class, + "programUid", + p -> when(p.getTrackedEntityType()).thenReturn(trackedEntityType)); - ElementWithOffset programElement = ElementWithOffset.of(program, zeroOffset); - ElementWithOffset programStageElement = - ElementWithOffset.of(programStage, zeroOffset); + ElementWithOffset programStage = + mockElementWithOffset(ProgramStage.class, "programStageUid"); - TrackedEntityType trackedEntityType = new TrackedEntityType(); - trackedEntityType.setUid("uid2"); + when(testedDimension.getProgram()).thenReturn(program); + when(testedDimension.getProgramStage()).thenReturn(programStage); - SqlParameterManager sqlParameterManager = new SqlParameterManager(); + when(testedDimension.isEventDimension()).thenReturn(true); - // When - String statement = - SqlQueryHelper.eventSelect( - programElement, programStageElement, trackedEntityType, sqlParameterManager); + assertEquals( + """ + (select field + from (select *, + row_number() over ( partition by programinstanceuid + order by occurreddate desc ) as rn + from analytics_tei_events_trackedentitytype events + where programstageuid = 'programStageUid' + and programinstanceuid = (select programInstanceUid + from (select *, + row_number() over ( partition by trackedentityinstanceuid + order by enrollmentdate desc ) as rn + from analytics_tei_enrollments_trackedentitytype + where programuid = 'programUid' + and t_1.trackedentityinstanceuid = trackedentityinstanceuid) en + where en.rn = 1) + and status != 'SCHEDULE') ev + where ev.rn = 1)""", + SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field").render()); - // Then assertEquals( - "select innermost_evt.* from (select *, row_number() over (partition by programinstanceuid order by occurreddate desc, created desc ) as rn from analytics_tei_events_uid2 where status != 'SCHEDULE' and programuid = :1 and programstageuid = :2) innermost_evt where innermost_evt.rn = 1", - statement); + """ + exists(select 1 + from (select * + from (select *, row_number() over (partition by trackedentityinstanceuid order by enrollmentdate desc) as rn + from analytics_tei_enrollments_trackedentitytype + where programuid = 'programUid' + and trackedentityinstanceuid = t_1.trackedentityinstanceuid) en + where en.rn = 1) as "enrollmentSubqueryAlias" + where exists(select 1 + from (select * + from (select *, row_number() over ( partition by programinstanceuid order by occurreddate desc ) as rn + from analytics_tei_events_trackedentitytype + where "enrollmentSubqueryAlias".programinstanceuid = programinstanceuid + and programstageuid = 'programStageUid' + and status != 'SCHEDULE') ev + where ev.rn = 1) as "prefix" + where field))""", + SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field").render()); } @Test - void testEventSelectPositiveOffset() { - // Given - int positiveOffset = 2; - - Program program = new Program(); - program.setUid("uid1"); + void test_subQuery_data_element() { + DimensionParam dimensionParam = mock(DimensionParam.class); - ProgramStage programStage = new ProgramStage(); - programStage.setProgram(program); + when(dimensionParam.isOfType(any())).thenReturn(true); - ElementWithOffset programElement = ElementWithOffset.of(program, positiveOffset); - ElementWithOffset programStageElement = - ElementWithOffset.of(programStage, positiveOffset); + DimensionIdentifier testedDimension = mock(DimensionIdentifier.class); + when(testedDimension.getDimension()).thenReturn(dimensionParam); + when(testedDimension.getPrefix()).thenReturn("prefix"); - TrackedEntityType trackedEntityType = new TrackedEntityType(); - trackedEntityType.setUid("uid2"); + TrackedEntityType trackedEntityType = mock(TrackedEntityType.class); + when(trackedEntityType.getUid()).thenReturn("trackedEntityType"); - SqlParameterManager sqlParameterManager = new SqlParameterManager(); + ElementWithOffset program = + mockElementWithOffset( + Program.class, + "programUid", + p -> when(p.getTrackedEntityType()).thenReturn(trackedEntityType)); - // When - String statement = - SqlQueryHelper.eventSelect( - programElement, programStageElement, trackedEntityType, sqlParameterManager); - - // Then - assertEquals( - "select innermost_evt.* from (select *, row_number() over (partition by programinstanceuid order by occurreddate asc, created asc ) as rn from analytics_tei_events_uid2 where status != 'SCHEDULE' and programuid = :1 and programstageuid = :2) innermost_evt where innermost_evt.rn = 2", - statement); - } - - @Test - void testEventSelectNegativeOffset() { - // Given - int positiveOffset = -3; + ElementWithOffset programStage = + mockElementWithOffset(ProgramStage.class, "programStageUid"); - Program program = new Program(); - program.setUid("uid1"); + when(testedDimension.getProgram()).thenReturn(program); + when(testedDimension.getProgramStage()).thenReturn(programStage); - ProgramStage programStage = new ProgramStage(); - programStage.setProgram(program); + when(testedDimension.isEventDimension()).thenReturn(true); - ElementWithOffset programElement = ElementWithOffset.of(program, positiveOffset); - ElementWithOffset programStageElement = - ElementWithOffset.of(programStage, positiveOffset); - - TrackedEntityType trackedEntityType = new TrackedEntityType(); - trackedEntityType.setUid("uid2"); + assertEquals( + """ + (select field + from analytics_tei_events_trackedentitytype + where programstageinstanceuid = (select programStageInstanceUid + from (select *, + row_number() over ( partition by programinstanceuid + order by occurreddate desc ) as rn + from analytics_tei_events_trackedentitytype events + where programstageuid = 'programStageUid' + and programinstanceuid = (select programInstanceUid + from (select *, + row_number() over ( partition by trackedentityinstanceuid + order by enrollmentdate desc ) as rn + from analytics_tei_enrollments_trackedentitytype + where programuid = 'programUid' + and t_1.trackedentityinstanceuid = trackedentityinstanceuid) en + where en.rn = 1) + and status != 'SCHEDULE') ev + where ev.rn = 1))""", + SqlQueryHelper.buildOrderSubQuery(testedDimension, () -> "field").render()); - SqlParameterManager sqlParameterManager = new SqlParameterManager(); + assertEquals( + """ + exists(select 1 + from (select * + from (select *, row_number() over (partition by trackedentityinstanceuid order by enrollmentdate desc) as rn + from analytics_tei_enrollments_trackedentitytype + where programuid = 'programUid' + and trackedentityinstanceuid = t_1.trackedentityinstanceuid) en + where en.rn = 1) as "enrollmentSubqueryAlias" + where exists(select 1 + from (select * + from (select *, row_number() over ( partition by programinstanceuid order by occurreddate desc ) as rn + from analytics_tei_events_trackedentitytype + where "enrollmentSubqueryAlias".programinstanceuid = programinstanceuid + and programstageuid = 'programStageUid' + and status != 'SCHEDULE') ev + where ev.rn = 1) as "prefix" + where exists(select 1 + from analytics_tei_events_trackedentitytype + where "prefix".programstageinstanceuid = programstageinstanceuid + and field)))""", + SqlQueryHelper.buildExistsValueSubquery(testedDimension, () -> "field").render()); + } - // When - String statement = - SqlQueryHelper.eventSelect( - programElement, programStageElement, trackedEntityType, sqlParameterManager); + private ElementWithOffset mockElementWithOffset( + Class clazz, String uid) { + return mockElementWithOffset(clazz, uid, element -> {}); + } - // Then - assertEquals( - "select innermost_evt.* from (select *, row_number() over (partition by programinstanceuid order by occurreddate desc, created desc ) as rn from analytics_tei_events_uid2 where status != 'SCHEDULE' and programuid = :1 and programstageuid = :2) innermost_evt where innermost_evt.rn = 4", - statement); + private ElementWithOffset mockElementWithOffset( + Class clazz, String uid, Consumer consumer) { + ElementWithOffset elementWithOffset = mock(ElementWithOffset.class); + T element = mock(clazz); + consumer.accept(element); + when(elementWithOffset.getElement()).thenReturn(element); + when(element.getUid()).thenReturn(uid); + return elementWithOffset; } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java index ee0f5fbf8516..d7266b8c5368 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/grid/ListGrid.java @@ -77,10 +77,6 @@ * @author Lars Helge Overland */ public class ListGrid implements Grid, Serializable { - - public static final String HAS_VALUE = ".hasValue"; - public static final String STATUS = ".status"; - public static final String EXISTS = ".exists"; public static final String LEGEND = ".legend"; private static final String REGRESSION_SUFFIX = "_regression";