diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java index fa270fbd3660..57f25bfa5e92 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java @@ -59,10 +59,11 @@ /** Provides methods targeting the generation of SQL statements for periods and time fields. */ public abstract class TimeFieldSqlRenderer { protected final SqlBuilder sqlBuilder; - protected final StatementBuilder statementBuilder = new DefaultStatementBuilder(); + protected final StatementBuilder statementBuilder; protected TimeFieldSqlRenderer(SqlBuilder sqlBuilder) { this.sqlBuilder = sqlBuilder; + this.statementBuilder = new DefaultStatementBuilder(sqlBuilder); } /** @@ -173,7 +174,7 @@ private void collectDateRangeSqlConditions( /** * Returns a string representing the SQL condition for the given {@link ColumnWithDateRange}. * - * @param dateRangeColumn + * @param dateRangeColumn the {@link ColumnWithDateRange} * @return the SQL statement */ private String getDateRangeCondition(ColumnWithDateRange dateRangeColumn) { diff --git a/dhis-2/dhis-services/dhis-service-core/pom.xml b/dhis-2/dhis-services/dhis-service-core/pom.xml index 2e76d7dac6b4..dae45dad2ee3 100644 --- a/dhis-2/dhis-services/dhis-service-core/pom.xml +++ b/dhis-2/dhis-services/dhis-service-core/pom.xml @@ -64,6 +64,10 @@ org.hisp.dhis dhis-support-jdbc + + org.hisp.dhis + dhis-support-sql + org.apache.httpcomponents.core5 diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java index 5cc71ff9837f..1846f361ed5a 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java @@ -111,6 +111,7 @@ import org.hisp.dhis.constant.Constant; import org.hisp.dhis.constant.ConstantService; import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.expression.dataitem.DimItemDataElementAndOperand; import org.hisp.dhis.expression.dataitem.DimItemIndicator; import org.hisp.dhis.expression.dataitem.DimItemProgramAttribute; @@ -179,6 +180,8 @@ public class DefaultExpressionService implements ExpressionService { private final I18nManager i18nManager; + private final SqlBuilder sqlBuilder; + // ------------------------------------------------------------------------- // Static data // ------------------------------------------------------------------------- @@ -279,13 +282,15 @@ public DefaultExpressionService( DimensionService dimensionService, IdentifiableObjectManager idObjectManager, I18nManager i18nManager, - CacheProvider cacheProvider) { + CacheProvider cacheProvider, + SqlBuilder sqlBuilder) { checkNotNull(expressionStore); checkNotNull(constantService); checkNotNull(dimensionService); checkNotNull(idObjectManager); checkNotNull(i18nManager); checkNotNull(cacheProvider); + checkNotNull(sqlBuilder); this.expressionStore = expressionStore; this.constantService = constantService; @@ -293,6 +298,7 @@ public DefaultExpressionService( this.idObjectManager = idObjectManager; this.i18nManager = i18nManager; this.constantMapCache = cacheProvider.createAllConstantsCache(); + this.sqlBuilder = sqlBuilder; } // ------------------------------------------------------------------------- @@ -726,6 +732,7 @@ private CommonExpressionVisitor newVisitor( .params(params) .info(params.getExpressionInfo()) .state(initialParsingState) + .sqlBuilder(sqlBuilder) .build(); } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/DefaultProgramIndicatorService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/DefaultProgramIndicatorService.java index 673d023106c4..e1da1b661f22 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/DefaultProgramIndicatorService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/DefaultProgramIndicatorService.java @@ -34,36 +34,7 @@ import static org.hisp.dhis.expression.ExpressionParams.DEFAULT_EXPRESSION_PARAMS; import static org.hisp.dhis.parser.expression.ExpressionItem.ITEM_GET_DESCRIPTIONS; import static org.hisp.dhis.parser.expression.ExpressionItem.ITEM_GET_SQL; -import static org.hisp.dhis.parser.expression.ParserUtils.COMMON_EXPRESSION_ITEMS; import static org.hisp.dhis.parser.expression.ProgramExpressionParams.DEFAULT_PROGRAM_EXPRESSION_PARAMS; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.AVG; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.A_BRACE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.COUNT; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_CONDITION; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_COUNT; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_COUNT_IF_CONDITION; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_COUNT_IF_VALUE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_DAYS_BETWEEN; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_HAS_VALUE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_MAX_VALUE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_MINUTES_BETWEEN; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_MIN_VALUE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_MONTHS_BETWEEN; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_OIZP; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_RELATIONSHIP_COUNT; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_WEEKS_BETWEEN; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_YEARS_BETWEEN; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_ZING; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.D2_ZPVC; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.HASH_BRACE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.MAX; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.MIN; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.PS_EVENTDATE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.STAGE_OFFSET; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.STDDEV; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.SUM; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.VARIANCE; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_BRACE; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; @@ -74,6 +45,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.antlr.Parser; @@ -84,6 +56,7 @@ import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.IdentifiableObjectStore; import org.hisp.dhis.commons.util.TextUtils; +import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.i18n.I18nManager; @@ -91,35 +64,7 @@ import org.hisp.dhis.parser.expression.ExpressionItem; import org.hisp.dhis.parser.expression.ExpressionItemMethod; import org.hisp.dhis.parser.expression.ProgramExpressionParams; -import org.hisp.dhis.parser.expression.function.RepeatableProgramStageOffset; -import org.hisp.dhis.parser.expression.function.VectorAvg; -import org.hisp.dhis.parser.expression.function.VectorCount; -import org.hisp.dhis.parser.expression.function.VectorMax; -import org.hisp.dhis.parser.expression.function.VectorMin; -import org.hisp.dhis.parser.expression.function.VectorStddevSamp; -import org.hisp.dhis.parser.expression.function.VectorSum; -import org.hisp.dhis.parser.expression.function.VectorVariance; import org.hisp.dhis.parser.expression.literal.SqlLiteral; -import org.hisp.dhis.program.dataitem.ProgramItemAttribute; -import org.hisp.dhis.program.dataitem.ProgramItemPsEventdate; -import org.hisp.dhis.program.dataitem.ProgramItemStageElement; -import org.hisp.dhis.program.function.D2Condition; -import org.hisp.dhis.program.function.D2Count; -import org.hisp.dhis.program.function.D2CountIfCondition; -import org.hisp.dhis.program.function.D2CountIfValue; -import org.hisp.dhis.program.function.D2DaysBetween; -import org.hisp.dhis.program.function.D2HasValue; -import org.hisp.dhis.program.function.D2MaxValue; -import org.hisp.dhis.program.function.D2MinValue; -import org.hisp.dhis.program.function.D2MinutesBetween; -import org.hisp.dhis.program.function.D2MonthsBetween; -import org.hisp.dhis.program.function.D2Oizp; -import org.hisp.dhis.program.function.D2RelationshipCount; -import org.hisp.dhis.program.function.D2WeeksBetween; -import org.hisp.dhis.program.function.D2YearsBetween; -import org.hisp.dhis.program.function.D2Zing; -import org.hisp.dhis.program.function.D2Zpvc; -import org.hisp.dhis.program.variable.ProgramVariableItem; import org.hisp.dhis.system.util.SqlUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @@ -146,6 +91,10 @@ public class DefaultProgramIndicatorService implements ProgramIndicatorService { private final Cache analyticsSqlCache; + private final SqlBuilder sqlBuilder; + + @Getter private final ImmutableMap programIndicatorItems; + public DefaultProgramIndicatorService( ProgramIndicatorStore programIndicatorStore, @Qualifier("org.hisp.dhis.program.ProgramIndicatorGroupStore") @@ -155,7 +104,8 @@ public DefaultProgramIndicatorService( ExpressionService expressionService, DimensionService dimensionService, I18nManager i18nManager, - CacheProvider cacheProvider) { + CacheProvider cacheProvider, + SqlBuilder sqlBuilder) { checkNotNull(programIndicatorStore); checkNotNull(programIndicatorGroupStore); checkNotNull(programStageService); @@ -164,6 +114,7 @@ public DefaultProgramIndicatorService( checkNotNull(dimensionService); checkNotNull(i18nManager); checkNotNull(cacheProvider); + checkNotNull(sqlBuilder); this.programIndicatorStore = programIndicatorStore; this.programIndicatorGroupStore = programIndicatorGroupStore; @@ -173,57 +124,10 @@ public DefaultProgramIndicatorService( this.dimensionService = dimensionService; this.i18nManager = i18nManager; this.analyticsSqlCache = cacheProvider.createAnalyticsSqlCache(); - } - - public static final ImmutableMap PROGRAM_INDICATOR_ITEMS = - ImmutableMap.builder() - - // Common functions - - .putAll(COMMON_EXPRESSION_ITEMS) - - // Program functions - - .put(D2_CONDITION, new D2Condition()) - .put(D2_COUNT, new D2Count()) - .put(D2_COUNT_IF_CONDITION, new D2CountIfCondition()) - .put(D2_COUNT_IF_VALUE, new D2CountIfValue()) - .put(D2_DAYS_BETWEEN, new D2DaysBetween()) - .put(D2_HAS_VALUE, new D2HasValue()) - .put(D2_MAX_VALUE, new D2MaxValue()) - .put(D2_MINUTES_BETWEEN, new D2MinutesBetween()) - .put(D2_MIN_VALUE, new D2MinValue()) - .put(D2_MONTHS_BETWEEN, new D2MonthsBetween()) - .put(D2_OIZP, new D2Oizp()) - .put(D2_RELATIONSHIP_COUNT, new D2RelationshipCount()) - .put(D2_WEEKS_BETWEEN, new D2WeeksBetween()) - .put(D2_YEARS_BETWEEN, new D2YearsBetween()) - .put(D2_ZING, new D2Zing()) - .put(D2_ZPVC, new D2Zpvc()) + this.sqlBuilder = sqlBuilder; - // Program functions for custom aggregation - - .put(AVG, new VectorAvg()) - .put(COUNT, new VectorCount()) - .put(MAX, new VectorMax()) - .put(MIN, new VectorMin()) - .put(STDDEV, new VectorStddevSamp()) - .put(SUM, new VectorSum()) - .put(VARIANCE, new VectorVariance()) - - // Data items - - .put(HASH_BRACE, new ProgramItemStageElement()) - .put(A_BRACE, new ProgramItemAttribute()) - .put(PS_EVENTDATE, new ProgramItemPsEventdate()) - - // Program variables - - .put(V_BRACE, new ProgramVariableItem()) - - // . functions - .put(STAGE_OFFSET, new RepeatableProgramStageOffset()) - .build(); + this.programIndicatorItems = new ExpressionMapBuilder(sqlBuilder).getExpressionItemMap(); + } // ------------------------------------------------------------------------- // ProgramIndicator CRUD @@ -518,10 +422,11 @@ private CommonExpressionVisitor newVisitor( .programStageService(programStageService) .i18nSupplier(Suppliers.memoize(i18nManager::getI18n)) .constantMap(expressionService.getConstantMap()) - .itemMap(PROGRAM_INDICATOR_ITEMS) + .itemMap(programIndicatorItems) .itemMethod(itemMethod) .params(params) .progParams(progParams) + .sqlBuilder(sqlBuilder) .build(); } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ExpressionMapBuilder.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ExpressionMapBuilder.java new file mode 100644 index 000000000000..c1833acf4bdc --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ExpressionMapBuilder.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.program; + +import static org.hisp.dhis.parser.expression.ParserUtils.COMMON_EXPRESSION_ITEMS; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.*; + +import com.google.common.collect.ImmutableMap; +import lombok.Getter; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.parser.expression.ExpressionItem; +import org.hisp.dhis.parser.expression.function.RepeatableProgramStageOffset; +import org.hisp.dhis.parser.expression.function.VectorAvg; +import org.hisp.dhis.parser.expression.function.VectorCount; +import org.hisp.dhis.parser.expression.function.VectorMax; +import org.hisp.dhis.parser.expression.function.VectorMin; +import org.hisp.dhis.parser.expression.function.VectorStddevSamp; +import org.hisp.dhis.parser.expression.function.VectorSum; +import org.hisp.dhis.parser.expression.function.VectorVariance; +import org.hisp.dhis.program.dataitem.ProgramItemAttribute; +import org.hisp.dhis.program.dataitem.ProgramItemPsEventdate; +import org.hisp.dhis.program.dataitem.ProgramItemStageElement; +import org.hisp.dhis.program.function.D2Condition; +import org.hisp.dhis.program.function.D2Count; +import org.hisp.dhis.program.function.D2CountIfCondition; +import org.hisp.dhis.program.function.D2CountIfValue; +import org.hisp.dhis.program.function.D2DaysBetween; +import org.hisp.dhis.program.function.D2HasValue; +import org.hisp.dhis.program.function.D2MaxValue; +import org.hisp.dhis.program.function.D2MinValue; +import org.hisp.dhis.program.function.D2MinutesBetween; +import org.hisp.dhis.program.function.D2MonthsBetween; +import org.hisp.dhis.program.function.D2Oizp; +import org.hisp.dhis.program.function.D2RelationshipCount; +import org.hisp.dhis.program.function.D2WeeksBetween; +import org.hisp.dhis.program.function.D2YearsBetween; +import org.hisp.dhis.program.function.D2Zing; +import org.hisp.dhis.program.function.D2Zpvc; +import org.hisp.dhis.program.variable.ProgramVariableItem; + +/** + * This component encapsulates the creation of the immutable expressions map. The map contains all + * the expression items that are used in the program expressions. + */ +@Getter +public class ExpressionMapBuilder { + + private final ImmutableMap expressionItemMap; + + public ExpressionMapBuilder(SqlBuilder sqlBuilder) { + expressionItemMap = + ImmutableMap.builder() + + // Common functions + + .putAll(COMMON_EXPRESSION_ITEMS) + + // Program functions + + .put(D2_CONDITION, new D2Condition().withSqlBuilder(sqlBuilder)) + .put(D2_COUNT, new D2Count().withSqlBuilder(sqlBuilder)) + .put(D2_COUNT_IF_CONDITION, new D2CountIfCondition().withSqlBuilder(sqlBuilder)) + .put(D2_COUNT_IF_VALUE, new D2CountIfValue().withSqlBuilder(sqlBuilder)) + .put(D2_DAYS_BETWEEN, new D2DaysBetween().withSqlBuilder(sqlBuilder)) + .put(D2_HAS_VALUE, new D2HasValue().withSqlBuilder(sqlBuilder)) + .put(D2_MAX_VALUE, new D2MaxValue().withSqlBuilder(sqlBuilder)) + .put(D2_MINUTES_BETWEEN, new D2MinutesBetween().withSqlBuilder(sqlBuilder)) + .put(D2_MIN_VALUE, new D2MinValue().withSqlBuilder(sqlBuilder)) + .put(D2_MONTHS_BETWEEN, new D2MonthsBetween().withSqlBuilder(sqlBuilder)) + .put(D2_OIZP, new D2Oizp().withSqlBuilder(sqlBuilder)) + .put(D2_RELATIONSHIP_COUNT, new D2RelationshipCount().withSqlBuilder(sqlBuilder)) + .put(D2_WEEKS_BETWEEN, new D2WeeksBetween().withSqlBuilder(sqlBuilder)) + .put(D2_YEARS_BETWEEN, new D2YearsBetween().withSqlBuilder(sqlBuilder)) + .put(D2_ZING, new D2Zing().withSqlBuilder(sqlBuilder)) + .put(D2_ZPVC, new D2Zpvc().withSqlBuilder(sqlBuilder)) + + // Program functions for custom aggregation + + .put(AVG, new VectorAvg()) + .put(COUNT, new VectorCount()) + .put(MAX, new VectorMax()) + .put(MIN, new VectorMin()) + .put(STDDEV, new VectorStddevSamp()) + .put(SUM, new VectorSum()) + .put(VARIANCE, new VectorVariance()) + + // Data items + + .put(HASH_BRACE, new ProgramItemStageElement().withSqlBuilder(sqlBuilder)) + .put(A_BRACE, new ProgramItemAttribute().withSqlBuilder(sqlBuilder)) + .put(PS_EVENTDATE, new ProgramItemPsEventdate().withSqlBuilder(sqlBuilder)) + + // Program variables + + .put(V_BRACE, new ProgramVariableItem().withSqlBuilder(sqlBuilder)) + + // . functions + .put(STAGE_OFFSET, new RepeatableProgramStageOffset()) + .build(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java index 52cca2b89166..7d8db85e73cf 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramExpressionItem.java @@ -30,13 +30,13 @@ import static org.hisp.dhis.analytics.DataType.BOOLEAN; import static org.hisp.dhis.analytics.DataType.NUMERIC; import static org.hisp.dhis.common.ValueType.NUMBER; -import static org.hisp.dhis.parser.expression.ParserUtils.castSql; import static org.hisp.dhis.parser.expression.ParserUtils.replaceSqlNull; import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.antlr.ParserExceptionWithoutContext; import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; import org.hisp.dhis.parser.expression.ExpressionItem; import org.hisp.dhis.program.dataitem.ProgramItemAttribute; @@ -58,6 +58,8 @@ */ public abstract class ProgramExpressionItem implements ExpressionItem { + private SqlBuilder sqlBuilder; + @Override public final Object getExpressionInfo(ExprContext ctx, CommonExpressionVisitor visitor) { throw new ParserExceptionWithoutContext( @@ -122,6 +124,11 @@ protected String replaceNullSqlValues( if (dataType == NUMERIC || dataType == BOOLEAN) { dataType = visitor.getParams().getDataType() == BOOLEAN ? BOOLEAN : NUMERIC; } - return replaceSqlNull(castSql(column, dataType), dataType); + return replaceSqlNull(sqlBuilder.cast(column, dataType), dataType); + } + + protected ExpressionItem withSqlBuilder(SqlBuilder sqlBuilder) { + this.sqlBuilder = sqlBuilder; + return this; } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java index b472290e850f..ec00162697ef 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java @@ -89,6 +89,7 @@ import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.hibernate.HibernateGenericStore; import org.hisp.dhis.i18n.I18nManager; import org.hisp.dhis.indicator.Indicator; @@ -107,6 +108,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; /** @@ -126,6 +128,8 @@ class ExpressionServiceTest extends TestBase { @Mock private CacheProvider cacheProvider; + @Spy private PostgreSqlBuilder sqlBuilder; + private DefaultExpressionService target; private CategoryOption categoryOptionA; @@ -248,7 +252,8 @@ public void setUp() { dimensionService, idObjectManager, i18nManager, - cacheProvider); + cacheProvider, + sqlBuilder); categoryOptionA = new CategoryOption("Under 5"); categoryOptionB = new CategoryOption("Over 5"); diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorFunctionsTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorFunctionsTest.java index a5de867bd46f..f50fa59cc0f6 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorFunctionsTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorFunctionsTest.java @@ -35,7 +35,6 @@ import static org.hisp.dhis.parser.expression.ExpressionItem.ITEM_GET_DESCRIPTIONS; import static org.hisp.dhis.parser.expression.ExpressionItem.ITEM_GET_SQL; import static org.hisp.dhis.program.AnalyticsType.ENROLLMENT; -import static org.hisp.dhis.program.DefaultProgramIndicatorService.PROGRAM_INDICATOR_ITEMS; import static org.hisp.dhis.program.variable.vEventCount.DEFAULT_COUNT_CONDITION; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -58,6 +57,7 @@ import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementDomain; +import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -72,6 +72,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; /** @@ -115,6 +116,8 @@ class ProgramSqlGeneratorFunctionsTest extends TestBase { @Mock private DimensionService dimensionService; + @Spy private PostgreSqlBuilder sqlBuilder; + @BeforeEach public void setUp() { dataElementA = createDataElement('A'); @@ -797,10 +800,11 @@ private Object test( .programIndicatorService(programIndicatorService) .programStageService(programStageService) .i18nSupplier(() -> new I18n(null, null)) - .itemMap(PROGRAM_INDICATOR_ITEMS) + .itemMap(new ExpressionMapBuilder(sqlBuilder).getExpressionItemMap()) .itemMethod(itemMethod) .params(params) .progParams(progParams) + .sqlBuilder(new PostgreSqlBuilder()) .build(); visitor.setExpressionLiteral(exprLiteral); diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorItemsTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorItemsTest.java index 542b78850e8a..45c28992aadd 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorItemsTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorItemsTest.java @@ -33,7 +33,6 @@ import static org.hisp.dhis.antlr.AntlrParserUtils.castString; import static org.hisp.dhis.parser.expression.ExpressionItem.ITEM_GET_DESCRIPTIONS; import static org.hisp.dhis.parser.expression.ExpressionItem.ITEM_GET_SQL; -import static org.hisp.dhis.program.DefaultProgramIndicatorService.PROGRAM_INDICATOR_ITEMS; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; @@ -51,6 +50,7 @@ import org.hisp.dhis.constant.Constant; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementDomain; +import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -64,6 +64,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -100,6 +101,8 @@ class ProgramSqlGeneratorItemsTest extends TestBase { @Mock private DimensionService dimensionService; + @Spy private PostgreSqlBuilder sqlBuilder; + @BeforeEach public void setUp() { dataElementA = createDataElement('A'); @@ -234,10 +237,11 @@ private Object test( .programStageService(programStageService) .i18nSupplier(() -> new I18n(null, null)) .constantMap(constantMap) - .itemMap(PROGRAM_INDICATOR_ITEMS) + .itemMap(new ExpressionMapBuilder(sqlBuilder).getExpressionItemMap()) .itemMethod(itemMethod) .params(params) .progParams(progParams) + .sqlBuilder(sqlBuilder) .build(); visitor.setExpressionLiteral(exprLiteral); diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorVariablesTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorVariablesTest.java index d6e5394f1725..b3eeced22957 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorVariablesTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/program/ProgramSqlGeneratorVariablesTest.java @@ -33,7 +33,6 @@ import static org.hisp.dhis.analytics.DataType.NUMERIC; import static org.hisp.dhis.antlr.AntlrParserUtils.castString; import static org.hisp.dhis.parser.expression.ExpressionItem.ITEM_GET_SQL; -import static org.hisp.dhis.program.DefaultProgramIndicatorService.PROGRAM_INDICATOR_ITEMS; import static org.hisp.dhis.program.variable.vEventCount.DEFAULT_COUNT_CONDITION; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -46,6 +45,7 @@ import org.hisp.dhis.antlr.literal.DefaultLiteral; import org.hisp.dhis.common.DimensionService; import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; @@ -57,6 +57,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; /** @@ -82,6 +83,8 @@ class ProgramSqlGeneratorVariablesTest extends TestBase { @Mock private I18n i18n; + @Spy private PostgreSqlBuilder sqlBuilder; + private CommonExpressionVisitor subject; private ProgramIndicator eventIndicator; @@ -304,10 +307,11 @@ private Object test( .programIndicatorService(programIndicatorService) .programStageService(programStageService) .i18nSupplier(() -> new I18n(null, null)) - .itemMap(PROGRAM_INDICATOR_ITEMS) + .itemMap(new ExpressionMapBuilder(sqlBuilder).getExpressionItemMap()) .itemMethod(ITEM_GET_SQL) .params(params) .progParams(progParams) + .sqlBuilder(sqlBuilder) .build(); subject.setExpressionLiteral(exprLiteral); diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml b/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml index 8f0f487bd983..479ba2b124a7 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml +++ b/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml @@ -31,6 +31,10 @@ org.hisp.dhis.parser dhis-antlr-expression-parser + + org.hisp.dhis + dhis-support-sql + diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java index 2287dc0da86e..1c06105f3bb3 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java @@ -27,6 +27,8 @@ */ package org.hisp.dhis.parser.expression; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -43,6 +45,7 @@ import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.QueryModifiers; import org.hisp.dhis.constant.Constant; +import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.expression.ExpressionInfo; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.i18n.I18n; @@ -60,9 +63,9 @@ */ @Getter @Setter -@Builder(toBuilder = true) public class CommonExpressionVisitor extends AntlrExpressionVisitor { - private final StatementBuilder statementBuilder = new DefaultStatementBuilder(); + + private StatementBuilder statementBuilder; private IdentifiableObjectManager idObjectManager; @@ -74,6 +77,8 @@ public class CommonExpressionVisitor extends AntlrExpressionVisitor { private TrackedEntityAttributeService attributeService; + private SqlBuilder sqlBuilder; + /** * A {@link Supplier} object that can return a {@link I18n} instance when needed. This is done * because retrieving a {@link I18n} instance can be expensive and is not needed for most parsing @@ -82,7 +87,7 @@ public class CommonExpressionVisitor extends AntlrExpressionVisitor { private Supplier i18nSupplier; /** Map of constant values to use in evaluating the expression. */ - @Builder.Default private Map constantMap = new HashMap<>(); + private Map constantMap = new HashMap<>(); /** Map of ExprItem object instances to call for each expression item. */ private Map itemMap; @@ -91,26 +96,67 @@ public class CommonExpressionVisitor extends AntlrExpressionVisitor { private ExpressionItemMethod itemMethod; /** Parameters to evaluate the expression to a value. */ - @Builder.Default private ExpressionParams params = ExpressionParams.builder().build(); + private ExpressionParams params; /** Parameters to generate SQL from a program expression. */ - @Builder.Default - private ProgramExpressionParams progParams = ProgramExpressionParams.builder().build(); + private ProgramExpressionParams progParams; /** State variables during an expression evaluation. */ - @Builder.Default private ExpressionState state = new ExpressionState(); + private ExpressionState state; /** * Information found from parsing the raw expression (contains nothing that is the result of data * or metadata found in the database). */ - @Builder.Default private ExpressionInfo info = new ExpressionInfo(); + private ExpressionInfo info; /** * Used to collect the string replacements to build a description. This may contain names of * metadata from the database. */ - @Builder.Default private Map itemDescriptions = new HashMap<>(); + private Map itemDescriptions; + + // ------------------------------------------------------------------------- + // Custom constructor + // ------------------------------------------------------------------------- + + @Builder(toBuilder = true) + public CommonExpressionVisitor( + IdentifiableObjectManager idObjectManager, + DimensionService dimensionService, + ProgramIndicatorService programIndicatorService, + ProgramStageService programStageService, + TrackedEntityAttributeService attributeService, + SqlBuilder sqlBuilder, + Supplier i18nSupplier, + Map constantMap, + Map itemMap, + ExpressionItemMethod itemMethod, + ExpressionParams params, + ProgramExpressionParams progParams, + ExpressionState state, + ExpressionInfo info, + Map itemDescriptions) { + + checkNotNull(sqlBuilder); + + this.statementBuilder = new DefaultStatementBuilder(sqlBuilder); + this.idObjectManager = idObjectManager; + this.dimensionService = dimensionService; + this.programIndicatorService = programIndicatorService; + this.programStageService = programStageService; + this.attributeService = attributeService; + this.sqlBuilder = sqlBuilder; + this.i18nSupplier = i18nSupplier; + this.constantMap = constantMap != null ? constantMap : new HashMap<>(); + this.itemMap = itemMap; + this.itemMethod = itemMethod; + this.params = params != null ? params : ExpressionParams.builder().build(); + this.progParams = progParams != null ? progParams : ProgramExpressionParams.builder().build(); + this.state = state != null ? state : new ExpressionState(); + this.info = info != null ? info : new ExpressionInfo(); + this.itemDescriptions = itemDescriptions != null ? itemDescriptions : new HashMap<>(); + } // ------------------------------------------------------------------------- // Visitor logic diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java index 5afb97088bb0..da19f9693fc0 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java @@ -38,8 +38,9 @@ import java.util.Date; import java.util.Optional; import java.util.regex.Matcher; -import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsConstants; +import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.period.Period; import org.hisp.dhis.program.AnalyticsPeriodBoundary; import org.hisp.dhis.program.AnalyticsType; @@ -49,11 +50,11 @@ /** * @author Lars Helge Overland */ -@NoArgsConstructor +@RequiredArgsConstructor public class DefaultStatementBuilder implements StatementBuilder { protected static final String QUOTE = "\""; - protected static final String SINGLE_QUOTE = "'"; + private final SqlBuilder sqlBuilder; @Override public String getProgramIndicatorDataValueSelectSql( @@ -109,7 +110,8 @@ public String getProgramIndicatorEventColumnSql( private String getProgramIndicatorDataElementInEventSelectSql( String columnName, String programStageUid) { - return format("case when ax.\"ps\" = '%s' then %s else null end", programStageUid, columnName); + String col = sqlBuilder.quote("ps"); + return format("case when ax.%s = '%s' then %s else null end", col, programStageUid, columnName); } private String getProgramIndicatorEventInEnrollmentSelectSql( @@ -283,19 +285,13 @@ public String getBoundaryCondition( protected String columnQuote(String column) { column = column.replace(QUOTE, (QUOTE + QUOTE)); - - return QUOTE + column + QUOTE; + return sqlBuilder.quote(column); } /** * Based on the given arguments, this method returns the column associated to the boundary object. * This column should be used as part of the boundary SQL statement. * - * @param boundary - * @param programIndicator - * @param timeField - * @param reportingStartDate - * @param reportingEndDate * @return the respective boundary column */ private String getBoundaryColumn( @@ -331,7 +327,6 @@ private String getBoundaryColumn( * SCHEDULE (which makes it backward compatible). In this case the logic will remain based on * "occurreddate". * - * @param column * @return the backwards compatible column */ private String keepOrderCompatibilityColumn(final String column) { diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index 42938d1403d8..2f3af2088b4f 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -32,6 +32,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.Validate; +import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.Index; import org.hisp.dhis.db.model.Table; @@ -228,6 +229,16 @@ public String jsonExtractNested(String json, String... expression) { return String.format("JSONExtractString(%s, '%s')", json, path); } + @Override + public String cast(String column, DataType dataType) { + return switch (dataType) { + case NUMERIC -> String.format("toDecimal64(%s, 8)", column); // 8 decimal places precision + case BOOLEAN -> + String.format("toUInt8(%s) != 0", column); // ClickHouse uses UInt8 for boolean + case TEXT -> String.format("toString(%s)", column); + }; + } + // Statements @Override diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 920ca25d1e8e..0408612d60ad 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -32,6 +32,7 @@ import java.util.Map; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.Validate; +import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.Index; import org.hisp.dhis.db.model.Table; @@ -231,6 +232,15 @@ public String jsonExtractNested(String column, String... expression) { return String.format("json_unquote(json_extract(%s, '%s'))", column, path); } + @Override + public String cast(String column, DataType dataType) { + return switch (dataType) { + case NUMERIC -> String.format("CAST(%s AS DECIMAL)", column); + case BOOLEAN -> String.format("CAST(%s AS DECIMAL) != 0", column); + case TEXT -> String.format("CAST(%s AS CHAR)", column); + }; + } + // Statements @Override diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java index aebd74100e11..65936912c159 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java @@ -29,6 +29,7 @@ import static org.hisp.dhis.commons.util.TextUtils.removeLastComma; +import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.db.model.Collation; import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.Index; @@ -249,6 +250,16 @@ public String jsonExtractNested(String column, String... expression) { return String.format("%s #>> '{%s}'", column, String.join(", ", expression)); } + @Override + public String cast(String column, DataType dataType) { + return column + + switch (dataType) { + case NUMERIC -> "::numeric"; + case BOOLEAN -> "::numeric!=0"; + case TEXT -> "::text"; + }; + } + // Statements @Override diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index 8655372ff807..6580e7a24dcd 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -28,6 +28,7 @@ package org.hisp.dhis.db.sql; import java.util.Collection; +import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.db.model.Index; import org.hisp.dhis.db.model.Table; @@ -287,6 +288,16 @@ public interface SqlBuilder { */ String jsonExtractNested(String json, String... expression); + /** + * Generates a SQL casting expression for the given column or expression. + * + * @param column The column or expression to be cast. Must not be null. + * @param dataType The target data type for the cast operation. Must not be null. + * @return A String containing the database-specific SQL casting expression. + * @see DataType + */ + String cast(String column, DataType dataType); + // Statements /**