Skip to content

Commit

Permalink
refactor: Reduce table joins for analytics [DHIS2-16705] (#19201)
Browse files Browse the repository at this point in the history
  • Loading branch information
@larshelge
larshelge authored Nov 18, 2024
1 parent 88f3a52 commit ddeb2bb
Show file tree
Hide file tree
Showing 6 changed files with 33 additions and 33 deletions.
9 changes: 4 additions & 5 deletions ...s/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,17 +181,17 @@ public class JdbcTrackedEntityAnalyticsTableManager extends AbstractJdbcTableMan
AnalyticsTableColumn.builder()
.name("ou")
.dataType(CHARACTER_11)
.selectExpression("ou.uid")
.selectExpression("ous.organisationunituid")
.build(),
AnalyticsTableColumn.builder()
.name("ouname")
.dataType(VARCHAR_255)
.selectExpression("ou.name")
.selectExpression("ous.name")
.build(),
AnalyticsTableColumn.builder()
.name("oucode")
.dataType(VARCHAR_50)
.selectExpression("ou.code")
.selectExpression("ous.code")
.build(),
AnalyticsTableColumn.builder()
.name("oulevel")
Expand Down Expand Up @@ -512,8 +512,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti
replaceQualify(
"""
\s from ${trackedentity} te \
left join ${organisationunit} ou on te.organisationunitid=ou.organisationunitid \
left join analytics_rs_orgunitstructure ous on ous.organisationunitid=ou.organisationunitid \
left join analytics_rs_orgunitstructure ous on te.organisationunitid=ous.organisationunitid \
left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid \
and (cast(${trackedEntityCreatedMonth} as date)=ougs.startdate \
or ougs.startdate is null)""",
Expand Down
18 changes: 8 additions & 10 deletions ...java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import static org.hisp.dhis.analytics.table.JdbcEventAnalyticsTableManager.EXPORTABLE_EVENT_STATUSES;
import static org.hisp.dhis.commons.util.TextUtils.removeLastComma;
import static org.hisp.dhis.db.model.DataType.CHARACTER_11;
import static org.hisp.dhis.db.model.DataType.CHARACTER_32;
import static org.hisp.dhis.db.model.DataType.DOUBLE;
import static org.hisp.dhis.db.model.DataType.GEOMETRY;
import static org.hisp.dhis.db.model.DataType.INTEGER;
Expand Down Expand Up @@ -162,19 +161,19 @@ public class JdbcTrackedEntityEnrollmentsAnalyticsTableManager extends AbstractJ
.name("ou")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ou.uid")
.selectExpression("ous.organisationunituid")
.build(),
AnalyticsTableColumn.builder()
.name("ouname")
.dataType(VARCHAR_255)
.nullable(NULL)
.selectExpression("ou.name")
.selectExpression("ous.name")
.build(),
AnalyticsTableColumn.builder()
.name("oucode")
.dataType(CHARACTER_32)
.dataType(VARCHAR_50)
.nullable(NULL)
.selectExpression("ou.code")
.selectExpression("ous.code")
.build(),
AnalyticsTableColumn.builder()
.name("oulevel")
Expand Down Expand Up @@ -286,12 +285,11 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti
replaceQualify(
"""
\sfrom ${enrollment} en \
inner join ${trackedentity} te on en.trackedentityid = te.trackedentityid \
and te.deleted = false and te.trackedentitytypeid =${trackedEntityTypeId} \
inner join ${trackedentity} te on en.trackedentityid=te.trackedentityid \
and te.deleted = false and te.trackedentitytypeid = ${trackedEntityTypeId} \
and te.lastupdated < '${startTime}' \
left join ${program} p on p.programid = en.programid \
left join ${organisationunit} ou on en.organisationunitid = ou.organisationunitid \
left join analytics_rs_orgunitstructure ous on ous.organisationunitid = ou.organisationunitid \
left join ${program} p on en.programid=p.programid \
left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \
where exists (select 1 from event ev where ev.deleted = false \
and ev.enrollmentid = en.enrollmentid \
and ev.status in (${statuses})) \
Expand Down
27 changes: 12 additions & 15 deletions ...main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEventsAnalyticsTableManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import static org.hisp.dhis.commons.util.TextUtils.removeLastComma;
import static org.hisp.dhis.commons.util.TextUtils.replace;
import static org.hisp.dhis.db.model.DataType.CHARACTER_11;
import static org.hisp.dhis.db.model.DataType.CHARACTER_32;
import static org.hisp.dhis.db.model.DataType.DOUBLE;
import static org.hisp.dhis.db.model.DataType.GEOMETRY;
import static org.hisp.dhis.db.model.DataType.INTEGER;
Expand Down Expand Up @@ -216,19 +215,19 @@ public class JdbcTrackedEntityEventsAnalyticsTableManager extends AbstractJdbcTa
.name("ou")
.dataType(CHARACTER_11)
.nullable(NULL)
.selectExpression("ou.uid")
.selectExpression("ous.organisationunituid")
.build(),
AnalyticsTableColumn.builder()
.name("ouname")
.dataType(VARCHAR_255)
.nullable(NULL)
.selectExpression("ou.name")
.selectExpression("ous.name")
.build(),
AnalyticsTableColumn.builder()
.name("oucode")
.dataType(CHARACTER_32)
.dataType(VARCHAR_50)
.nullable(NULL)
.selectExpression("ou.code")
.selectExpression("ous.code")
.build(),
AnalyticsTableColumn.builder()
.name("oulevel")
Expand Down Expand Up @@ -331,11 +330,10 @@ private List<Integer> getDataYears(AnalyticsTableUpdateParams params, TrackedEnt
select temp.supportedyear from \
(select distinct extract(year from ${eventDateExpression}) as supportedyear \
from ${trackedentity} te \
inner join ${trackedentitytype} tet on tet.trackedentitytypeid = te.trackedentitytypeid \
inner join ${enrollment} en on en.trackedentityid = te.trackedentityid \
inner join ${event} ev on ev.enrollmentid = en.enrollmentid \
inner join ${enrollment} en on te.trackedentityid=en.trackedentityid \
inner join ${event} ev on en.enrollmentid=ev.enrollmentid \
where ev.lastupdated <= '${startTime}' \
and tet.trackedentitytypeid = ${tetId} \
and te.trackedentitytypeid = ${tetId} \
and (${eventDateExpression}) is not null \
and (${eventDateExpression}) > '1000-01-01' \
and ev.deleted = false \
Expand Down Expand Up @@ -415,13 +413,12 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti
replaceQualify(
"""
\s from ${event} ev \
inner join ${enrollment} en on en.enrollmentid = ev.enrollmentid and en.deleted = false \
inner join ${trackedentity} te on te.trackedentityid = en.trackedentityid \
inner join ${enrollment} en on en.enrollmentid=ev.enrollmentid and en.deleted = false \
inner join ${trackedentity} te on te.trackedentityid=en.trackedentityid \
and te.deleted = false and te.trackedentitytypeid = ${tetId} and te.lastupdated < '${startTime}' \
left join ${programstage} ps on ps.programstageid = ev.programstageid \
left join ${program} p on p.programid = ps.programid \
left join ${organisationunit} ou on ev.organisationunitid = ou.organisationunitid \
left join analytics_rs_orgunitstructure ous on ous.organisationunitid = ou.organisationunitid \
left join ${programstage} ps on ev.programstageid=ps.programstageid \
left join ${program} p on ps.programid=p.programid \
left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \
where ev.status in (${statuses}) \
${partitionClause} \
and ev.deleted = false\s""",
Expand Down
3 changes: 1 addition & 2 deletions ...2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/DataType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ public enum DataType {
DOUBLE,
BOOLEAN,
CHARACTER_11,
CHARACTER_32,
VARCHAR_50,
VARCHAR_255,
TEXT,
Expand All @@ -58,7 +57,7 @@ public enum DataType {
EnumSet.of(SMALLINT, BIGINT, INTEGER, DECIMAL, FLOAT, DOUBLE);

private static final EnumSet<DataType> TYPES_CHARACTER =
EnumSet.of(CHARACTER_11, CHARACTER_32, VARCHAR_50, VARCHAR_255, TEXT);
EnumSet.of(CHARACTER_11, VARCHAR_50, VARCHAR_255, TEXT);

/**
* Indicates if the data type is numeric.
Expand Down
1 change: 0 additions & 1 deletion ...ervices/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,6 @@ protected String getDataTypeName(DataType dataType) {
case DOUBLE -> dataTypeDouble();
case BOOLEAN -> dataTypeBoolean();
case CHARACTER_11 -> dataTypeCharacter(11);
case CHARACTER_32 -> dataTypeCharacter(32);
case VARCHAR_50 -> dataTypeVarchar(50);
case VARCHAR_255 -> dataTypeVarchar(255);
case TEXT -> dataTypeText();
Expand Down
8 changes: 8 additions & 0 deletions ...c/main/java/org/hisp/dhis/resourcetable/table/OrganisationUnitStructureResourceTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ private List<Column> getColumns() {
Lists.newArrayList(
new Column("organisationunitid", DataType.BIGINT, Nullable.NOT_NULL),
new Column("organisationunituid", DataType.CHARACTER_11, Nullable.NOT_NULL),
new Column("code", DataType.VARCHAR_50, Nullable.NULL),
new Column("name", DataType.VARCHAR_255, Nullable.NOT_NULL),
new Column("openingdate", DataType.DATE, Nullable.NULL),
new Column("closeddate", DataType.DATE, Nullable.NULL),
new Column("level", DataType.INTEGER, Nullable.NOT_NULL),
new Column("path", DataType.VARCHAR_255, Nullable.NULL));

Expand Down Expand Up @@ -147,6 +151,10 @@ List<Object[]> createBatchObjects(List<OrganisationUnit> units, int level) {

values.add(unit.getId());
values.add(unit.getUid());
values.add(unit.getCode());
values.add(unit.getName());
values.add(unit.getOpeningDate());
values.add(unit.getClosedDate());
values.add(level);
values.add(unit.getPath());

Expand Down

0 comments on commit ddeb2bb

Please sign in to comment.