From 7813c418486adc4cb461af9c0e40461dbede75a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 08:15:13 +0100 Subject: [PATCH 01/24] chore(deps): bump com.fasterxml.jackson.core:jackson-annotations (#19327) Bumps [com.fasterxml.jackson.core:jackson-annotations](https://github.com/FasterXML/jackson) from 2.18.1 to 2.18.2. - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-annotations dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dhis-2/dhis-test-e2e/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-test-e2e/pom.xml b/dhis-2/dhis-test-e2e/pom.xml index e9c118efe62..e7f49aaf27d 100644 --- a/dhis-2/dhis-test-e2e/pom.xml +++ b/dhis-2/dhis-test-e2e/pom.xml @@ -17,7 +17,7 @@ 2.11.0 2.24.2 5.5.0 - 2.18.1 + 2.18.2 33.3.1-jre 1.5 5.2.1 From fcabfec556d161fa6347ecc3c2fb3064b69ce9af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 08:15:32 +0100 Subject: [PATCH 02/24] chore(deps): bump org.springframework.session:spring-session-core (#19325) Bumps [org.springframework.session:spring-session-core](https://github.com/spring-projects/spring-session) from 3.3.3 to 3.4.0. - [Release notes](https://github.com/spring-projects/spring-session/releases) - [Changelog](https://github.com/spring-projects/spring-session/blob/main/RELEASE.adoc) - [Commits](https://github.com/spring-projects/spring-session/compare/3.3.3...3.4.0) --- updated-dependencies: - dependency-name: org.springframework.session:spring-session-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dhis-2/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index 6a6dde5ee90..bb3a37e044c 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -104,7 +104,7 @@ 6.1.12 - 3.3.3 + 3.4.0 2.7.18 2.7.4 1.1.5.RELEASE From 1a9bb8b3295d011405b940e87d251b62f6b15824 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 08:15:46 +0100 Subject: [PATCH 03/24] chore(deps-dev): bump commons-io:commons-io in /dhis-2 (#19326) Bumps commons-io:commons-io from 2.17.0 to 2.18.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dhis-2/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index bb3a37e044c..ad2662b5a00 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -196,7 +196,7 @@ 1.5 3.6.1 1.9.0 - 2.17.0 + 2.18.0 2.1.1 1.6.0 5.3.1 From 5ee7059ab758183d641f1939b4ca32a29687d66f Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Thu, 28 Nov 2024 14:57:53 +0700 Subject: [PATCH 04/24] Fix:Data integrity check for data capture orgunits within data view orgunits (#19317) * Fix query for data view/data entry orgunits --- .../users_capture_ou_not_in_data_view_ou.yaml | 94 ++++++++++--------- ...grityUsersCaptureOrgunitNotInDataView.java | 12 ++- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/users/users_capture_ou_not_in_data_view_ou.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/users/users_capture_ou_not_in_data_view_ou.yaml index a18454f4c46..c49b5d20f56 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/users/users_capture_ou_not_in_data_view_ou.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/users/users_capture_ou_not_in_data_view_ou.yaml @@ -31,60 +31,62 @@ section: Users section_order: 1 summary_sql: >- WITH user_data_orgunits AS ( - SELECT userinfoid, array_agg(path) AS data_orgunits_paths - FROM organisationunit - JOIN usermembership ON organisationunit.organisationunitid = usermembership.organisationunitid - WHERE path IS NOT NULL - GROUP BY userinfoid + SELECT u.userinfoid, array_agg(ou.path) AS data_orgunits_paths + FROM usermembership u + JOIN organisationunit ou ON ou.organisationunitid = u.organisationunitid + WHERE ou.path IS NOT NULL + GROUP BY userinfoid ), user_view_orgunits AS ( - SELECT userinfoid, array_agg(path) AS view_orgunit_paths - FROM organisationunit - JOIN userdatavieworgunits ON organisationunit.organisationunitid = userdatavieworgunits.organisationunitid - WHERE path IS NOT NULL - GROUP BY userinfoid + SELECT u.userinfoid, array_agg(ou.path) AS view_orgunit_paths + FROM userdatavieworgunits u + JOIN organisationunit ou ON ou.organisationunitid = u.organisationunitid + WHERE path IS NOT NULL + GROUP BY userinfoid ), - usercount as ( - SELECT COUNT(*) as usercount FROM userinfo - ) - SELECT COUNT(ui.uid) as value, - 100.0 * COUNT(ui.uid) / NULLIF(usercount.usercount, 0) as percent + rs as ( + SELECT ui.uid, + ui.username as name, + NULL as comment, + array_agg(ou.name) as refs FROM userinfo ui INNER JOIN ( SELECT u.userinfoid, ARRAY_AGG(dop) FILTER (WHERE dop IS NOT NULL) AS invalid_data_orgunits FROM user_data_orgunits u INNER JOIN user_view_orgunits v ON u.userinfoid = v.userinfoid LEFT JOIN LATERAL ( - SELECT path AS dop - FROM unnest(u.data_orgunits_paths) AS path - WHERE NOT EXISTS ( - SELECT 1 - FROM unnest(COALESCE(v.view_orgunit_paths, '{}')) AS view_path - WHERE path LIKE view_path || '%' - ) + SELECT path AS dop + FROM unnest(u.data_orgunits_paths) AS path + WHERE NOT EXISTS ( + SELECT 1 + FROM unnest(COALESCE(v.view_orgunit_paths, '{}')) AS view_path + WHERE path LIKE view_path || '%' + ) ) invalid_paths ON true GROUP BY u.userinfoid HAVING array_length(ARRAY_AGG(dop) FILTER (WHERE dop IS NOT NULL), 1) > 0 ) x on x.userinfoid = ui.userinfoid INNER JOIN organisationunit ou on ou.path = any(x.invalid_data_orgunits) - JOIN usercount on true - GROUP BY usercount.usercount; + GROUP BY ui.uid, ui.username) + SELECT COUNT(*) as count, + 100.0 * COUNT(*) / NULLIF((SELECT COUNT(*) FROM userinfo), 0) as percent + FROM rs; details_sql: >- WITH user_data_orgunits AS ( - SELECT userinfoid, array_agg(path) AS data_orgunits_paths - FROM organisationunit - JOIN usermembership ON organisationunit.organisationunitid = usermembership.organisationunitid - WHERE organisationunit.path IS NOT NULL - GROUP BY userinfoid + SELECT u.userinfoid, array_agg(ou.path) AS data_orgunits_paths + FROM usermembership u + JOIN organisationunit ou ON ou.organisationunitid = u.organisationunitid + WHERE ou.path IS NOT NULL + GROUP BY userinfoid ), user_view_orgunits AS ( - SELECT userinfoid, array_agg(path) AS view_orgunit_paths - FROM organisationunit - JOIN userdatavieworgunits ON organisationunit.organisationunitid = userdatavieworgunits.organisationunitid - WHERE path IS NOT NULL - GROUP BY userinfoid + SELECT u.userinfoid, array_agg(ou.path) AS view_orgunit_paths + FROM userdatavieworgunits u + JOIN organisationunit ou ON ou.organisationunitid = u.organisationunitid + WHERE path IS NOT NULL + GROUP BY userinfoid ) - SELECT ui.uid, + SELECT ui.uid, ui.username as name, NULL as comment, array_agg(ou.name) as refs @@ -94,13 +96,13 @@ details_sql: >- FROM user_data_orgunits u INNER JOIN user_view_orgunits v ON u.userinfoid = v.userinfoid LEFT JOIN LATERAL ( - SELECT path AS dop - FROM unnest(u.data_orgunits_paths) AS path - WHERE NOT EXISTS ( - SELECT 1 - FROM unnest(COALESCE(v.view_orgunit_paths, '{}')) AS view_path - WHERE path LIKE view_path || '%' - ) + SELECT path AS dop + FROM unnest(u.data_orgunits_paths) AS path + WHERE NOT EXISTS ( + SELECT 1 + FROM unnest(COALESCE(v.view_orgunit_paths, '{}')) AS view_path + WHERE path LIKE view_path || '%' + ) ) invalid_paths ON true GROUP BY u.userinfoid HAVING array_length(ARRAY_AGG(dop) FILTER (WHERE dop IS NOT NULL), 1) > 0 @@ -110,12 +112,14 @@ details_sql: >- details_id_type: users severity: SEVERE introduction: > - Users who can enter data should be able to view their own data. This situation occurs when a user has - a data capture organisation unit which is not within the data view organisation unit hierarchy. + Users who can enter data should be able to view their own data. This check identifies users who have + a data capture organisation unit which is not within one of their data view organisation unit hierarchy. This can lead to a situation where a user can enter data, but cannot view the data that they have entered. recommendation: > Users should at least have access to view the data that they have entered. Using the results of the details SQL view, identify the affected users and the organisation units where they have access to enter data. The user should have at least all of their data capture organisation units specified in their data view organisation units. Alternatively, you can set the data view organisation units - to be at a higher level in the hierarchy. \ No newline at end of file + to be at a higher level in the hierarchy. Setting the data view organisation unit to a level above the data + capture organisation unit will allow the user to view all data entered at the data view organisation unit + and below. \ No newline at end of file diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityUsersCaptureOrgunitNotInDataView.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityUsersCaptureOrgunitNotInDataView.java index b39cbd132c4..d976ff23f99 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityUsersCaptureOrgunitNotInDataView.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityUsersCaptureOrgunitNotInDataView.java @@ -34,14 +34,15 @@ import java.util.Set; import org.hisp.dhis.http.HttpStatus; +import org.hisp.dhis.jsontree.JsonArray; import org.hisp.dhis.jsontree.JsonList; import org.hisp.dhis.jsontree.JsonString; import org.hisp.dhis.test.webapi.json.domain.JsonDataIntegrityDetails; import org.junit.jupiter.api.Test; /** - * Integrity check to identify users who have a data view organisation unit, but who cannot access - * data which they have possibly entered at a higher level of the hierarchy. {@see + * Integrity check to identify users who have a data view organisation unit which does not have + * access to their data entry organisation units. {@see * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/users/users_capture_ou_not_in_data_view_ou.yaml} * * @author Jason P. Pickering @@ -116,9 +117,10 @@ void testDataCaptureUnitInDataViewHierarchy() { + userRoleUid + "'}]}")); - // Note that there are already two users which exist due to the overall test setup, thus, five - // users in total. Only userB should be flagged. - assertHasDataIntegrityIssues(DETAILS_ID_TYPE, CHECK_NAME, 20, userBUid, "janedoe", null, true); + JsonArray users = GET("/users").content().getArray("users"); + assertEquals(4, users.size()); + // 1 user out of 4, thus 25% of users have data integrity issues + assertHasDataIntegrityIssues(DETAILS_ID_TYPE, CHECK_NAME, 25, userBUid, "janedoe", null, true); JsonDataIntegrityDetails details = getDetails(CHECK_NAME); JsonList issues = details.getIssues(); From e6cb5db9a3fee3fde1c622f400084b8e206fcd3d Mon Sep 17 00:00:00 2001 From: Mohamed Ameen Date: Thu, 28 Nov 2024 10:39:13 +0100 Subject: [PATCH 05/24] fix: [DHIS2-15272] Handle final expiring day fully (#19318) --- .../validation/validator/event/DateValidator.java | 7 +++++-- .../validator/event/DateValidatorTest.java | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidator.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidator.java index 58375dbc206..938d7a53413 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidator.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidator.java @@ -115,8 +115,11 @@ private void validateExpiryPeriodType(Reporter reporter, Event event, Program pr if (eventPeriod .getEndDate() - .toInstant() - .plus(ofDays(program.getExpiryDays())) + .toInstant() // This will be 00:00 time of the period end date. + .plus( + ofDays( + program.getExpiryDays() + + 1L)) // Extra day added to account for final 24 hours of expiring day .isBefore(Instant.now())) { reporter.addError(event, E1047, event); } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidatorTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidatorTest.java index 2c52c6713e1..5f5cf8caf99 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidatorTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DateValidatorTest.java @@ -236,6 +236,21 @@ void shouldPassValidationForEventWhenDateBelongsToPastPeriodWithZeroExpiryDays() assertIsEmpty(reporter.getErrors()); } + @Test + void shouldPassValidationForEventWhenDateBelongsPastEventPeriodButWithinExpiryDays() { + when(preheat.getProgram(MetadataIdentifier.ofUid(PROGRAM_WITH_REGISTRATION_ID))) + .thenReturn(getProgramWithRegistration(7)); + Event event = new Event(); + event.setEvent(UID.generate()); + event.setProgram(MetadataIdentifier.ofUid(PROGRAM_WITH_REGISTRATION_ID)); + event.setOccurredAt(sevenDaysAgo()); + event.setStatus(EventStatus.ACTIVE); + + validator.validate(reporter, bundle, event); + + assertIsEmpty(reporter.getErrors()); + } + @Test void shouldPassValidationForEventWhenScheduledDateBelongsToFuturePeriod() { when(preheat.getProgram(MetadataIdentifier.ofUid(PROGRAM_WITH_REGISTRATION_ID))) From d16f3979b84b6b97f13f0cb409af9d08b6fb76a1 Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Thu, 28 Nov 2024 12:30:47 +0100 Subject: [PATCH 06/24] Fix invalid quoting on ou column [DHIS-16705] (#19329) --- .../src/main/java/org/hisp/dhis/analytics/OrgUnitField.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/OrgUnitField.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/OrgUnitField.java index f16cbb92e06..a6082dd3a33 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/OrgUnitField.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/OrgUnitField.java @@ -223,7 +223,7 @@ private String ouQuote(String tableAlias, String col, boolean noColumnAlias) { + ((noColumnAlias) ? "" : " as " + col); } - return quote(tableAlias, col); + return sqlBuilder.quote(tableAlias, col); } @Override From 4f4187f1accd064121d9fc8876e630c60b7b82fa Mon Sep 17 00:00:00 2001 From: Jan Bernitt Date: Fri, 29 Nov 2024 09:00:04 +0100 Subject: [PATCH 07/24] fix: OpenAPI - object list extraction [DHIS2-17200] (#19320) * fix: parameter type variables, property field vs method deduplication based on Java property name * fix: remove non-public object list parameters form OpenAPI * chore: adds descriptions to all object list parameters * test: adds controller tests for the new and fixed OpenAPI features --- .../OrganisationUnitService.java | 3 +- .../DefaultOrganisationUnitService.java | 5 +- .../hisp/dhis/query/GetObjectListParams.java | 56 ++++++++++-- .../org/hisp/dhis/query/GetObjectParams.java | 5 ++ .../user/hibernate/HibernateUserStore.java | 2 + .../controller/OpenApiControllerTest.java | 59 +++++++++++++ .../AbstractFullReadOnlyController.java | 8 +- .../MessageConversationController.java | 5 ++ .../DataElementOperandController.java | 8 ++ .../dimension/DimensionController.java | 2 +- .../controller/event/ProgramController.java | 2 + .../TrackedEntityAttributeController.java | 5 ++ .../OrganisationUnitController.java | 86 ++++++++++++++++--- .../controller/user/UserController.java | 57 ++++++++++-- .../controller/user/UserRoleController.java | 5 ++ .../validation/ValidationRuleController.java | 5 ++ .../dhis/webapi/openapi/ApiExtractor.java | 62 +++++++++---- .../dhis/webapi/openapi/OpenApiObject.java | 2 + .../hisp/dhis/webapi/openapi/Property.java | 44 +++++++--- 19 files changed, 352 insertions(+), 69 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java index f76dcab600e..5b3dde26e68 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java @@ -35,6 +35,7 @@ import javax.annotation.Nonnull; import org.hisp.dhis.common.UID; import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.feedback.BadRequestException; import org.hisp.dhis.hierarchy.HierarchyViolationException; import org.hisp.dhis.program.Program; import org.hisp.dhis.user.User; @@ -304,7 +305,7 @@ List getOrganisationUnitsAtLevels( * @return the count of member OrganisationUnits. */ Long getOrganisationUnitHierarchyMemberCount( - OrganisationUnit parent, Object member, String collectionName); + OrganisationUnit parent, Object member, String collectionName) throws BadRequestException; OrganisationUnitDataSetAssociationSet getOrganisationUnitDataSetAssociationSet(User user); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java index 280d0e6c2ec..7a3bf1cc9ee 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java @@ -55,6 +55,7 @@ import org.hisp.dhis.configuration.ConfigurationService; import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.expression.ExpressionService; +import org.hisp.dhis.feedback.BadRequestException; import org.hisp.dhis.hierarchy.HierarchyViolationException; import org.hisp.dhis.organisationunit.comparator.OrganisationUnitLevelComparator; import org.hisp.dhis.setting.UserSettings; @@ -354,7 +355,9 @@ public List getOrganisationUnitsViolatingExclusiveGroupSets() @Override @Transactional(readOnly = true) public Long getOrganisationUnitHierarchyMemberCount( - OrganisationUnit parent, Object member, String collectionName) { + OrganisationUnit parent, Object member, String collectionName) throws BadRequestException { + if (!collectionName.matches("[a-zA-Z0-9_]{1,30}")) + throw new BadRequestException("Not a valid property name: " + collectionName); return organisationUnitStore.getOrganisationUnitHierarchyMemberCount( parent, member, collectionName); } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectListParams.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectListParams.java index fb440e4aacb..f7c387b6eb3 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectListParams.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectListParams.java @@ -40,6 +40,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; +import org.hisp.dhis.common.OpenApi; /** * Base for parameters supported by CRUD {@code CRUD.getObjectList}. @@ -51,25 +52,62 @@ @EqualsAndHashCode(callSuper = true) public class GetObjectListParams extends GetObjectParams { + @OpenApi.Description( + """ + Filter results using `filter={property}:{operator}[:{value}]` expressions as described in detail in + [Metadata-object-filter](https://docs.dhis2.org/en/develop/using-the-api/dhis-core-version-master/metadata.html#webapi_metadata_object_filter). + """) @JsonProperty("filter") @CheckForNull List filters; + @OpenApi.Description( + """ + Adds ordering to the result list. + Case-sensitive: `{property}:asc`, `{property}:desc` + Case-insensitive: `{property}:iasc`, `{property}:idesc` + Only supports properties that are both persisted and simple. + """) @JsonProperty("order") @CheckForNull List orders; - @JsonProperty Junction.Type rootJunction = Junction.Type.AND; + @OpenApi.Description("Combine `filter`s with `AND` (default) or `OR` logic.") + @JsonProperty + Junction.Type rootJunction = Junction.Type.AND; - @JsonProperty boolean paging = true; - @JsonProperty int page = 1; - @JsonProperty int pageSize = 50; + @OpenApi.Description( + "Use paging controlled by `page` and `pageSize` or return all matches (use with caution).") + @JsonProperty + boolean paging = true; - /** - * A special filter that matches the query term against the ID and code (equals) and against name - * (contains). Can be used in combination with regular filters. - */ - @JsonProperty String query; + @OpenApi.Description("The page number to show.") + @JsonProperty + int page = 1; + + @OpenApi.Description("The maximum number of elements on a page.") + @JsonProperty + int pageSize = 50; + + @OpenApi.Description( + """ + Adds a filter equivalent to the following three `filter`s combined _OR_ (independent of `rootJunction`): + `id:eq:{query}`, `code:eq:{query}`, `name:like:{query}`. Can be used in addition to regular `filter`s. + """) + @JsonProperty + String query; + + @OpenApi.Ignore + @CheckForNull + public List getFilters() { + return filters; + } + + @OpenApi.Ignore + @CheckForNull + public List getOrders() { + return orders; + } @Nonnull @JsonIgnore diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectParams.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectParams.java index 484ca06890b..f086eb47355 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectParams.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/GetObjectParams.java @@ -60,6 +60,11 @@ public class GetObjectParams { */ private static final String FIELD_SPLIT = ",(?![^\\[\\]]*\\]|[^\\(\\)]*\\)|([a-zA-Z0-9]+,?)+\\))"; + @OpenApi.Description( + """ + Limit the response to specific field(s).\s + See [Metadata-field-filter](https://docs.dhis2.org/en/develop/using-the-api/dhis-core-version-master/metadata.html#webapi_metadata_field_filter). + """) @OpenApi.Shared.Inline @OpenApi.Property(OpenApi.PropertyNames[].class) @JsonProperty diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/user/hibernate/HibernateUserStore.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/user/hibernate/HibernateUserStore.java index 688c3a76c70..2ab8d5cb73b 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/user/hibernate/HibernateUserStore.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/user/hibernate/HibernateUserStore.java @@ -304,6 +304,8 @@ private Query getUserQuery(UserQueryParams params, List orders, Query hql += hlp.whereAnd() + " u.selfRegistered = true "; } + // TODO(JB) shoundn't UserInvitationStatus.NONE match "u.invitation = false" and null mean no + // filter at all? if (UserInvitationStatus.ALL.equals(params.getInvitationStatus())) { hql += hlp.whereAnd() + " u.invitation = true "; } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/OpenApiControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/OpenApiControllerTest.java index ed4040d58fb..2830af71000 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/OpenApiControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/OpenApiControllerTest.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.webapi.controller; +import static java.util.stream.Collectors.toSet; import static org.hisp.dhis.http.HttpClientAdapter.Accept; import static org.hisp.dhis.test.utils.Assertions.assertContains; import static org.hisp.dhis.test.utils.Assertions.assertGreaterOrEqual; @@ -43,10 +44,19 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Set; +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonMap; import org.hisp.dhis.jsontree.JsonMixed; import org.hisp.dhis.jsontree.JsonNodeType; import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.jsontree.JsonString; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.test.webapi.H2ControllerIntegrationTestBase; +import org.hisp.dhis.webapi.openapi.OpenApiObject; +import org.hisp.dhis.webapi.openapi.OpenApiObject.ParameterObject; +import org.hisp.dhis.webapi.openapi.OpenApiObject.ResponseObject; +import org.hisp.dhis.webapi.openapi.OpenApiObject.SchemaObject; import org.junit.jupiter.api.Test; import org.openapitools.codegen.DefaultGenerator; import org.openapitools.codegen.config.CodegenConfigurator; @@ -151,6 +161,55 @@ void testGetOpenApiDocument_DefaultValue() { assertEquals(50, pageSize.getNumber("schema.default").integer()); } + /** Check shared parameter objects handling works */ + @Test + void testGetOpenApiDocument_ParameterObjects() { + OpenApiObject doc = + GET("/openapi/openapi.json?scope=entity:OrganisationUnit") + .content() + .as(OpenApiObject.class); + JsonList parameters = + doc.$paths().get("/api/organisationUnits/").get().parameters(); + Set allRefs = + parameters.stream() + .map(p -> p.getString("$ref")) + .filter(JsonValue::exists) + .map(JsonString::string) + .collect(toSet()); + // check one of each group to make sure the inheritance handling works as expected + assertTrue( + allRefs.containsAll( + Set.of( + "#/components/parameters/GetObjectListParams.filter", + "#/components/parameters/GetOrganisationUnitObjectListParams.level", + "#/components/parameters/GetObjectParams.defaults"))); + // check "fields" is inlined (no reference) as it depend on the entity type + assertTrue(parameters.stream().anyMatch(p -> "fields".equals(p.getString("name").string()))); + } + + /** Tests the "generics" handling of object list response */ + @Test + void testGetOpenApiDocument_GetObjectListResponse() { + OpenApiObject doc = + GET("/openapi/openapi.json?scope=entity:OrganisationUnit") + .content() + .as(OpenApiObject.class); + ResponseObject response = + doc.$paths().get("/api/organisationUnits/").get().responses().get("200"); + JsonMap properties = + response.content().get("application/json").schema().properties(); + + assertEquals( + Set.of("pager", "organisationUnits"), + properties.keys().collect(toSet()), + "there should only be a pager and an entity list property"); + + SchemaObject listSchema = properties.get("organisationUnits"); + assertEquals("array", listSchema.$type()); + assertEquals( + "#/components/schemas/OrganisationUnit", listSchema.items().getString("$ref").string()); + } + @Test void testGetOpenApiDocument_ReadOnly() { JsonObject doc = GET("/openapi/openapi.json?scope=entity.JobConfiguration").content(); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AbstractFullReadOnlyController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AbstractFullReadOnlyController.java index d0f6cad51d7..9d88e596e41 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AbstractFullReadOnlyController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AbstractFullReadOnlyController.java @@ -50,7 +50,6 @@ import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; -import lombok.Value; import org.hisp.dhis.attribute.AttributeService; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.common.DhisApiVersion; @@ -159,16 +158,15 @@ protected void addProgrammaticFilters(Consumer add) {} // GET Full // -------------------------------------------------------------------------- - @Value @OpenApi.Shared(value = false) - protected static class ObjectListResponse { + protected static class GetObjectListResponse { @OpenApi.Property Pager pager; @OpenApi.Property(name = "path$", value = OpenApi.EntityType[].class) List entries; } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping public @ResponseBody ResponseEntity> getObjectList( P params, HttpServletResponse response, @CurrentUser UserDetails currentUser) @@ -507,7 +505,7 @@ protected void modifyGetObjectList(P params, Query query) { // by default: nothing special to do } - protected void getEntityListPostProcess(P params, List entities) {} + protected void getEntityListPostProcess(P params, List entities) throws BadRequestException {} private long countGetObjectList(P params, List additionalFilters) throws BadRequestException { diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/MessageConversationController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/MessageConversationController.java index e4a5ad88d55..fa0c1d28d26 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/MessageConversationController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/MessageConversationController.java @@ -122,7 +122,12 @@ public class MessageConversationController @Data @EqualsAndHashCode(callSuper = true) public static final class GetMessageConversationObjectListParams extends GetObjectListParams { + @OpenApi.Description( + "Adds a _fuzzy_ search filter for the query term in subject, message text or sender name") String queryString; + + @OpenApi.Description( + "The operator used when using the `queryString` filter (default is `token`)") String queryOperator; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandController.java index 61a5b860dbe..27c4849b915 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandController.java @@ -85,10 +85,18 @@ public class DataElementOperandController { @Data @EqualsAndHashCode(callSuper = true) + @OpenApi.Property public static final class GetDataElementOperandObjectListParams extends GetObjectListParams { + @OpenApi.Description( + "When set all existing operands are the basis of the result list (takes precedence).") boolean persisted; + + @OpenApi.Description( + "Whether to include totals when loading operands by `dataSet` or data element groups from `filter`s.") boolean totals; + @OpenApi.Description( + "When set the operands linked to the specified dataset are the basis for the result list.") @OpenApi.Property({UID.class, DataSet.class}) String dataSet; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dimension/DimensionController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dimension/DimensionController.java index a90fc377956..6c7255226ce 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dimension/DimensionController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/dimension/DimensionController.java @@ -123,7 +123,7 @@ protected DimensionalObject getEntity(String uid) throws NotFoundException { * DimensionalObject} and there is no specific {@link InternalHibernateGenericStore} to retrieve * them from. */ - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @Override @GetMapping public @ResponseBody ResponseEntity> getObjectList( diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramController.java index b1b3a4434d5..015385051d4 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramController.java @@ -77,6 +77,8 @@ public class ProgramController @Data @EqualsAndHashCode(callSuper = true) public static class GetProgramObjectListParams extends GetObjectListParams { + @OpenApi.Description( + "Limit the results to programs accessible to the current user based on data sharing read access instead of metadata sharing read access.") boolean userFilter; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/TrackedEntityAttributeController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/TrackedEntityAttributeController.java index fd0d5213e8a..43f9c0e21ec 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/TrackedEntityAttributeController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/TrackedEntityAttributeController.java @@ -87,6 +87,11 @@ public class TrackedEntityAttributeController @Data @EqualsAndHashCode(callSuper = true) public static final class GetTrackedEntityAttributeObjectListParams extends GetObjectListParams { + @OpenApi.Description( + """ + Limits the results to searchable attributes of type `TEXT`, `LONG_TEXT`, `PHONE_NUMBER`, `EMAIL`, `USERNAME`, `URL` + as well as unique attributes. Can be combined with further `filter`s. + """) boolean indexableOnly; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/organisationunit/OrganisationUnitController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/organisationunit/OrganisationUnitController.java index bf2e39ce0d0..6d6ca6fdffa 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/organisationunit/OrganisationUnitController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/organisationunit/OrganisationUnitController.java @@ -102,11 +102,8 @@ public class OrganisationUnitController OrganisationUnit, OrganisationUnitController.GetOrganisationUnitObjectListParams> { @Autowired private OrganisationUnitService organisationUnitService; - @Autowired private VersionService versionService; - @Autowired private OrgUnitSplitService orgUnitSplitService; - @Autowired private OrgUnitMergeService orgUnitMergeService; @ResponseStatus(HttpStatus.OK) @@ -120,20 +117,80 @@ public class OrganisationUnitController @Data @EqualsAndHashCode(callSuper = true) + @OpenApi.Property public static final class GetOrganisationUnitObjectListParams extends GetObjectListParams { + @OpenApi.Description( + """ + When set for each organisation unit in the result list a count is added as property `memberCount`. + This count is the number of organisation units in the unit's subtree (including itself) where + the `memberObject` is a member of the relation defined by the `memberCollection` property. + Use with caution, as this is an expensive operation. + """) @OpenApi.Property(UID.class) String memberObject; + @OpenApi.Description( + """ + Name of the organisation unit collection property to use when checking of the `memberObject` is a member. + For example, `groups`, `dataSets`, `programs`, `users`, `categoryOptions`. + """) String memberCollection; - Integer parentLevel; + + @OpenApi.Ignore Integer parentLevel; + + @OpenApi.Description( + """ + Limits results to organisation units on the given level (absolute starting with 1 for the root). + When used for list relative to a parent this is the level relative to the parent level where `level=1` are + all direct children of the parent. + Can be combined with further `filter`s and/or one of the `withinUser*`/`userOnly*` limitations. + """) Integer level; + + @OpenApi.Description( + """ + Limits results to organisation units on the given level or above (absolute starting with 1 for the root). + Can be combined with further `filter`s and/or one of the `withinUser*`/`userOnly*` limitations. + """) Integer maxLevel; + + @OpenApi.Description( + """ + Limits result to organisation units for which current user has data capture privileges. + Can be combined with further `filter`s and/or one of the `level`/`maxLevel` limitations. + """) boolean withinUserHierarchy; + + @OpenApi.Description( + """ + Limits result to organisation units for which the current user has search privileges. + Can be combined with further `filter`s and/or one of the `level`/`maxLevel` limitations. + """) boolean withinUserSearchHierarchy; + + @OpenApi.Description( + """ + Limits result to organisation units that are explicitly listed in the current user's data capture set (excluding any non-listed children). + Can be combined with further `filter`s and/or one of the `level`/`maxLevel` limitations. + """) boolean userOnly; + + @OpenApi.Description( + """ + Limits result to organisation units that are explicitly listed in the current user's data view set (excluding any non-listed children). + Can be combined with further `filter`s and/or one of the `level`/`maxLevel` limitations. + """) boolean userDataViewOnly; + + @OpenApi.Description( + """ + Limits result to organisation units that are explicitly listed in the current user's data view set (excluding any non-listed children) + with fallback to the user's data capture set if the data view set is empty. + Can be combined with further `filter`s and/or one of the `level`/`maxLevel` limitations. + """) boolean userDataViewFallback; + @OpenApi.Description("Shorthand equivalent for the URL parameter `order=level:asc`.") boolean levelSorted; } @@ -146,7 +203,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return ok("Organisation units merged"); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping(value = "/{uid}", params = "includeChildren=true") public @ResponseBody ResponseEntity> getIncludeChildren( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -157,7 +214,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getChildren(uid, params, response, currentUser); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping("/{uid}/children") public @ResponseBody ResponseEntity> getChildren( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -173,7 +230,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getObjectListWith(params, response, currentUser, children); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping(value = "/{uid}", params = "level") public @ResponseBody ResponseEntity> getObjectWithLevel( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -185,7 +242,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getChildrenWithLevel(uid, level, params, response, currentUser); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping(value = "/{uid}/children", params = "level") public @ResponseBody ResponseEntity> getChildrenWithLevel( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -200,7 +257,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getObjectListWith(params, response, currentUser, childrenWithLevel); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping(value = "/{uid}", params = "includeDescendants=true") public @ResponseBody ResponseEntity> getIncludeDescendants( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -211,7 +268,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getDescendants(uid, params, response, currentUser); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping("/{uid}/descendants") public @ResponseBody ResponseEntity> getDescendants( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -223,7 +280,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getObjectListWith(params, response, currentUser, List.of(descendants)); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping(value = "/{uid}", params = "includeAncestors=true") public @ResponseBody ResponseEntity> getIncludeAncestors( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -234,7 +291,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getAncestors(uid, params, response, currentUser); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping("/{uid}/ancestors") public @ResponseBody ResponseEntity> getAncestors( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -252,7 +309,7 @@ public static final class GetOrganisationUnitObjectListParams extends GetObjectL return getObjectListWith(params, response, currentUser, List.of(ancestors)); } - @OpenApi.Response(ObjectListResponse.class) + @OpenApi.Response(GetObjectListResponse.class) @GetMapping("/{uid}/parents") public @ResponseBody ResponseEntity> getParents( @OpenApi.Param(UID.class) @PathVariable("uid") String uid, @@ -323,7 +380,8 @@ protected List getAdditionalFilters(GetOrganisationUnitObjectListPara @Override protected void getEntityListPostProcess( - GetOrganisationUnitObjectListParams params, List entities) { + GetOrganisationUnitObjectListParams params, List entities) + throws BadRequestException { String memberObject = params.getMemberObject(); String memberCollection = params.getMemberCollection(); if (memberObject != null && memberCollection != null) { diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserController.java index ce8be4be648..72aad3fcd1a 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserController.java @@ -152,21 +152,64 @@ public class UserController @Data @EqualsAndHashCode(callSuper = true) public static final class GetUserObjectListParams extends GetObjectListParams { + @OpenApi.Description( + "Limits results to users with the given phone number (shorthand for `filter=phoneNumber:eq:{value}`)") String phoneNumber; + + @OpenApi.Description( + "Limits results to users that are members of a group the current user can manage.") boolean canManage; + + @OpenApi.Description( + "Limits result to users that have no authority the current user doesn't have as well.") boolean authSubset; + + @OpenApi.Description( + "Limits results to users that were logged in on or after the given date(-time) (shorthand for `filter=lastLogin:ge:{date}`).") Date lastLogin; + + @OpenApi.Description( + "Limits results to users that haven't logged in for at least this number of months.") Integer inactiveMonths; + + @OpenApi.Description( + "Limits results to users that haven't logged in since this date(-time) (shorthand for `filter=lastLogin:lt:{date}`).") Date inactiveSince; + + @OpenApi.Description( + "Limits results to users that have self registered (shorthand for `filter=selfRegistered:eq:true`)") boolean selfRegistered; + + @OpenApi.Description( + """ + Limits results to users that with the provided invitation status + (`ALL` equals `filter=invitation:eq:true`, `EXPIRED` also requires the invitation to have expired by now).""") UserInvitationStatus invitationStatus; + + @OpenApi.Description("Shorthand for `orgUnitBoundary=DATA_CAPTURE`") boolean userOrgUnits; - boolean includeChildren; + + @OpenApi.Description( + """ + Limits results to users that have a common organisation unit connection with the current user. + The `orgUnitBoundary` determines if the data capture, data view or search sets are considered. + When `includeChildren=true` is used the comparison includes the subtree of all units in the compared set. + """) UserOrgUnitType orgUnitBoundary; + @OpenApi.Description("See `orgUnitBoundary`") + boolean includeChildren; + + @OpenApi.Description( + """ + Limits results to users that have data capture connection to the given organisation unit. + The compared set can be changed using `orgUnitBoundary`. + """) @OpenApi.Property({UID.class, OrganisationUnit.class}) String ou; + @OpenApi.Description( + "Shorthand for `canManage=true` + `authSubset=true` (takes precedence over individual parameters)") boolean manage; @JsonIgnore @@ -192,11 +235,6 @@ protected List getPreQueryMatches(GetUserObjectListParams params) throws Co if (!params.isUsingAnySpecialFilters()) return null; UserQueryParams queryParams = toUserQueryParams(params); - String ou = params.getOu(); - if (ou != null) { - queryParams.addOrganisationUnit(organisationUnitService.getOrganisationUnit(ou)); - } - if (params.isManage()) { queryParams.setCanManage(true); queryParams.setAuthSubset(true); @@ -217,7 +255,12 @@ private UserQueryParams toUserQueryParams(GetUserObjectListParams params) { res.setInvitationStatus(params.getInvitationStatus()); res.setUserOrgUnits(params.isUserOrgUnits()); res.setIncludeOrgUnitChildren(params.isIncludeChildren()); - res.setOrgUnitBoundary(params.getOrgUnitBoundary()); + String ou = params.getOu(); + if (ou != null) { + res.addOrganisationUnit(organisationUnitService.getOrganisationUnit(ou)); + } + UserOrgUnitType boundary = params.getOrgUnitBoundary(); + if (boundary != null) res.setOrgUnitBoundary(boundary); return res; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserRoleController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserRoleController.java index 612103f1a06..1f68c506ebe 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserRoleController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/user/UserRoleController.java @@ -66,6 +66,11 @@ public class UserRoleController @Data @EqualsAndHashCode(callSuper = true) public static final class GetUserRoleObjectListParams extends GetObjectListParams { + @OpenApi.Description( + """ + Limits results to those roles that the current user can issue/grant to other users. + Can be combined with further `filter`s. + """) boolean canIssue; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/validation/ValidationRuleController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/validation/ValidationRuleController.java index 5edbeef3f17..090a4e57984 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/validation/ValidationRuleController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/validation/ValidationRuleController.java @@ -73,6 +73,11 @@ public class ValidationRuleController @Data @EqualsAndHashCode(callSuper = true) public static final class GetValidationRuleObjectListParams extends GetObjectListParams { + @OpenApi.Description( + """ + Limits results to validation rules that are connected to the given dataset. + Can be combined with further `filter`s. + """) @OpenApi.Property({UID.class, DataSet.class}) String dataSet; } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/ApiExtractor.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/ApiExtractor.java index 37c15a76cbb..4ea4dd7cabb 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/ApiExtractor.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/ApiExtractor.java @@ -50,6 +50,7 @@ import java.lang.reflect.Parameter; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; import java.lang.reflect.WildcardType; import java.util.ArrayList; import java.util.Arrays; @@ -467,8 +468,23 @@ private void extractParameters(Api.Endpoint endpoint, Set consumes) { requestBody.getDescription().setIfAbsent(extractDescription(p, p.getType())); Api.Schema type = extractSchema(endpoint, p.getParameterizedType()); consumes.forEach(mediaType -> requestBody.getConsumes().putIfAbsent(mediaType, type)); - } else if (isParams(p)) { - extractParams(endpoint, p.getType()); + } else if (isOfSuitableParamsObjectType(p)) { + if (isOfConstructableParamsObjectType(p.getParameterizedType())) { + extractParams(endpoint, p.getType()); + } else if (p.getParameterizedType() instanceof TypeVariable v + && v.getName().equals("P")) { + // Note: this makes simplified assumptions and works using an approximation of the type + // variable substitution + // it only works under the assumption that the parameter is called P and is always the 2nd + // class level type variable + Type superclass = endpoint.getIn().getSource().getGenericSuperclass(); + if (superclass instanceof ParameterizedType pts) { + Type actualType = pts.getActualTypeArguments()[1]; + if (isOfConstructableParamsObjectType(actualType)) { + extractParams(endpoint, (Class) actualType); + } + } + } } } } @@ -537,18 +553,29 @@ private void extractParams(Api.Endpoint endpoint, OpenApi.Params params) { private void extractParams(Api.Endpoint endpoint, Class paramsObject) { Collection properties = getProperties(paramsObject); OpenApi.Shared shared = paramsObject.getAnnotation(OpenApi.Shared.class); + boolean useDeclaringClass = shared != null && shared.name().isEmpty(); String sharedName = getSharedName(paramsObject, shared, null); if (sharedName != null) { Map, List> sharedParameters = api.getComponents().getParameters(); - boolean addShared = !sharedParameters.containsKey(paramsObject); + Set> addSharedTypes = + properties.stream().map(Property::getDeclaringClass).collect(toSet()); + // if the type is contained at this point the type has been analysed before and should be + // ignored + addSharedTypes.removeAll(sharedParameters.keySet()); properties.forEach( property -> { Api.Parameter parameter = extractParameter(endpoint, property); if (!parameter.getSource().isAnnotationPresent(OpenApi.Shared.Inline.class)) { - parameter.getSharedName().setValue(sharedName); - if (addShared) { + Class shardType = paramsObject; + String name = sharedName; + if (useDeclaringClass) { + shardType = property.getDeclaringClass(); + name = getSharedName(shardType, shared, null); + } + parameter.getSharedName().setValue(name); + if (addSharedTypes.contains(shardType)) { sharedParameters - .computeIfAbsent(paramsObject, key -> new ArrayList<>()) + .computeIfAbsent(shardType, key -> new ArrayList<>()) .add(parameter); } } @@ -864,18 +891,19 @@ private static Class getSubstitutedType(Api.Endpoint endpoint, Class type) /** * @return is this a parameter objects with properties which are parameters? */ - private static boolean isParams(Parameter source) { + private static boolean isOfSuitableParamsObjectType(Parameter source) { Class type = source.getType(); - if (type.isAnnotationPresent(OpenApi.Params.class)) { - return true; - } - if (type.isInterface() - || type.isEnum() - || IdentifiableObject.class.isAssignableFrom(type) - || source.getAnnotations().length > 0 - || source.isAnnotationPresent(OpenApi.Ignore.class) - || !(source.getParameterizedType() instanceof Class)) return false; - return stream(type.getDeclaredConstructors()).anyMatch(c -> c.getParameterCount() == 0); + if (type.isAnnotationPresent(OpenApi.Params.class)) return true; + return !type.isInterface() + && !type.isEnum() + && !IdentifiableObject.class.isAssignableFrom(type) + && source.getAnnotations().length == 0 + && !source.isAnnotationPresent(OpenApi.Ignore.class); + } + + private static boolean isOfConstructableParamsObjectType(Type source) { + return (source instanceof Class type) + && stream(type.getDeclaredConstructors()).anyMatch(c -> c.getParameterCount() == 0); } private static EndpointMapping getMapping(Method source) { diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/OpenApiObject.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/OpenApiObject.java index 1829fcffdde..11f3a98c122 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/OpenApiObject.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/OpenApiObject.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Stream; import org.hisp.dhis.common.OpenApi; @@ -292,6 +293,7 @@ default Set parameterNames() { return parameters().stream() .map(ParameterObject::resolve) .map(ParameterObject::name) + .filter(Objects::nonNull) .collect(toUnmodifiableSet()); } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/Property.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/Property.java index f2d24708d86..f26a5e42046 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/Property.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/openapi/Property.java @@ -74,6 +74,7 @@ class Property { private static final Map, List> PROPERTIES = new ConcurrentHashMap<>(); + @Nonnull Class declaringClass; @Nonnull String name; @Nonnull Type type; @Nonnull AnnotatedElement source; @@ -83,6 +84,7 @@ class Property { private Property(Field f) { this( + f.getDeclaringClass(), getPropertyName(f), getType(f, f.getGenericType()), f, @@ -93,6 +95,7 @@ private Property(Field f) { private Property(Method m, Field f) { this( + m.getDeclaringClass(), getPropertyName(m), getType(m, isSetter(m) ? m.getGenericParameterTypes()[0] : m.getGenericReturnType()), m, @@ -103,6 +106,7 @@ private Property(Method m, Field f) { private Property(JsonObject.Property p, Type type) { this( + p.in(), p.jsonName(), getType(p.source(), type), p.source(), @@ -125,27 +129,34 @@ private static List propertiesIn(Class object) { (method, field) -> add.accept(new Property(method, field)); boolean includeByDefault = isIncludeAllByDefault(object); - Map propertyFields = new HashMap<>(); - fieldsIn(object).forEach(field -> propertyFields.putIfAbsent(getPropertyName(field), field)); + Map fieldsByJavaPropertyName = new HashMap<>(); + fieldsIn(object) + .forEach(field -> fieldsByJavaPropertyName.putIfAbsent(getJavaPropertyName(field), field)); Set ignoredFields = - propertyFields.values().stream() + fieldsByJavaPropertyName.values().stream() .filter(f -> f.isAnnotationPresent(OpenApi.Ignore.class)) - .map(Property::getPropertyName) + .map(Property::getJavaPropertyName) .collect(Collectors.toSet()); - propertyFields.values().stream() + fieldsByJavaPropertyName.values().stream() .filter(Property::isProperty) .filter(f -> includeByDefault || isExplicitlyIncluded(f)) .forEach(addField); methodsIn(object) .filter(method -> Property.isGetter(method) || Property.isSetter(method)) .filter(Property::isExplicitlyIncluded) - .filter(method -> !ignoredFields.contains(getPropertyName(method))) - .forEach(method -> addMethod.accept(method, propertyFields.get(getPropertyName(method)))); + .filter(method -> !ignoredFields.contains(getJavaPropertyName(method))) + .forEach( + method -> + addMethod.accept( + method, fieldsByJavaPropertyName.get(getJavaPropertyName(method)))); if (properties.isEmpty() || includeByDefault) { methodsIn(object) .filter(Property::isGetter) - .filter(method -> !ignoredFields.contains(getPropertyName(method))) - .forEach(method -> addMethod.accept(method, propertyFields.get(getPropertyName(method)))); + .filter(method -> !ignoredFields.contains(getJavaPropertyName(method))) + .forEach( + method -> + addMethod.accept( + method, fieldsByJavaPropertyName.get(getJavaPropertyName(method)))); } return List.copyOf(properties.values()); } @@ -239,11 +250,7 @@ private static Type getType(AnnotatedElement source, Type type) { } private static String getPropertyName(T member) { - String name = member.getName(); - if (member instanceof Method) { - String prop = name.substring(name.startsWith("is") ? 2 : 3); - name = Character.toLowerCase(prop.charAt(0)) + prop.substring(1); - } + String name = getJavaPropertyName(member); OpenApi.Property oap = member.getAnnotation(OpenApi.Property.class); String nameOverride = oap == null ? "" : oap.name(); if (!nameOverride.isEmpty()) { @@ -254,6 +261,15 @@ private static String getPropertyName(T me return nameOverride.isEmpty() ? name : nameOverride; } + private static String getJavaPropertyName(T member) { + String name = member.getName(); + if (member instanceof Method) { + String prop = name.substring(name.startsWith("is") ? 2 : 3); + name = Character.toLowerCase(prop.charAt(0)) + prop.substring(1); + } + return name; + } + @CheckForNull private static Boolean isRequired(AnnotatedElement source, Type type) { JsonProperty a = source.getAnnotation(JsonProperty.class); From 6488d940cd83d004d441fcb9ab46244524475cb4 Mon Sep 17 00:00:00 2001 From: teleivo Date: Fri, 29 Nov 2024 10:32:51 +0100 Subject: [PATCH 08/24] fix: /tracker/events?dataElementIdScheme!=UID with relationships (#19341) --- .../dhis/tracker/export/event/JdbcEventStore.java | 6 ++---- .../export/IdSchemeExportControllerTest.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java index 8c67cb2780e..c2168d1916d 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java @@ -609,14 +609,12 @@ private String buildSql( PageParams pageParams, MapSqlParameterSource mapSqlParameterSource, User user) { - StringBuilder sqlBuilder = new StringBuilder("select "); + StringBuilder sqlBuilder = new StringBuilder("select *"); if (TrackerIdScheme.UID != queryParams.getIdSchemeParams().getDataElementIdScheme().getIdScheme()) { sqlBuilder.append( - "event.*, cm.*,eventdatavalue.value as ev_eventdatavalue, de.uid as de_uid, de.code as" + ", eventdatavalue.value as ev_eventdatavalue, de.uid as de_uid, de.code as" + " de_code, de.name as de_name, de.attributevalues as de_attributevalues"); - } else { - sqlBuilder.append("*"); } sqlBuilder.append(" from ("); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java index a1b532b49ca..63d56a47764 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java @@ -287,6 +287,21 @@ void shouldExportEventUsingNonUIDDataElementIdSchemeEvenIfItHasNoDataValues() { assertEquals("jxgFyJEMUPf", actual.getEvent()); } + @Test + void shouldExportEventUsingNonUIDDataElementIdSchemeIfItHasRelationships() { + Event event = get(Event.class, "pTzf9KYMk72"); + assertNotEmpty(event.getRelationshipItems(), "test expects an event with relationships"); + + JsonEvent actual = + GET( + "/tracker/events/{id}?fields=event,relationships&dataElementIdScheme=NAME", + event.getUid()) + .content(HttpStatus.OK) + .as(JsonEvent.class); + + assertEquals("pTzf9KYMk72", actual.getEvent()); + } + @Test void shouldExportEventsUsingNonUIDDataElementIdScheme() { Event event1 = get(Event.class, "QRYjLTiJTrA"); From 26443ba0308dc267b6451244ee5608a81b124001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 29 Nov 2024 12:15:31 +0100 Subject: [PATCH 09/24] refactor: Move StatementBuilder to expression parser [DHIS2-16705] (#19343) --- .../dhis/program/AnalyticsPeriodBoundaryType.java | 4 ++-- .../analytics/event/data/TimeFieldSqlRenderer.java | 6 +++--- .../table/JdbcEventAnalyticsTableManager.java | 2 +- .../JdbcTrackedEntityAnalyticsTableManager.java | 1 - ...ckedEntityEnrollmentsAnalyticsTableManager.java | 5 +++++ .../program/function/ProgramCountFunction.java | 2 +- .../program/function/ProgramMinMaxFunction.java | 2 +- .../dhis-support-expression-parser/pom.xml | 14 +++++++------- .../parser/expression/CommonExpressionVisitor.java | 6 +++--- .../statement/DefaultStatementBuilder.java} | 4 ++-- .../expression/statement}/StatementBuilder.java | 2 +- dhis-2/dhis-support/dhis-support-jdbc/pom.xml | 8 -------- 12 files changed, 26 insertions(+), 30 deletions(-) rename dhis-2/dhis-support/{dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java => dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java} (99%) rename dhis-2/dhis-support/{dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc => dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement}/StatementBuilder.java (99%) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundaryType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundaryType.java index 550c28cc2b1..6a069c56a97 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundaryType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundaryType.java @@ -57,11 +57,11 @@ public String getValue() { return value; } - public Boolean isEndBoundary() { + public boolean isEndBoundary() { return this == BEFORE_END_OF_REPORTING_PERIOD || this == BEFORE_START_OF_REPORTING_PERIOD; } - public Boolean isStartBoundary() { + public boolean isStartBoundary() { return this == AFTER_END_OF_REPORTING_PERIOD || this == AFTER_START_OF_REPORTING_PERIOD; } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java index 8cccb77fdc9..fa270fbd366 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/TimeFieldSqlRenderer.java @@ -51,15 +51,15 @@ import org.hisp.dhis.common.DateRange; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.db.sql.SqlBuilder; -import org.hisp.dhis.jdbc.PostgreSqlStatementBuilder; -import org.hisp.dhis.jdbc.StatementBuilder; +import org.hisp.dhis.parser.expression.statement.DefaultStatementBuilder; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; /** Provides methods targeting the generation of SQL statements for periods and time fields. */ public abstract class TimeFieldSqlRenderer { protected final SqlBuilder sqlBuilder; - protected final StatementBuilder statementBuilder = new PostgreSqlStatementBuilder(); + protected final StatementBuilder statementBuilder = new DefaultStatementBuilder(); protected TimeFieldSqlRenderer(SqlBuilder sqlBuilder) { this.sqlBuilder = sqlBuilder; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index e28f4314b7e..a67cc779d1b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -414,7 +414,6 @@ private List getColumns(Program program) { } } if (sqlBuilder.supportsDeclarativePartitioning()) { - // Add the year column required for declarative partitioning columns.add(getPartitionColumn()); } @@ -827,6 +826,7 @@ private List getDataYears( "fromDate", toMediumDate(params.getFromDate()))) : EMPTY; + String sql = replaceQualify( """ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index afbfdf81dbe..41ba840e7eb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -238,7 +238,6 @@ private Stream getAllTrackedEntityAttributes( TrackedEntityType trackedEntityType, Map> programsByTetUid) { if (programsByTetUid.containsKey(trackedEntityType.getUid())) { - return getAllTrackedEntityAttributesByPrograms( trackedEntityType, programsByTetUid.get(trackedEntityType.getUid())); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java index 07ef5fe4df1..fc9f6f1b510 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityEnrollmentsAnalyticsTableManager.java @@ -287,6 +287,11 @@ and ev.status in (${statuses})) \ invokeTimeAndLog(sql.toString(), "Populating table: '{}'", tableName); } + /** + * Returns a list of fixed columns. + * + * @return a list of {@link AnalyticsTableColumn}. + */ private List getFixedCols() { List columns = new ArrayList<>(); columns.addAll(FIXED_COLS); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java index 62a4978b05b..be852b9cfb3 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramCountFunction.java @@ -30,10 +30,10 @@ import java.util.Date; import org.hisp.dhis.analytics.AnalyticsConstants; import org.hisp.dhis.antlr.ParserExceptionWithoutContext; -import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; import org.hisp.dhis.parser.expression.ProgramExpressionParams; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.ProgramExpressionItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.dataitem.ProgramItemStageElement; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java index f37ae35271f..efa538adce0 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/function/ProgramMinMaxFunction.java @@ -29,10 +29,10 @@ import java.util.Date; import org.hisp.dhis.analytics.AnalyticsConstants; -import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.CommonExpressionVisitor; import org.hisp.dhis.parser.expression.ProgramExpressionParams; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.AnalyticsType; import org.hisp.dhis.program.ProgramExpressionItem; import org.hisp.dhis.program.ProgramIndicator; diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml b/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml index 731b7b59219..8f0f487bd98 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml +++ b/dhis-2/dhis-support/dhis-support-expression-parser/pom.xml @@ -23,10 +23,6 @@ org.hisp.dhis dhis-api - - org.hisp.dhis - dhis-support-jdbc - org.hisp.dhis dhis-support-system @@ -37,17 +33,21 @@ + + org.springframework + spring-core + org.apache.commons - commons-text + commons-lang3 org.apache.commons - commons-math3 + commons-text org.apache.commons - commons-lang3 + commons-math3 com.google.guava diff --git a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java index 78f832cc96d..2287dc0da86 100644 --- a/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/CommonExpressionVisitor.java @@ -46,9 +46,9 @@ import org.hisp.dhis.expression.ExpressionInfo; import org.hisp.dhis.expression.ExpressionParams; import org.hisp.dhis.i18n.I18n; -import org.hisp.dhis.jdbc.PostgreSqlStatementBuilder; -import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import org.hisp.dhis.parser.expression.statement.DefaultStatementBuilder; +import org.hisp.dhis.parser.expression.statement.StatementBuilder; import org.hisp.dhis.program.ProgramIndicatorService; import org.hisp.dhis.program.ProgramStageService; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; @@ -62,7 +62,7 @@ @Setter @Builder(toBuilder = true) public class CommonExpressionVisitor extends AntlrExpressionVisitor { - private final StatementBuilder statementBuilder = new PostgreSqlStatementBuilder(); + private final StatementBuilder statementBuilder = new DefaultStatementBuilder(); private IdentifiableObjectManager idObjectManager; diff --git a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java similarity index 99% rename from dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java rename to dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java index a5248d65eed..5afb97088bb 100644 --- a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/PostgreSqlStatementBuilder.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/DefaultStatementBuilder.java @@ -25,7 +25,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.jdbc; +package org.hisp.dhis.parser.expression.statement; import static java.lang.String.format; import static org.apache.commons.lang3.StringUtils.SPACE; @@ -50,7 +50,7 @@ * @author Lars Helge Overland */ @NoArgsConstructor -public class PostgreSqlStatementBuilder implements StatementBuilder { +public class DefaultStatementBuilder implements StatementBuilder { protected static final String QUOTE = "\""; protected static final String SINGLE_QUOTE = "'"; diff --git a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java similarity index 99% rename from dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java rename to dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java index eb239c2986a..e51f1ac6b58 100644 --- a/dhis-2/dhis-support/dhis-support-jdbc/src/main/java/org/hisp/dhis/jdbc/StatementBuilder.java +++ b/dhis-2/dhis-support/dhis-support-expression-parser/src/main/java/org/hisp/dhis/parser/expression/statement/StatementBuilder.java @@ -25,7 +25,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.jdbc; +package org.hisp.dhis.parser.expression.statement; import java.util.Date; import org.hisp.dhis.program.AnalyticsPeriodBoundary; diff --git a/dhis-2/dhis-support/dhis-support-jdbc/pom.xml b/dhis-2/dhis-support/dhis-support-jdbc/pom.xml index 31f716b1296..1f838a287c6 100644 --- a/dhis-2/dhis-support/dhis-support-jdbc/pom.xml +++ b/dhis-2/dhis-support/dhis-support-jdbc/pom.xml @@ -42,10 +42,6 @@ org.springframework spring-context - - org.springframework - spring-core - org.springframework spring-beans @@ -58,10 +54,6 @@ com.google.guava guava - - org.apache.commons - commons-lang3 - org.projectlombok lombok From 8aba326ed180849ad323ca9b7c91d9b29f4323cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 29 Nov 2024 12:46:29 +0100 Subject: [PATCH 10/24] refactor: Use primitive boolean over objects (#19344) --- .../dhis/program/AnalyticsPeriodBoundary.java | 16 ++++++++-------- .../dhis/program/ProgramExpressionService.java | 2 -- .../org/hisp/dhis/program/ProgramService.java | 2 -- .../dhis/program/ProgramStageSectionService.java | 6 ------ .../dhis/program/ProgramStageSectionStore.java | 1 - .../hisp/dhis/program/ProgramStageService.java | 2 -- .../org/hisp/dhis/program/ProgramStageStore.java | 1 - .../java/org/hisp/dhis/program/ProgramStore.java | 2 -- .../dhis/program/ProgramTempOwnerService.java | 3 --- .../hisp/dhis/program/ProgramTempOwnerStore.java | 3 --- 10 files changed, 8 insertions(+), 30 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundary.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundary.java index 0b3b940f9b2..4773a7d6b01 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundary.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/AnalyticsPeriodBoundary.java @@ -177,35 +177,35 @@ public Date getBoundaryDate(Date reportingStartDate, Date reportingEndDate) { return returnDate; } - public Boolean isCohortDateBoundary() { + public boolean isCohortDateBoundary() { return !isEventDateBoundary(); } - public Boolean isEnrollmentHavingEventDateCohortBoundary() { + public boolean isEnrollmentHavingEventDateCohortBoundary() { return boundaryTarget.startsWith(COHORT_HAVING_PROGRAM_STAGE_PREFIX); } - public Boolean isDataElementCohortBoundary() { + public boolean isDataElementCohortBoundary() { return boundaryTarget.startsWith(COHORT_HAVING_DATA_ELEMENT_PREFIX); } - public Boolean isAttributeCohortBoundary() { + public boolean isAttributeCohortBoundary() { return boundaryTarget.startsWith(COHORT_HAVING_ATTRIBUTE_PREFIX); } - public Boolean isEventDateBoundary() { + public boolean isEventDateBoundary() { return boundaryTarget.equals(AnalyticsPeriodBoundary.EVENT_DATE); } - public Boolean isEnrollmentDateBoundary() { + public boolean isEnrollmentDateBoundary() { return boundaryTarget.equals(AnalyticsPeriodBoundary.ENROLLMENT_DATE); } - public Boolean isIncidentDateBoundary() { + public boolean isIncidentDateBoundary() { return boundaryTarget.equals(AnalyticsPeriodBoundary.INCIDENT_DATE); } - public Boolean isScheduledDateBoundary() { + public boolean isScheduledDateBoundary() { return boundaryTarget.equals(AnalyticsPeriodBoundary.SCHEDULED_DATE); } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramExpressionService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramExpressionService.java index 65a86a7ae46..e1ccca53b97 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramExpressionService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramExpressionService.java @@ -45,8 +45,6 @@ * @author Chau Thu Tran */ public interface ProgramExpressionService { - String ID = ProgramExpressionService.class.getName(); - String INVALID_CONDITION = "Expression is not well-formed"; /** diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramService.java index cf03dd1db65..bb06946f6f7 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramService.java @@ -40,8 +40,6 @@ * @author Abyot Asalefew */ public interface ProgramService { - String ID = ProgramService.class.getName(); - /** * Adds an {@link Program} * diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionService.java index c5d01b9b1f8..9e0b3b7ea12 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionService.java @@ -31,12 +31,6 @@ * @author Chau Thu Tran */ public interface ProgramStageSectionService { - String ID = ProgramStageSection.class.getName(); - - // ------------------------------------------------------------------------- - // ProgramStageSection - // ------------------------------------------------------------------------- - /** * Adds an {@link ProgramStageSection} * diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionStore.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionStore.java index 03a45bf88c7..f9a3b7456c7 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionStore.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageSectionStore.java @@ -35,6 +35,5 @@ * @author Chau Thu Tran */ public interface ProgramStageSectionStore extends IdentifiableObjectStore { - List getAllByDataElement(List dataElements); } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageService.java index 6d56b5dbf85..7eb2b9d52c2 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageService.java @@ -34,8 +34,6 @@ * @author Abyot Asalefew */ public interface ProgramStageService { - String ID = ProgramStageService.class.getName(); - // ------------------------------------------------------------------------- // ProgramStage // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageStore.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageStore.java index 6593dbe11b9..a31cb13e09d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageStore.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStageStore.java @@ -35,7 +35,6 @@ * @author Chau Thu Tran */ public interface ProgramStageStore extends IdentifiableObjectStore { - /** * Retrieve a program stage by name and a program * diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStore.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStore.java index 9daa925b996..04c4c145876 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStore.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramStore.java @@ -37,8 +37,6 @@ * @author Chau Thu Tran */ public interface ProgramStore extends IdentifiableObjectStore { - String ID = ProgramStore.class.getName(); - /** * Get {@link Program} by a type * diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerService.java index 8dfd51263e8..25c886410d1 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerService.java @@ -33,9 +33,6 @@ * @author Ameen Mohamed */ public interface ProgramTempOwnerService { - - String ID = ProgramTempOwnerService.class.getName(); - /** * Adds program temp owner * diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerStore.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerStore.java index 11fb884aee6..9cc196696f5 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerStore.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/program/ProgramTempOwnerStore.java @@ -33,9 +33,6 @@ * @author Ameen Mohamed */ public interface ProgramTempOwnerStore { - - String ID = ProgramTempOwnerStore.class.getName(); - /** * Adds program temp owner record * From dd7a71cdd4383e383fb45f451dc097e351dda82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Fri, 29 Nov 2024 14:04:46 +0100 Subject: [PATCH 11/24] fix: Remove unnecessary join for analytics tables [DHIS2-16705] (#19345) --- .../dhis/analytics/table/EventAnalyticsColumn.java | 2 +- .../table/JdbcEventAnalyticsTableManager.java | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java index 4ca92d43f72..e5ad325bdf8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/EventAnalyticsColumn.java @@ -83,7 +83,7 @@ public final class EventAnalyticsColumn { .name(EventAnalyticsColumnName.AO_COLUMN_NAME) .dataType(CHARACTER_11) .nullable(NOT_NULL) - .selectExpression("ao.uid") + .selectExpression("acs.categoryoptioncombouid") .build(); private static final AnalyticsTableColumn ENROLLMENT_DATE = AnalyticsTableColumn.builder() diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index a67cc779d1b..ae382bd86fe 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -340,7 +340,6 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ inner join ${programstage} ps on ev.programstageid=ps.programstageid \ inner join ${program} pr on en.programid=pr.programid and en.deleted = false \ - inner join ${categoryoptioncombo} ao on ev.attributeoptioncomboid=ao.categoryoptioncomboid \ left join ${trackedentity} te on en.trackedentityid=te.trackedentityid and te.deleted = false \ left join ${organisationunit} registrationou on te.organisationunitid=registrationou.organisationunitid \ inner join ${organisationunit} ou on ev.organisationunitid=ou.organisationunitid \ @@ -499,6 +498,10 @@ private List getColumnForDataElement( String sql = getSelectForInsert(dataElement, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); + if (withLegendSet) { + return getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataFilterClause); + } + if (dataElement.getValueType().isOrganisationUnit()) { columns.addAll(getColumnForOrgUnitDataElement(dataElement, dataFilterClause)); } @@ -512,9 +515,7 @@ private List getColumnForDataElement( .skipIndex(skipIndex) .build()); - return withLegendSet - ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataFilterClause) - : columns; + return columns; } /** @@ -654,7 +655,7 @@ private List getColumnForAttributeWithLegendSet( .selectExpression(sql) .build(); }) - .collect(toList()); + .toList(); } /** From 82047c63b94b2b64e01e95a195d5a2fbf1ae745a Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Sat, 30 Nov 2024 17:32:37 +0700 Subject: [PATCH 12/24] feat: Data integrity check for empty custom data entry forms (#19332) * Add integrity check for empty custom data entry forms * Add tests for program and program stages * Remove unused variable --- .../main/resources/data-integrity-checks.yaml | 3 +- ...atasets_custom_data_entry_forms_empty.yaml | 59 ++++ ...programs_custom_data_entry_form_empty.yaml | 79 +++++ .../DataIntegrityYamlReaderTest.java | 2 +- .../src/main/resources/i18n_global.properties | 2 + ...ptyCustomDataEntryFormsControllerTest.java | 332 ++++++++++++++++++ 6 files changed, 475 insertions(+), 2 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_custom_data_entry_forms_empty.yaml create mode 100644 dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/programs/programs_custom_data_entry_form_empty.yaml create mode 100644 dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityEmptyCustomDataEntryFormsControllerTest.java diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml index f8b538046fd..a316fe406c4 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks.yaml @@ -20,6 +20,7 @@ checks: - categories/category_option_combinations_no_names.yaml - datasets/datasets_empty.yaml - datasets/datasets_not_assigned_to_org_units.yaml + - datasets/datasets_custom_data_entry_forms_empty.yaml - orgunits/compulsory_orgunit_groups.yaml - orgunits/orgunit_open_date_gt_closed_date.yaml - orgunits/orgunits_multiple_spaces.yaml @@ -37,6 +38,7 @@ checks: - option_sets/unused_option_sets.yaml - option_sets/option_sets_wrong_sort_order.yaml - option_sets/option_groups_empty.yaml + - programs/programs_custom_data_entry_form_empty.yaml - program_indicators/program_indicators_without_expression.yaml - program_rules/program_rules_no_action.yaml - program_rules/program_rules_without_condition.yaml @@ -49,7 +51,6 @@ checks: - analytical_objects/push_analysis_no_recipients.yaml - data_elements/aggregate_des_no_groups.yaml - data_elements/aggregate_des_abandoned.yaml - - data_elements/aggregate_des_inconsistent_agg_operator.yaml - data_elements/aggregate_des_excess_groupset_membership.yaml - data_elements/aggregate_des_no_analysis.yaml - data_elements/aggregate_des_datasets_different_period_types.yaml diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_custom_data_entry_forms_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_custom_data_entry_forms_empty.yaml new file mode 100644 index 00000000000..54b40004642 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_custom_data_entry_forms_empty.yaml @@ -0,0 +1,59 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: datasets_custom_data_entry_forms_empty +description: Datasets which have custom data entry forms which are empty. +section: Data sets +section_order: 3 +summary_sql: >- + WITH rs as ( + SELECT a.uid,a.name,b.name as comment from dataset a + INNER JOIN ( + SELECT dataentryformid, name from dataentryform WHERE + length(replace(htmlcode, ' ', '')) = 0 OR htmlcode IS NULL) b + ON a.dataentryform = b.dataentryformid) + select count(*) as value, + 100*count(*) / NULLIF( (select count(*) from dataset),0) as percent + from rs; +details_sql: >- + SELECT a.uid,a.name,b.name as comment from dataset a + INNER JOIN ( + SELECT dataentryformid, name from dataentryform WHERE + length(replace(htmlcode, ' ', '')) = 0 OR htmlcode IS NULL) b + ON a.dataentryform = b.dataentryformid; +severity: WARNING +introduction: > + Custom data entry forms can be created with HTML code to provide + a richer data entry experience for users. This check identifies + datasets which have custom data entry forms that are empty. This + may cause issues during data entry. +details_id_type: dataSets +recommendation: > + If the dataset should have a custom data entry form, be sure to + populate the HTML code field in the data entry form. If the dataset + should not have a custom data entry form, you should assign delete + the form and create a section form instead. diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/programs/programs_custom_data_entry_form_empty.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/programs/programs_custom_data_entry_form_empty.yaml new file mode 100644 index 00000000000..cbb67285f08 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/programs/programs_custom_data_entry_form_empty.yaml @@ -0,0 +1,79 @@ +# Copyright (c) 2004-2022, University of Oslo +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# Neither the name of the HISP project nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +--- +name: programs_custom_data_entry_forms_empty +description: Programs which have custom data entry forms which are empty. +section: Programs +section_order: 1 +summary_sql: >- + WITH rs as ( + SELECT p.uid,p.name, a.name as comment + from programstage a + INNER JOIN program p ON a.programid = p.programid + INNER JOIN ( + SELECT dataentryformid, name from dataentryform WHERE + length(replace(htmlcode, ' ', '')) = 0 OR htmlcode IS NULL) b + ON a.dataentryformid = b.dataentryformid + UNION + SELECT a.uid, a.name, 'Enrollment' as comment + from program a + INNER JOIN ( + SELECT dataentryformid, name from dataentryform WHERE + length(replace(htmlcode, ' ', '')) = 0 OR htmlcode IS NULL) b + ON a.dataentryformid = b.dataentryformid + + ) + select count(*) as value, + 100.0 * count(*) / NULLIF( (select count(*) from program),0) as percent + from rs; +details_sql: >- + SELECT p.uid,p.name, a.name as comment + from programstage a + INNER JOIN program p ON a.programid = p.programid + INNER JOIN ( + SELECT dataentryformid, name from dataentryform WHERE + length(replace(htmlcode, ' ', '')) = 0 OR htmlcode IS NULL) b + ON a.dataentryformid = b.dataentryformid + UNION + SELECT a.uid, a.name, 'Enrollment' as comment + from program a + INNER JOIN ( + SELECT dataentryformid, name from dataentryform WHERE + length(replace(htmlcode, ' ', '')) = 0 OR htmlcode IS NULL) b + ON a.dataentryformid = b.dataentryformid; +severity: WARNING +introduction: > + Custom data entry forms can be created with HTML code to provide + a richer data entry experience for users. This check identifies + any program stages which have custom data entry forms that are empty. This + may cause issues during data entry. +details_id_type: programs +recommendation: > + If the program stage, or event program, should have a custom data entry form, be sure to + populate the HTML code field in the data entry form. If the program stage or event program + should not have a custom data entry form, you should assign delete + the empty custom form and create an appropriate form instead. \ No newline at end of file diff --git a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java index 45caf579b78..1e73189444b 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/test/java/org/hisp/dhis/dataintegrity/DataIntegrityYamlReaderTest.java @@ -57,7 +57,7 @@ void testReadDataIntegrityYaml() { List checks = new ArrayList<>(); readYaml(checks, "data-integrity-checks.yaml", "data-integrity-checks", CLASS_PATH); - assertEquals(83, checks.size()); + assertEquals(85, checks.size()); // Names should be unique List allNames = checks.stream().map(DataIntegrityCheck::getName).toList(); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global.properties b/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global.properties index 308f3764127..1c3fa2aca13 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global.properties +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global.properties @@ -2062,6 +2062,7 @@ data_integrity.data_elements_aggregate_no_groups.name=Aggregate data elements no data_integrity.data_elements_aggregate_with_different_period_types.name=Aggregate data elements which belong to data sets with different period types. data_integrity.data_elements_without_datasets.name=Aggregate data elements not assigned to any data sets data_integrity.datasets_empty.name=Data sets with no data elements +data_integrity.datasets_custom_data_entry_forms_empty.name=Datasets which have custom data entry forms which are empty. data_integrity.datasets_not_assigned_to_org_units.name=Data sets not assigned to any organisation units data_integrity.data_elements_excess_groupset_membership.name=Data elements which belong to multiple groups in a group set. data_integrity.category_option_group_sets_scarce.name=Category option groups should have at least two members. @@ -2100,6 +2101,7 @@ data_integrity.periods_3y_future.name=Periods which are more than three years in data_integrity.periods_distant_past.name=Periods which are in the distant past. data_integrity.periods_same_start_end_date.name=Periods with the same start and end dates data_integrity.periods_same_start_date_period_type.name=Periods with the same start date and period type +data_integrity.programs_custom_data_entry_forms_empty.name=Programs which have custom data entry forms which are empty. data_integrity.program_rules_message_no_template.name=Program rules actions which should send or schedule a message without a message template. data_integrity.program_rules_no_action.name=Program rules with no action. data_integrity.program_rules_no_expression.name=Program rules with no expression. diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityEmptyCustomDataEntryFormsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityEmptyCustomDataEntryFormsControllerTest.java new file mode 100644 index 00000000000..1006e7529f1 --- /dev/null +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityEmptyCustomDataEntryFormsControllerTest.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.dataintegrity; + +import static org.hisp.dhis.http.HttpAssertions.assertStatus; + +import org.hisp.dhis.http.HttpStatus; +import org.junit.jupiter.api.Test; + +/** + * Tests for aggregate datasets with empty data entry forms. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/datasets/datasets_custom_data_entry_forms_empty.yaml} + * + *

Tests for programs and program stages with empty data entry forms. {@see + * dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/programs/programs_custom_data_entry_form_empty.yaml} + * + * @author Jason P. Pickering + */ +class DataIntegrityEmptyCustomDataEntryFormsControllerTest + extends AbstractDataIntegrityIntegrationTest { + + @Test + void testEmptyAggregateDataEntryForm() { + + final String check = "datasets_custom_data_entry_forms_empty"; + final String dataSetUID = "CowXAwmulDG"; + + String defaultCatCombo = getDefaultCatCombo(); + String dataElementA = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataElements", + "{ 'name': 'ANC1', 'shortName': 'ANC1', 'valueType' : 'NUMBER'," + + "'domainType' : 'AGGREGATE', 'aggregationType' : 'SUM' }")); + + assertStatus( + HttpStatus.CREATED, + POST( + "/dataSets", + """ + { + "name": "Test", + "id": "%s", + "shortName": "Test", + "periodType": "Monthly", + "categoryCombo": {"id": "%s"}, + "dataSetElements": [ + { + "dataSet": {"id": "%s"}, + "dataElement": {"id": "%s"} + } + ] + } + """ + .formatted(dataSetUID, defaultCatCombo, dataSetUID, dataElementA))); + + assertStatus( + HttpStatus.NO_CONTENT, + POST( + "/dataSets/" + dataSetUID + "/form", + """ + { + "style": "NORMAL", + "htmlCode": "

Test

" + } + """)); + + assertHasNoDataIntegrityIssues("dataSets", check, true); + + // The API silently ignores empty strings, so we use a space here instead. + assertStatus( + HttpStatus.NO_CONTENT, + POST( + "/dataSets/" + dataSetUID + "/form", + """ + { + "style": "NORMAL", + "htmlCode": " " + } + """)); + + assertHasDataIntegrityIssues("dataSets", check, 100, dataSetUID, "Test", "Test", true); + } + + @Test + void testNoEventForm() { + + final String check = "programs_custom_data_entry_forms_empty"; + final String programUID = "IpHINAT79UW"; + final String programStageUID = "Zj7UnCAulGe"; + String defaultCatCombo = getDefaultCatCombo(); + + assertHasNoDataIntegrityIssues("programs", check, false); + + // No form should be OK + assertStatus( + HttpStatus.OK, + POST( + "/metadata", + """ + { + "programStages": [ + { + "id": "%s", + "name": "Test", + "shortName": "Test", + "programStageDataElements": [] + } + ], + "programs": [ + { + "id": "%s", + "name": "Test", + "shortName": "Test", + "programType": "WITHOUT_REGISTRATION", + "categoryCombo": {"id": "%s"}, + "programStages": [{"id": "%s"}] + } + ] + } + """ + .formatted(programStageUID, programUID, defaultCatCombo, programStageUID))); + + assertHasNoDataIntegrityIssues("programs", check, true); + } + + @Test + void testNonEmptyEventFormHasNoDataIntegrityIssues() { + final String check = "programs_custom_data_entry_forms_empty"; + final String programUID = "IpHINAT79UW"; + final String programStageUID = "Zj7UnCAulGe"; + final String dataEntryFormUID = "V2wox39jSdy"; + String defaultCatCombo = getDefaultCatCombo(); + + assertStatus( + HttpStatus.OK, + POST( + "/metadata", + """ + { + "dataEntryForms": [ + { + "id": "%s", + "name": "Test", + "htmlCode": "

Test

" + } + ], + "programStages": [ + { + "id": "%s", + "name": "Test", + "shortName": "Test", + "programStageDataElements": [], + "dataEntryForm": {"id": "%s"} + } + ], + "programs": [ + { + "id": "%s", + "name": "Test", + "shortName": "Test", + "programType": "WITHOUT_REGISTRATION", + "categoryCombo": {"id": "%s"}, + "programStages": [{"id": "%s"}] + } + ] + } + """ + .formatted( + dataEntryFormUID, + programStageUID, + dataEntryFormUID, + programUID, + defaultCatCombo, + programStageUID))); + + assertHasNoDataIntegrityIssues("programs", check, true); + } + + @Test + void testEmptyEventFormHasIntegrityIssues() { + final String check = "programs_custom_data_entry_forms_empty"; + final String programUID = "IpHINAT79UW"; + final String programStageUID = "Zj7UnCAulGe"; + final String dataEntryFormUID = "V2wox39jSdy"; + String defaultCatCombo = getDefaultCatCombo(); + + assertStatus( + HttpStatus.OK, + POST( + "/metadata", + """ + { + "dataEntryForms": [ + { + "id": "%s", + "name": "Test", + "htmlCode": "" + } + ], + "programStages": [ + { + "id": "%s", + "name": "Test PS", + "shortName": "Test PS", + "programStageDataElements": [], + "dataEntryForm": {"id": "%s"} + } + ], + "programs": [ + { + "id": "%s", + "name": "Test", + "shortName": "Test", + "programType": "WITHOUT_REGISTRATION", + "categoryCombo": {"id": "%s"}, + "programStages": [{"id": "%s"}] + } + ] + } + """ + .formatted( + dataEntryFormUID, + programStageUID, + dataEntryFormUID, + programUID, + defaultCatCombo, + programStageUID))); + + assertHasDataIntegrityIssues("programs", check, 100, programUID, "Test", "Test PS", true); + } + + @Test + void testNonEmptyTrackerFormHasNoDataIntegrityIssues() { + final String check = "programs_custom_data_entry_forms_empty"; + final String programUID = "IpHINAT79UW"; + final String dataEntryFormUID = "V2wox39jSdy"; + String defaultCatCombo = getDefaultCatCombo(); + + assertStatus( + HttpStatus.OK, + POST( + "/metadata", + """ + { + "dataEntryForms": [ + { + "id": "%s", + "name": "Test", + "htmlCode": "

Test

" + } + ], + "programs": [ + { + "id": "%s", + "name": "Test", + "shortName": "Test", + "programType": "WITH_REGISTRATION", + "categoryCombo": {"id": "%s"}, + "dataEntryForm": {"id": "%s"} + } + ] + } + """ + .formatted(dataEntryFormUID, programUID, defaultCatCombo, dataEntryFormUID))); + + assertHasNoDataIntegrityIssues("programs", check, true); + } + + @Test + void testEmptyTrackerFormHasDataIntegrityIssues() { + final String check = "programs_custom_data_entry_forms_empty"; + final String programUID = "IpHINAT79UW"; + final String dataEntryFormUID = "V2wox39jSdy"; + String defaultCatCombo = getDefaultCatCombo(); + + assertStatus( + HttpStatus.OK, + POST( + "/metadata", + """ + { + "dataEntryForms": [ + { + "id": "%s", + "name": "Test", + "htmlCode": "" + } + ], + "programs": [ + { + "id": "%s", + "name": "Test", + "shortName": "Test", + "programType": "WITH_REGISTRATION", + "categoryCombo": {"id": "%s"}, + "dataEntryForm": {"id": "%s"} + } + ] + } + """ + .formatted(dataEntryFormUID, programUID, defaultCatCombo, dataEntryFormUID))); + + assertHasDataIntegrityIssues("programs", check, 100, programUID, "Test", "Enrollment", true); + } +} From 66241f59257efcc89d8fcdaa16fa453b4d73c8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Sun, 1 Dec 2024 19:38:58 +0100 Subject: [PATCH 13/24] fix: Simplify org unit group set join in analytics [DHIS2-16705] (#19349) --- .../table/AbstractEventJdbcTableManager.java | 8 +++---- .../JdbcEnrollmentAnalyticsTableManager.java | 10 ++++----- .../table/JdbcEventAnalyticsTableManager.java | 21 +++++++++---------- ...dbcTrackedEntityAnalyticsTableManager.java | 20 ++++++++---------- .../table/DatePeriodResourceTable.java | 10 ++++++--- 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index f00f880375e..0a60a2d5042 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -224,20 +224,20 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f * * @param attribute the {@link TrackedEntityAttribute}. * @param columnExpression the column expression. - * @param dataClause the data type related clause like "NUMERIC". + * @param dataFilterClause the data filter clause. * @return a select statement. */ protected String getSelectSubquery( - TrackedEntityAttribute attribute, String columnExpression, String dataClause) { + TrackedEntityAttribute attribute, String columnExpression, String dataFilterClause) { return replaceQualify( """ (select ${columnExpression} from ${trackedentityattributevalue} \ where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}${dataClause})\ + and trackedentityattributeid=${attributeId}${dataFilterClause})\ ${closingParentheses} as ${attributeUid}""", Map.of( "columnExpression", columnExpression, - "dataClause", dataClause, + "dataFilterClause", dataFilterClause, "attributeId", String.valueOf(attribute.getId()), "closingParentheses", getClosingParentheses(columnExpression), "attributeUid", quote(attribute.getUid()))); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index 31999aeab2e..f78f2aad73f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -152,21 +152,19 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti """ \s from ${enrollment} en \ inner join ${program} pr on en.programid=pr.programid \ - left join ${trackedentity} te on en.trackedentityid=te.trackedentityid \ - and te.deleted = false \ + left join ${trackedentity} te on en.trackedentityid=te.trackedentityid and te.deleted = false \ left join ${organisationunit} registrationou on te.organisationunitid=registrationou.organisationunitid \ inner join ${organisationunit} ou on en.organisationunitid=ou.organisationunitid \ + left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date)=dps.dateperiod \ left join analytics_rs_orgunitstructure ous on en.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on en.organisationunitid=ougs.organisationunitid \ - and (cast(${enrollmentDateMonth} as date)=ougs.startdate or ougs.startdate is null) \ - left join analytics_rs_dateperiodstructure dps on cast(en.enrollmentdate as date)=dps.dateperiod \ - where pr.programid=${programId} \ + where pr.programid=${programId} \ and en.organisationunitid is not null \ + and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ and en.lastupdated <= '${startTime}' \ and en.occurreddate is not null \ and en.deleted = false\s""", Map.of( - "enrollmentDateMonth", sqlBuilder.dateTrunc("month", "en.enrollmentdate"), "programId", String.valueOf(program.getId()), "startTime", toLongDate(params.getStartTime()))); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index ae382bd86fe..aa63a4d5ed9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -343,22 +343,21 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti left join ${trackedentity} te on en.trackedentityid=te.trackedentityid and te.deleted = false \ left join ${organisationunit} registrationou on te.organisationunitid=registrationou.organisationunitid \ inner join ${organisationunit} ou on ev.organisationunitid=ou.organisationunitid \ + left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as date)=dps.dateperiod \ left join analytics_rs_orgunitstructure ous on ev.organisationunitid=ous.organisationunitid \ left join analytics_rs_organisationunitgroupsetstructure ougs on ev.organisationunitid=ougs.organisationunitid \ - and (cast(${eventDateMonth} as date)=ougs.startdate or ougs.startdate is null) \ left join ${organisationunit} enrollmentou on en.organisationunitid=enrollmentou.organisationunitid \ inner join analytics_rs_categorystructure acs on ev.attributeoptioncomboid=acs.categoryoptioncomboid \ - left join analytics_rs_dateperiodstructure dps on cast(${eventDateExpression} as date)=dps.dateperiod \ where ev.lastupdated < '${startTime}' ${partitionClause} \ and pr.programid=${programId} \ and ev.organisationunitid is not null \ and (${eventDateExpression}) is not null \ + and (ougs.startdate is null or dps.monthstartdate=ougs.startdate) \ and dps.year >= ${firstDataYear} \ and dps.year <= ${latestDataYear} \ and ev.status in (${exportableEventStatues}) \ and ev.deleted = false""", Map.of( - "eventDateMonth", sqlBuilder.dateTrunc("month", eventDateExpression), "eventDateExpression", eventDateExpression, "partitionClause", partitionClause, "startTime", toLongDate(params.getStartTime()), @@ -596,12 +595,12 @@ private List getColumnForAttribute(TrackedEntityAttribute DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - String dataExpression = getDataFilterClause(attribute); - String sql = getSelectSubquery(attribute, selectExpression, dataExpression); + String dataFilterClause = getDataFilterClause(attribute); + String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataExpression)); + columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); } columns.add( @@ -701,12 +700,12 @@ private List getColumnsForOrgUnitTrackedEntityAttribute( } /** - * Creates a select statement for the given select expression. + * Retyrns a select statement for the given select expression. * * @param dataElement the data element to create the select statement for. * @param selectExpression the select expression. * @param dataFilterClause the data filter clause. - * @return A SQL select expression for the data element. + * @return a select expression. */ private String getSelectForInsert( DataElement dataElement, String selectExpression, String dataFilterClause) { @@ -773,7 +772,7 @@ private List getColumnFromDataElementWithLegendSet( * is valid according to the value type. For other value types, returns the empty string. * * @param dataElement the {@link DataElement}. - * @return an filter expression. + * @return a data filter clause. */ private String getDataFilterClause(DataElement dataElement) { String uid = dataElement.getUid(); @@ -794,7 +793,7 @@ private String getDataFilterClause(DataElement dataElement) { * is valid according to the value type. For other value types, returns the empty string. * * @param attribute the {@link TrackedEntityAttribute}. - * @return an filter expression. + * @return a data filter clause. */ private String getDataFilterClause(TrackedEntityAttribute attribute) { if (attribute.isNumericType()) { @@ -810,7 +809,7 @@ private String getDataFilterClause(TrackedEntityAttribute attribute) { * @param program the {@link Program}. * @param firstDataYear the first year to include. * @param lastDataYear the last data year to include. - * @return a list of years for which data exist. + * @return a list of years for which data exists. */ private List getDataYears( AnalyticsTableUpdateParams params, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 41ba840e7eb..4071615a7e8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -354,12 +354,10 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti .append( replaceQualify( """ - \s from ${trackedentity} te \ + \sfrom ${trackedentity} te \ left join analytics_rs_orgunitstructure ous on te.organisationunitid=ous.organisationunitid \ - left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid \ - and (cast(${trackedEntityCreatedMonth} as date)=ougs.startdate \ - or ougs.startdate is null)""", - Map.of("trackedEntityCreatedMonth", sqlBuilder.dateTrunc("month", "te.created")))); + left join analytics_rs_organisationunitgroupsetstructure ougs on te.organisationunitid=ougs.organisationunitid""", + Map.of())); ((List) params.getExtraParam(trackedEntityType.getUid(), ALL_NON_CONFIDENTIAL_TET_ATTRIBUTES)) @@ -376,14 +374,14 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti sql.append( replaceQualify( """ - \s where te.trackedentitytypeid = ${tetId} \ + \swhere te.trackedentitytypeid = ${tetId} \ and te.lastupdated < '${startTime}' \ and exists (select 1 from ${enrollment} en \ - where en.trackedentityid = te.trackedentityid \ - and exists (select 1 from ${event} ev \ - where ev.enrollmentid = en.enrollmentid \ - and ev.status in (${statuses}) \ - and ev.deleted = false)) \ + where en.trackedentityid = te.trackedentityid \ + and exists (select 1 from ${event} ev \ + where ev.enrollmentid = en.enrollmentid \ + and ev.status in (${statuses}) \ + and ev.deleted = false)) \ and te.created is not null \ and te.deleted = false""", Map.of( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java index ca86139beed..444ca2dc446 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DatePeriodResourceTable.java @@ -49,6 +49,7 @@ import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.resourcetable.ResourceTable; import org.hisp.dhis.resourcetable.ResourceTableType; +import org.hisp.dhis.util.DateUtils; /** * @author Lars Helge Overland @@ -75,7 +76,8 @@ private List getColumns() { List columns = Lists.newArrayList( new Column("dateperiod", DataType.DATE, Nullable.NOT_NULL), - new Column("year", DataType.INTEGER, Nullable.NOT_NULL)); + new Column("year", DataType.INTEGER, Nullable.NOT_NULL), + new Column("monthstartdate", DataType.DATE, Nullable.NOT_NULL)); for (PeriodType periodType : PeriodType.PERIOD_TYPES) { columns.add(new Column(periodType.getName().toLowerCase(), DataType.VARCHAR_50)); @@ -119,12 +121,14 @@ public Optional> getPopulateTempTableContent() { Calendar calendar = PeriodType.getCalendar(); for (Date day : days) { - List values = new ArrayList<>(); + final int year = PeriodType.getCalendar().fromIso(day).getYear(); + final Date monthStartDate = DateUtils.dateTruncMonth(day); - int year = PeriodType.getCalendar().fromIso(day).getYear(); + List values = new ArrayList<>(); values.add(day); values.add(year); + values.add(monthStartDate); for (PeriodType periodType : periodTypes) { values.add(periodType.createPeriod(day, calendar).getIsoDate()); From a5e70251c9a01b8b6786ab4dadb4925217a2c1f9 Mon Sep 17 00:00:00 2001 From: teleivo Date: Mon, 2 Dec 2024 06:00:11 +0100 Subject: [PATCH 14/24] test: simplify metadata assertions (#19347) I initially though I would need to also assert on nested relationship objects. This is why I tried to make generic assertions that could be reused for fields like dataValues on the event and on a relationships event. Since we do not need that make the assertions less complicated. --- .../export/IdSchemeExportControllerTest.java | 173 ++++++------------ 1 file changed, 56 insertions(+), 117 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java index 63d56a47764..96f8bb02364 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/IdSchemeExportControllerTest.java @@ -83,7 +83,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.function.Executable; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -158,120 +157,74 @@ void shouldExportMetadataUsingGivenIdScheme(TrackerIdSchemeParam idSchemeParam) Event event = get(Event.class, "QRYjLTiJTrA"); assertNotEmpty(event.getEventDataValues(), "test expects an event with data values"); - // maps JSON fields to idScheme request parameters - Map idSchemeRequestParams = - Map.of( + List idSchemeRequestParams = + List.of( "orgUnit", - "orgUnit", - "program", "program", "programStage", - "programStage", - "attributeOptionCombo", "categoryOptionCombo", - "attributeCategoryOptions", "categoryOption", - "dataValues", "dataElement"); - // maps JSON fields to expected metadata identifier in the requested idScheme and form. many - // category options are mapped to a single string value in the event.attributeCategoryOptions - Map> metadata = - Map.of( - "orgUnit", - actual -> - (() -> - assertIdScheme( - idSchemeParam.getIdentifier(event.getOrganisationUnit()), - actual, - idSchemeParam, - "orgUnit")), - "program", - actual -> - (() -> - assertIdScheme( - idSchemeParam.getIdentifier(event.getProgramStage().getProgram()), - actual, - idSchemeParam, - "program")), - "programStage", - actual -> - (() -> - assertIdScheme( - idSchemeParam.getIdentifier(event.getProgramStage()), - actual, - idSchemeParam, - "programStage")), - "attributeOptionCombo", - actual -> - (() -> - assertIdScheme( - idSchemeParam.getIdentifier(event.getAttributeOptionCombo()), - actual, - idSchemeParam, - "attributeOptionCombo")), - "attributeCategoryOptions", - json -> - (() -> { - String field = "attributeCategoryOptions"; - List expected = - event.getAttributeOptionCombo().getCategoryOptions().stream() - .map(co -> idSchemeParam.getIdentifier(co)) - .toList(); - assertNotEmpty( - expected, - String.format( - "metadata corresponding to field \"%s\" has no value in test data for" - + " idScheme '%s'", - field, idSchemeParam)); - assertTrue( - json.has(field), - () -> - String.format( - "field \"%s\" is not in response %s for idScheme '%s'", - field, json, idSchemeParam)); - assertContainsOnly( - expected, Arrays.asList(json.getString(field).string().split(","))); - }), - "dataValues", - json -> - (() -> { - String field = "dataValues"; - List expected = - event.getEventDataValues().stream() - .map( - dv -> - idSchemeParam.getIdentifier( - get(DataElement.class, dv.getDataElement()))) - .toList(); - assertNotEmpty( - expected, - String.format( - "metadata corresponding to field \"%s\" has no value in test data for" - + " idScheme '%s'", - field, idSchemeParam)); - assertTrue( - json.has(field), - () -> - String.format( - "field \"%s\" is not in response %s for idScheme '%s'", - field, json, idSchemeParam)); - List actual = - json.getList(field, JsonObject.class) - .toList(el -> el.getString("dataElement").string("")); - assertContainsOnly(expected, actual); - })); - String fields = metadata.keySet().stream().collect(Collectors.joining(",")); String idSchemes = - metadata.keySet().stream() - .map(m -> idSchemeRequestParams.get(m) + "IdScheme=" + idSchemeParam) + idSchemeRequestParams.stream() + .map(p -> p + "IdScheme=" + idSchemeParam) .collect(Collectors.joining("&")); JsonEvent actual = - GET("/tracker/events/{id}?fields={fields}&{idSchemes}", event.getUid(), fields, idSchemes) + GET( + "/tracker/events/{id}?fields=orgUnit,program,programStage,attributeOptionCombo,attributeCategoryOptions,dataValues&{idSchemes}", + event.getUid(), + idSchemes) .content(HttpStatus.OK) .as(JsonEvent.class); - assertMetadataIdScheme(metadata, actual, idSchemeParam, "event"); + assertAll( + "event metadata assertions for idScheme=" + idSchemeParam, + () -> + assertIdScheme( + idSchemeParam.getIdentifier(event.getOrganisationUnit()), + actual, + idSchemeParam, + "orgUnit"), + () -> + assertIdScheme( + idSchemeParam.getIdentifier(event.getProgramStage().getProgram()), + actual, + idSchemeParam, + "program"), + () -> + assertIdScheme( + idSchemeParam.getIdentifier(event.getProgramStage()), + actual, + idSchemeParam, + "programStage"), + () -> + assertIdScheme( + idSchemeParam.getIdentifier(event.getAttributeOptionCombo()), + actual, + idSchemeParam, + "attributeOptionCombo"), + () -> { + String field = "attributeCategoryOptions"; + List expected = + event.getAttributeOptionCombo().getCategoryOptions().stream() + .map(co -> idSchemeParam.getIdentifier(co)) + .toList(); + assertNotEmpty( + expected, + String.format( + "metadata corresponding to field \"%s\" has no value in test data for" + + " idScheme '%s'", + field, idSchemeParam)); + assertTrue( + actual.has(field), + () -> + String.format( + "field \"%s\" is not in response %s for idScheme '%s'", + field, actual, idSchemeParam)); + assertContainsOnly(expected, Arrays.asList(actual.getString(field).string().split(","))); + }, + () -> assertDataValues(actual, event, idSchemeParam)); } @Test @@ -363,7 +316,7 @@ void shouldExportEventDataValuesEquallyWithIdSchemeUIDAndName() { assertNotNull(nameDataValue, "event should have dataValues"); // dataElement is asserted in other tests assertAll( - "assert event fields", + "assert dataValue fields", () -> assertEquals(uidDataValue.getValue(), nameDataValue.getValue(), "value"), () -> assertEquals(uidDataValue.getCreatedAt(), nameDataValue.getCreatedAt(), "createdAt"), () -> assertEquals(uidDataValue.getUpdatedAt(), nameDataValue.getUpdatedAt(), "updatedAt"), @@ -409,20 +362,6 @@ public static Stream shouldExportMetadataUsingGivenIdSchem TrackerIdSchemeParam.ofAttribute(METADATA_ATTRIBUTE)); } - /** - * Asserts that every metadata key from {@code expected} is a field in the {@code actual} JSON and - * that its string value matches the requested {@code idSchemeParam}. - */ - private void assertMetadataIdScheme( - Map> expected, - JsonObject actual, - TrackerIdSchemeParam idSchemeParam, - String objectName) { - List assertions = - expected.entrySet().stream().map(e -> e.getValue().apply(actual)).toList(); - assertAll(objectName + " metadata assertions for idScheme=" + idSchemeParam, assertions); - } - private static void assertIdScheme( String expected, JsonObject actual, TrackerIdSchemeParam idSchemeParam, String field) { assertNotEmpty( From ee26fa3086fe3bdca471099cad8a64cb9b1b3e0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 07:59:01 +0100 Subject: [PATCH 15/24] chore(deps): bump org.jsoup:jsoup from 1.18.1 to 1.18.3 in /dhis-2 (#19360) Bumps [org.jsoup:jsoup](https://github.com/jhy/jsoup) from 1.18.1 to 1.18.3. - [Release notes](https://github.com/jhy/jsoup/releases) - [Changelog](https://github.com/jhy/jsoup/blob/master/CHANGES.md) - [Commits](https://github.com/jhy/jsoup/compare/jsoup-1.18.1...jsoup-1.18.3) --- updated-dependencies: - dependency-name: org.jsoup:jsoup dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dhis-2/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index ad2662b5a00..b166c128041 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -178,7 +178,7 @@ 1.0.0 3.0.1 2.9.0 - 1.18.1 + 1.18.3 33.3.1-jre From 77be2641ac4c02fa76919b25ddb2410d3734256d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 08:56:08 +0100 Subject: [PATCH 16/24] chore(deps-dev): bump io.qameta.allure:allure-junit5 (#19353) Bumps [io.qameta.allure:allure-junit5](https://github.com/allure-framework/allure-java) from 2.29.0 to 2.29.1. - [Release notes](https://github.com/allure-framework/allure-java/releases) - [Commits](https://github.com/allure-framework/allure-java/compare/2.29.0...2.29.1) --- updated-dependencies: - dependency-name: io.qameta.allure:allure-junit5 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dhis-2/dhis-test-e2e/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-test-e2e/pom.xml b/dhis-2/dhis-test-e2e/pom.xml index e7f49aaf27d..8f3e7e87eaa 100644 --- a/dhis-2/dhis-test-e2e/pom.xml +++ b/dhis-2/dhis-test-e2e/pom.xml @@ -30,7 +30,7 @@ 1.5.3 4.2.2 1.18.36 - 2.29.0 + 2.29.1 4.27.0 2.0.16 4.5.14 From 9edee0fded00dec6ac8eed28040554bc2bc3a82c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 08:56:23 +0100 Subject: [PATCH 17/24] chore(deps): bump me.fabriciorby:maven-surefire-junit5-tree-reporter (#19352) Bumps [me.fabriciorby:maven-surefire-junit5-tree-reporter](https://github.com/fabriciorby/maven-surefire-junit5-tree-reporter) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/fabriciorby/maven-surefire-junit5-tree-reporter/releases) - [Commits](https://github.com/fabriciorby/maven-surefire-junit5-tree-reporter/compare/v1.3.0...v1.4.0) --- updated-dependencies: - dependency-name: me.fabriciorby:maven-surefire-junit5-tree-reporter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dhis-2/dhis-test-e2e/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-test-e2e/pom.xml b/dhis-2/dhis-test-e2e/pom.xml index 8f3e7e87eaa..9957dd40f7e 100644 --- a/dhis-2/dhis-test-e2e/pom.xml +++ b/dhis-2/dhis-test-e2e/pom.xml @@ -12,7 +12,7 @@ 2.43.0 3.13.0 3.5.2 - 1.3.0 + 1.4.0 5.11.3 2.11.0 2.24.2 From ce3f6b071ed6dc65319b1ca9004ce2ce7bde4e30 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 2 Dec 2024 14:56:54 +0700 Subject: [PATCH 18/24] fix: Adjust SQL for data integrity check of COCs without names [DHIS2-19090] (#19342) --- .../category_option_combinations_no_names.yaml | 11 ++++++++--- ...rityCategoryOptionCombosNoNamesControllerTest.java | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_no_names.yaml b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_no_names.yaml index d95e3b0d6c8..d17e0627f70 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_no_names.yaml +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/resources/data-integrity-checks/categories/category_option_combinations_no_names.yaml @@ -31,13 +31,18 @@ section: Categories section_order: 8 summary_sql: >- WITH coc_no_names as ( - SELECT uid,name FROM categoryoptioncombo WHERE name IS NULL OR name = '' + SELECT uid FROM categoryoptioncombo WHERE name IS NULL OR name = '' ) SELECT COUNT(*) as value, - 100 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoptioncombo), 0) as percent + 100.0 * COUNT(*) / NULLIF( (SELECT COUNT(*) FROM categoryoptioncombo), 0) as percent FROM coc_no_names; details_sql: >- - SELECT uid,name FROM categoryoptioncombo WHERE name IS NULL OR name = ''; + SELECT coc.uid, + array_to_string(array_agg(co.name),';') as name FROM categoryoptioncombo coc + INNER JOIN categoryoptioncombos_categoryoptions cocs_co on cocs_co.categoryoptioncomboid = coc.categoryoptioncomboid + INNER JOIN categoryoption co on co.categoryoptionid = cocs_co.categoryoptionid + WHERE coc.name IS NULL OR coc.name = '' + GROUP BY coc.uid; details_id_type: categoryOptionCombos severity: SEVERE introduction: > diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionCombosNoNamesControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionCombosNoNamesControllerTest.java index b451a208c82..1f45bcaaa4e 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionCombosNoNamesControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionCombosNoNamesControllerTest.java @@ -78,7 +78,7 @@ void testCategoryOptionCombosNoNames() { checkDataIntegritySummary(check, 1, 50, true); assertHasDataIntegrityIssues( - "categoryOptionCombos", check, 50, redCategoryOptionComboId, "", "", true); + "categoryOptionCombos", check, 50, redCategoryOptionComboId, "Red", "", true); } @Test From 7ac63b67b4b95b16c4646891b0dda877864551d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 08:57:07 +0100 Subject: [PATCH 19/24] chore(deps): bump org.apache.velocity:velocity-engine-core in /dhis-2 (#19337) Bumps org.apache.velocity:velocity-engine-core from 2.4 to 2.4.1. --- updated-dependencies: - dependency-name: org.apache.velocity:velocity-engine-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dhis-2/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index b166c128041..72a481453a6 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -133,7 +133,7 @@ 6.1.0 - 2.4 + 2.4.1 3.1 From f016834f5261fdccb26e84cddbf9a5f926a89e4b Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 2 Dec 2024 15:05:41 +0700 Subject: [PATCH 20/24] Prevent import of duplicate default COCs (#19101) * Prevent import of duplicate COCs * Fix code smells * Linting * feat: Update error code & remove redundant equals method * Rework some code * Fix code smells * Minor * Partially fix test * Fix/disable more tests * Add additional checks for COCs * Linting * Update tests * Linting * Rework tests * Rework methods a bit * Linting * Fix various issues * Linting * More sonar cube stuff * Linting * More sonar cube * fix: Remove custom COC defaults * Remove custom default * Linting * Try and fix tracker test * Linting * Try and fix tracker tests * Remove check on bad options in a COC for now * Fix event test * Add missing files * Add test to delete duplicated default COC * Add missing assertion * Linting --------- Co-authored-by: David Mackessy Co-authored-by: Enrico Co-authored-by: Jan Bernitt --- .../org/hisp/dhis/feedback/ErrorCode.java | 2 + .../CategoryOptionComboObjectBundleHook.java | 72 +++++++++++ .../resources/tracker/idSchemesMetadata.json | 1 + .../validation/EventImportValidationTest.java | 12 +- .../tracker/tracker_basic_metadata.json | 118 ++++++++++++------ .../enrollments-cat-write-access.json | 29 +++++ .../validations/events-cat-write-access.json | 38 +----- .../CategoryOptionComboControllerTest.java | 60 +++++++++ ...rityCategoryOptionComboDuplicatedTest.java | 94 +++++++------- 9 files changed, 306 insertions(+), 120 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/CategoryOptionComboObjectBundleHook.java create mode 100644 dhis-2/dhis-test-integration/src/test/resources/tracker/validations/enrollments-cat-write-access.json diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index dea0cc65ed4..1c1cf44d0af 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -67,6 +67,8 @@ public enum ErrorCode { E1120("Update cannot be applied as it would make existing data values inaccessible"), E1121("Data element `{0}` value type cannot be changed as it has associated data values"), + E1122("Category option combo {0} cannot be associated with the default category combo"), + E1125("Category option combo {0} contains options not associated with category combo {1}"), /* Org unit merge */ E1500("At least two source orgs unit must be specified"), E1501("Target org unit must be specified"), diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/CategoryOptionComboObjectBundleHook.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/CategoryOptionComboObjectBundleHook.java new file mode 100644 index 00000000000..8347f96b4e8 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/CategoryOptionComboObjectBundleHook.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dxf2.metadata.objectbundle.hooks; + +import java.util.function.Consumer; +import lombok.AllArgsConstructor; +import org.hisp.dhis.category.CategoryCombo; +import org.hisp.dhis.category.CategoryOptionCombo; +import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.dxf2.metadata.objectbundle.ObjectBundle; +import org.hisp.dhis.feedback.ErrorCode; +import org.hisp.dhis.feedback.ErrorReport; +import org.springframework.stereotype.Component; + +@Component +@AllArgsConstructor +public class CategoryOptionComboObjectBundleHook + extends AbstractObjectBundleHook { + private final CategoryService categoryService; + + private void checkNonStandardDefaultCatOptionCombo( + CategoryOptionCombo categoryOptionCombo, Consumer addReports) { + + CategoryCombo categoryCombo = categoryOptionCombo.getCategoryCombo(); + CategoryCombo defaultCombo = categoryService.getDefaultCategoryCombo(); + if (!categoryCombo.getUid().equals(defaultCombo.getUid())) { + return; + } + + CategoryOptionCombo defaultCatOptionCombo = categoryService.getDefaultCategoryOptionCombo(); + + if (!categoryOptionCombo.getUid().equals(defaultCatOptionCombo.getUid())) { + addReports.accept( + new ErrorReport( + CategoryOptionCombo.class, ErrorCode.E1122, categoryOptionCombo.getName())); + } + } + + @Override + public void validate( + CategoryOptionCombo categoryOptionCombo, + ObjectBundle bundle, + Consumer addReports) { + + checkNonStandardDefaultCatOptionCombo(categoryOptionCombo, addReports); + } +} diff --git a/dhis-2/dhis-test-e2e/src/test/resources/tracker/idSchemesMetadata.json b/dhis-2/dhis-test-e2e/src/test/resources/tracker/idSchemesMetadata.json index 195b156af94..216c33d1dca 100644 --- a/dhis-2/dhis-test-e2e/src/test/resources/tracker/idSchemesMetadata.json +++ b/dhis-2/dhis-test-e2e/src/test/resources/tracker/idSchemesMetadata.json @@ -742,6 +742,7 @@ ], "categoryCombos": [ { + "code": "TA_CATEGORY_COMBO_ATTRIBUTE", "name": "TA Category combo attribute", "created": "2022-05-30T11:40:03.717", diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java index 646276e5e0e..5394088d2d0 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java @@ -161,12 +161,20 @@ void testTrackerAndProgramEventUpdateSuccess() throws IOException { @Test void testCantWriteAccessCatCombo() throws IOException { - TrackerObjects trackerObjects = fromJson("tracker/validations/events-cat-write-access.json"); + TrackerObjects trackerObjects = + fromJson("tracker/validations/enrollments-cat-write-access.json"); TrackerImportParams params = new TrackerImportParams(); - injectSecurityContextUser(userService.getUser(USER_6)); ImportReport importReport = trackerImportService.importTracker(params, trackerObjects); + assertNoErrors(importReport); + + trackerObjects = fromJson("tracker/validations/events-cat-write-access.json"); + params = new TrackerImportParams(); + injectSecurityContextUser(userService.getUser(USER_6)); + + importReport = trackerImportService.importTracker(params, trackerObjects); + assertHasOnlyErrors( importReport, ValidationCode.E1096, diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_metadata.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_metadata.json index 66db4ea628a..41715dd25d9 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_metadata.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/tracker_basic_metadata.json @@ -559,7 +559,7 @@ "ignoreOverdueEvents": false, "programSections": [], "created": "2019-03-29T12:41:33.001", - "shortName": "Tracker Program", + "shortName": "Tracker Program no default CC", "incidentDateLabel": "IncidentDate", "translations": [], "style": { @@ -571,10 +571,75 @@ }, "programStages": [ { - "id": "Qmqxq907VNz" + "id": "OilWH8Cj6QU" + } + ], + "selectEnrollmentDatesInFuture": false, + "categoryCombo": { + "id": "WG3syvpSE9o" + }, + "notificationTemplates": [], + "displayIncidentDate": true, + "accessLevel": "OPEN", + "featureType": "POLYGON", + "id": "qiHs9hD2Opf", + "lastUpdatedBy": { + "id": "tTgjgobT1oS" + }, + "completeEventsExpiryDays": 10, + "organisationUnits": [ + { + "id": "QfUVllTs6cS" }, { - "id": "OilWH8Cj6QU" + "id": "QfUVllTs6cZ" + }, + { + "id": "QfUVllTs6cW" + } + ], + "onlyEnrollOnce": false, + "lastUpdated": "2019-03-29T12:41:33.039", + "trackedEntityType": { + "id": "bPJ0FMtcnEh" + }, + "name": "Tracker Program no default CC", + "enrollmentDateLabel": "EnrollmentDate", + "maxTeiCountToReturn": 0, + "attributeValues": [], + "programTrackedEntityAttributes": [], + "useFirstStageDuringRegistration": false, + "minAttributesRequiredToSearch": 1, + "skipOffline": false, + "version": 0, + "programType": "WITH_REGISTRATION", + "selectIncidentDatesInFuture": false, + "displayFrontPageList": false, + "sharing": { + "public": "rw------", + "external": false, + "owner": "tTgjgobT1oS", + "users": {}, + "userGroups": {} + } + }, + { + "ignoreOverdueEvents": false, + "programSections": [], + "created": "2019-03-29T12:41:33.001", + "shortName": "Tracker Program", + "incidentDateLabel": "IncidentDate", + "translations": [], + "style": { + "color": "#d32f2f" + }, + "expiryDays": 0, + "user": { + "id": "tTgjgobT1oS" + }, + "programStages": [ + { + "id": "Qmqxq907VNz" }, { "id": "m7HlXaMvgX2" @@ -2672,10 +2737,10 @@ ], "categoryOptions": [ { - "id": "xYerKDKCefk", - "code": "default", - "name": "default", - "shortName": "default", + "id": "WAFwwid9TzE", + "code": "default2", + "name": "default2", + "shortName": "default2", "startDate": "2019-01-01T00:00:00.000", "attributeValues": [], "lastUpdated": "2019-03-25T13:40:24.783", @@ -2812,14 +2877,14 @@ ], "categories": [ { - "id": "GLevLNI9wkl", - "name": "default", - "shortName": "default", - "code": "default", + "id": "Vk8PHR8HBAe", + "name": "default2", + "shortName": "default2", + "code": "default2", "dataDimensionType": "DISAGGREGATION", "categoryOptions": [ { - "id": "xYerKDKCefk" + "id": "WAFwwid9TzE" } ], "dataDimension": false, @@ -2886,13 +2951,13 @@ ], "categoryCombos": [ { - "id": "bjDvmb4bfuf", - "code": "default", - "name": "default", + "id": "WG3syvpSE9o", + "code": "default2", + "name": "default2", "dataDimensionType": "DISAGGREGATION", "categories": [ { - "id": "GLevLNI9wkl" + "id": "Vk8PHR8HBAe" } ], "created": "2019-03-25T13:40:24.771", @@ -2978,25 +3043,6 @@ } ], "categoryOptionCombos": [ - { - "id": "HllvX50cXC0", - "code": "default", - "name": "default", - "categoryOptions": [ - { - "id": "xYerKDKCefk" - } - ], - "categoryCombo": { - "id": "bjDvmb4bfuf" - }, - "translations": [], - "startDate": "2019-01-01T00:00:00.000", - "lastUpdated": "2019-03-25T13:40:24.779", - "ignoreApproval": false, - "attributeValues": [], - "created": "2019-03-25T13:40:24.775" - }, { "id": "KKKKX50cXC0", "name": "accesstest1", @@ -3007,7 +3053,7 @@ } ], "categoryCombo": { - "id": "bjDvmb4bfuf" + "id": "WG3syvpSE9o" }, "translations": [], "startDate": "2019-01-01T00:00:00.000", diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/enrollments-cat-write-access.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/enrollments-cat-write-access.json new file mode 100644 index 00000000000..7a18012a347 --- /dev/null +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/enrollments-cat-write-access.json @@ -0,0 +1,29 @@ +{ + "trackedEntities": [], + "enrollments": [ + { + "enrollment": "CzXv1fa1kbX", + "trackedEntity": "Kj6vYde4LHh", + "program": { + "idScheme": "UID", + "identifier": "qiHs9hD2Opf" + }, + "status": "COMPLETED", + "orgUnit": { + "idScheme": "UID", + "identifier": "QfUVllTs6cS" + }, + "enrolledAt": "2019-08-19T00:00:00.000", + "occurredAt": "2019-08-19T00:00:00.000", + "followUp": false, + "deleted": false, + "storedBy": "admin", + "events": [], + "relationships": [], + "attributes": [], + "notes": [] + } + ], + "events": [], + "relationships": [] +} \ No newline at end of file diff --git a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json index 254eda32269..d4c1aef6ca9 100644 --- a/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json +++ b/dhis-2/dhis-test-integration/src/test/resources/tracker/validations/events-cat-write-access.json @@ -1,35 +1,4 @@ { - "importMode": "COMMIT", - "idSchemes": { - "dataElementIdScheme": { - "idScheme": "UID" - }, - "orgUnitIdScheme": { - "idScheme": "UID" - }, - "programIdScheme": { - "idScheme": "UID" - }, - "programStageIdScheme": { - "idScheme": "UID" - }, - "idScheme": { - "idScheme": "UID" - }, - "categoryOptionComboIdScheme": { - "idScheme": "UID" - }, - "categoryOptionIdScheme": { - "idScheme": "UID" - } - }, - "importStrategy": "CREATE", - "atomicMode": "ALL", - "flushMode": "AUTO", - "validationMode": "FULL", - "skipPatternValidation": false, - "skipSideEffects": false, - "skipRuleEngine": false, "trackedEntities": [], "enrollments": [], "events": [ @@ -38,13 +7,13 @@ "status": "ACTIVE", "program": { "idScheme": "UID", - "identifier": "E8o1E9tAppy" + "identifier": "qiHs9hD2Opf" }, "programStage": { "idScheme": "UID", "identifier": "OilWH8Cj6QU" }, - "enrollment": "MNWZ6hnuhSw", + "enrollment": "CzXv1fa1kbX", "orgUnit": { "idScheme": "UID", "identifier": "QfUVllTs6cS" @@ -64,6 +33,5 @@ "notes": [] } ], - "relationships": [], - "username": "system-process" + "relationships": [] } \ No newline at end of file diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CategoryOptionComboControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CategoryOptionComboControllerTest.java index ab13aa6564e..dcd62796401 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CategoryOptionComboControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CategoryOptionComboControllerTest.java @@ -27,19 +27,26 @@ */ package org.hisp.dhis.webapi.controller; +import static org.hisp.dhis.http.HttpAssertions.assertStatus; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; import org.hisp.dhis.category.CategoryCombo; import org.hisp.dhis.category.CategoryOption; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.http.HttpStatus; import org.hisp.dhis.jsontree.JsonArray; +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonObject; import org.hisp.dhis.test.webapi.H2ControllerIntegrationTestBase; import org.hisp.dhis.test.webapi.json.domain.JsonCategoryOptionCombo; +import org.hisp.dhis.test.webapi.json.domain.JsonErrorReport; import org.hisp.dhis.test.webapi.json.domain.JsonIdentifiableObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; @@ -115,4 +122,57 @@ void catOptionCombosExcludingDefaultTest() { assertFalse( catOptionComboNames.contains("default"), "default catOptionCombo is not in payload"); } + + @Test + @DisplayName("Duplicate default category option combos should not be allowed") + void catOptionCombosDuplicatedDefaultTest() { + JsonObject response = + GET("/categoryOptionCombos?filter=name:eq:default&fields=id,categoryCombo[id],categoryOptions[id]") + .content(); + JsonList catOptionCombos = + response.getList("categoryOptionCombos", JsonCategoryOptionCombo.class); + String defaultCatOptionComboOptions = + catOptionCombos.get(0).getCategoryOptions().get(0).getId(); + String defaultCatOptionComboCatComboId = catOptionCombos.get(0).getCategoryCombo().getId(); + response = + POST( + "/categoryOptionCombos/", + """ + { "name": "Not default", + "categoryOptions" : [{"id" : "%s"}], + "categoryCombo" : {"id" : "%s"} } + """ + .formatted(defaultCatOptionComboOptions, defaultCatOptionComboCatComboId)) + .content(HttpStatus.CONFLICT); + + JsonErrorReport error = + response.find(JsonErrorReport.class, report -> report.getErrorCode() == ErrorCode.E1122); + assertNotNull(error); + assertEquals( + "Category option combo Not default cannot be associated with the default category combo", + error.getMessage()); + } + + @Test + @DisplayName("Can delete a duplicate default COC") + void canAllowDeleteDuplicatedDefaultCOC() { + // Revert to the service layer as the API should not allow us to create a duplicate default COC + CategoryOptionCombo defaultCOC = categoryService.getDefaultCategoryOptionCombo(); + CategoryCombo categoryCombo = + categoryService.getCategoryCombo(defaultCOC.getCategoryCombo().getUid()); + CategoryOptionCombo existingCategoryOptionCombo = + categoryService.getCategoryOptionCombo(defaultCOC.getUid()); + CategoryOptionCombo categoryOptionComboDuplicate = new CategoryOptionCombo(); + categoryOptionComboDuplicate.setAutoFields(); + categoryOptionComboDuplicate.setCategoryCombo(categoryCombo); + Set newCategoryOptions = + new HashSet<>(existingCategoryOptionCombo.getCategoryOptions()); + categoryOptionComboDuplicate.setCategoryOptions(newCategoryOptions); + categoryOptionComboDuplicate.setName("dupDefault"); + categoryService.addCategoryOptionCombo(categoryOptionComboDuplicate); + + // Can delete the duplicated default COC + assertStatus( + HttpStatus.OK, DELETE("/categoryOptionCombos/" + categoryOptionComboDuplicate.getUid())); + } } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboDuplicatedTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboDuplicatedTest.java index 66016541b3c..f7e4ed3a702 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboDuplicatedTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityCategoryOptionComboDuplicatedTest.java @@ -28,8 +28,14 @@ package org.hisp.dhis.webapi.controller.dataintegrity; import static org.hisp.dhis.http.HttpAssertions.assertStatus; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import java.util.HashSet; import java.util.Set; +import org.hisp.dhis.category.CategoryCombo; +import org.hisp.dhis.category.CategoryOption; +import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.http.HttpStatus; import org.hisp.dhis.jsontree.JsonList; import org.hisp.dhis.jsontree.JsonObject; @@ -48,8 +54,6 @@ class DataIntegrityCategoryOptionComboDuplicatedTest extends AbstractDataIntegri private final String check = "category_option_combos_have_duplicates"; - private String cocWithOptionsA; - private String categoryOptionRed; @Test @@ -80,44 +84,40 @@ void testCategoryOptionCombosDuplicated() { + categoryColor + "'}]} ")); - cocWithOptionsA = - assertStatus( - HttpStatus.CREATED, - POST( - "/categoryOptionCombos", - """ - { "name": "Reddish", - "categoryOptions" : [{"id" : "%s"}], - "categoryCombo" : {"id" : "%s"} } - """ - .formatted(categoryOptionRed, testCatCombo))); + HttpResponse response = GET("/categoryOptionCombos?fields=id,name&filter=name:eq:Red"); + assertStatus(HttpStatus.OK, response); + JsonObject responseContent = response.content(); - assertStatus( - HttpStatus.CREATED, - POST( - "/categoryOptionCombos", - """ - { "name": "Not Red", - "categoryOptions" : [{"id" : "%s"}]}, - "categoryCombo" : {"id" : "%s"} } - """ - .formatted(categoryOptionRed, testCatCombo))); + JsonList catOptionCombos = + responseContent.getList("categoryOptionCombos", JsonCategoryOptionCombo.class); + assertEquals(1, catOptionCombos.size()); + String redCategoryOptionComboId = catOptionCombos.get(0).getId(); + + /*We must resort to the service layer as the API will not allow us to create a duplicate*/ + CategoryCombo categoryCombo = categoryService.getCategoryCombo(testCatCombo); + CategoryOptionCombo existingCategoryOptionCombo = + categoryService.getCategoryOptionCombo(redCategoryOptionComboId); + CategoryOptionCombo categoryOptionComboDuplicate = new CategoryOptionCombo(); + categoryOptionComboDuplicate.setAutoFields(); + categoryOptionComboDuplicate.setCategoryCombo(categoryCombo); + Set newCategoryOptions = + new HashSet<>(existingCategoryOptionCombo.getCategoryOptions()); + categoryOptionComboDuplicate.setCategoryOptions(newCategoryOptions); + categoryOptionComboDuplicate.setName("Reddish"); + manager.persist(categoryOptionComboDuplicate); + dbmsManager.clearSession(); + String categoryOptionComboDuplicatedID = categoryOptionComboDuplicate.getUid(); + assertNotNull(categoryOptionComboDuplicatedID); assertNamedMetadataObjectExists("categoryOptionCombos", "default"); assertNamedMetadataObjectExists("categoryOptionCombos", "Red"); assertNamedMetadataObjectExists("categoryOptionCombos", "Reddish"); - assertNamedMetadataObjectExists("categoryOptionCombos", "Not Red"); - /*We need to get the Red category option combo to be able to check the data integrity issues*/ + /* There are three total category option combos, so we expect 33% */ + checkDataIntegritySummary(check, 1, 33, true); - JsonObject response = GET("/categoryOptionCombos?fields=id,name&filter=name:eq:Red").content(); - JsonList catOptionCombos = - response.getList("categoryOptionCombos", JsonCategoryOptionCombo.class); - String redCategoryOptionComboId = catOptionCombos.get(0).getId(); - /* There are four total category option combos, so we expect 25% */ - checkDataIntegritySummary(check, 1, 25, true); - - Set expectedCategoryOptCombos = Set.of(cocWithOptionsA, redCategoryOptionComboId); + Set expectedCategoryOptCombos = + Set.of(categoryOptionComboDuplicatedID, redCategoryOptionComboId); Set expectedMessages = Set.of("Red", "Reddish"); checkDataIntegrityDetailsIssues( check, expectedCategoryOptCombos, expectedMessages, Set.of(), "categoryOptionCombos"); @@ -135,27 +135,27 @@ void testCategoryOptionCombosNotDuplicated() { HttpStatus.CREATED, POST("/categoryOptions", "{ 'name': 'Blue', 'shortName': 'Blue' }")); - cocWithOptionsA = + String categoryColor = assertStatus( HttpStatus.CREATED, POST( - "/categoryOptionCombos", - """ - { "name": "Color", - "categoryOptions" : [{"id" : "%s"} ] } - """ - .formatted(categoryOptionRed))); + "/categories", + "{ 'name': 'Color', 'shortName': 'Color', 'dataDimensionType': 'DISAGGREGATION' ," + + "'categoryOptions' : [{'id' : '" + + categoryOptionRed + + "'}, {'id' : '" + + categoryOptionBlue + + "'} ] }")); assertStatus( HttpStatus.CREATED, POST( - "/categoryOptionCombos", - """ - { "name": "Colour", - "categoryOptions" : [{"id" : "%s"} ] } - """ - .formatted(categoryOptionBlue))); - + "/categoryCombos", + "{ 'name' : 'Color', " + + "'dataDimensionType' : 'DISAGGREGATION', 'categories' : [" + + "{'id' : '" + + categoryColor + + "'}]} ")); assertHasNoDataIntegrityIssues("categoryOptionCombos", check, true); } From 95b621c5060329524f00a967540315e479e979ae Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Mon, 2 Dec 2024 09:54:35 +0100 Subject: [PATCH 21/24] chore: move SqlBuilder to own module (#19348) * chore: move SqlBuilder to own module (DHIS-16707) --- .../dhis-service-analytics/pom.xml | 4 + .../dhis/analytics/config/ServiceConfig.java | 4 +- .../table/setting/AnalyticsTableSettings.java | 31 -- ...sTest.java => SqlBuilderSettingsTest.java} | 10 +- dhis-2/dhis-support/dhis-support-sql/pom.xml | 107 +++++ .../dhis/db}/AnalyticsSqlBuilderProvider.java | 12 +- .../org/hisp/dhis/db}/SqlBuilderProvider.java | 12 +- .../dhis/db}/init/AnalyticsDatabaseInit.java | 8 +- .../org/hisp/dhis/db/model/Collation.java | 0 .../java/org/hisp/dhis/db/model/Column.java | 0 .../java/org/hisp/dhis/db/model/DataType.java | 0 .../java/org/hisp/dhis/db/model/Database.java | 4 - .../java/org/hisp/dhis/db/model/Index.java | 0 .../org/hisp/dhis/db/model/IndexFunction.java | 0 .../org/hisp/dhis/db/model/IndexType.java | 0 .../java/org/hisp/dhis/db/model/Logged.java | 0 .../java/org/hisp/dhis/db/model/Table.java | 0 .../hisp/dhis/db/model/TablePartition.java | 0 .../dhis/db/model/constraint/Nullable.java | 0 .../hisp/dhis/db/model/constraint/Unique.java | 0 .../dhis/db/setting/SqlBuilderSettings.java | 121 +++++ .../hisp/dhis/db/sql/AbstractSqlBuilder.java | 10 +- .../hisp/dhis/db/sql/AnalyticsSqlBuilder.java | 0 .../dhis/db/sql/ClickHouseSqlBuilder.java | 0 .../db/sql/ClickhouseAnalyticsSqlBuilder.java | 0 .../dhis/db/sql/DorisAnalyticsSqlBuilder.java | 0 .../org/hisp/dhis/db/sql/DorisSqlBuilder.java | 0 .../hisp/dhis/db/sql/PostgreSqlBuilder.java | 0 .../db/sql/PostgresAnalyticsSqlBuilder.java | 0 .../java/org/hisp/dhis/db/sql/SqlBuilder.java | 0 .../org/hisp/dhis/db/model/ColumnTest.java | 56 +++ .../org/hisp/dhis/db/model/DataTypeTest.java | 53 ++ .../org/hisp/dhis/db/model/IndexTest.java | 76 +++ .../org/hisp/dhis/db/model/TableTest.java | 145 ++++++ .../db/setting/SqlBuilderSettingsTest.java | 69 +++ .../dhis/db/sql/ClickHouseSqlBuilderTest.java | 381 +++++++++++++++ .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 367 ++++++++++++++ .../dhis/db/sql/PostgreSqlBuilderTest.java | 452 ++++++++++++++++++ dhis-2/dhis-support/pom.xml | 1 + dhis-2/dhis-test-integration/pom.xml | 6 + dhis-2/pom.xml | 5 + 41 files changed, 1871 insertions(+), 63 deletions(-) rename dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/setting/{AnalyticsTableSettingsTest.java => SqlBuilderSettingsTest.java} (93%) create mode 100644 dhis-2/dhis-support/dhis-support-sql/pom.xml rename dhis-2/{dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql => dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db}/AnalyticsSqlBuilderProvider.java (86%) rename dhis-2/{dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql => dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db}/SqlBuilderProvider.java (88%) rename dhis-2/{dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table => dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db}/init/AnalyticsDatabaseInit.java (96%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/Collation.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/Column.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/DataType.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/Database.java (93%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/Index.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/IndexFunction.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/IndexType.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/Logged.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/Table.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/TablePartition.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/constraint/Nullable.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/model/constraint/Unique.java (100%) create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/setting/SqlBuilderSettings.java rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java (96%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilder.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/ClickhouseAnalyticsSqlBuilder.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/DorisAnalyticsSqlBuilder.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java (100%) rename dhis-2/{dhis-services/dhis-service-analytics => dhis-support/dhis-support-sql}/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java (100%) create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/ColumnTest.java create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/IndexTest.java create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/TableTest.java create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/setting/SqlBuilderSettingsTest.java create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java create mode 100644 dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/pom.xml b/dhis-2/dhis-services/dhis-service-analytics/pom.xml index 8e8e14dabae..c047a707af1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/pom.xml +++ b/dhis-2/dhis-services/dhis-service-analytics/pom.xml @@ -48,6 +48,10 @@ org.hisp.dhis dhis-support-jdbc + + org.hisp.dhis + dhis-support-sql + org.hisp.dhis dhis-support-system diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/config/ServiceConfig.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/config/ServiceConfig.java index d7883952b6f..b9e50dde9f1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/config/ServiceConfig.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/config/ServiceConfig.java @@ -38,10 +38,10 @@ import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.dataelement.DataElementService; +import org.hisp.dhis.db.AnalyticsSqlBuilderProvider; +import org.hisp.dhis.db.SqlBuilderProvider; import org.hisp.dhis.db.sql.AnalyticsSqlBuilder; -import org.hisp.dhis.db.sql.AnalyticsSqlBuilderProvider; import org.hisp.dhis.db.sql.SqlBuilder; -import org.hisp.dhis.db.sql.SqlBuilderProvider; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; import org.hisp.dhis.resourcetable.ResourceTableService; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java index 18ea01e6485..75fbb8d6b10 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettings.java @@ -32,8 +32,6 @@ import static org.hisp.dhis.db.model.Logged.LOGGED; import static org.hisp.dhis.db.model.Logged.UNLOGGED; import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_DATABASE; -import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_DATABASE_CATALOG; -import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_DATABASE_DRIVER_FILENAME; import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_TABLE_SKIP_COLUMN; import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_TABLE_SKIP_INDEX; import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_TABLE_UNLOGGED; @@ -113,35 +111,6 @@ public boolean isAnalyticsDatabaseConfigured() { return config.isAnalyticsDatabaseConfigured(); } - /** - * Returns the configured analytics {@link Database}. The default is {@link Database#POSTGRESQL}. - * - * @return the analytics {@link Database}. - */ - public Database getAnalyticsDatabase() { - String value = config.getProperty(ANALYTICS_DATABASE); - String valueUpperCase = StringUtils.trimToEmpty(value).toUpperCase(); - return getAndValidateDatabase(valueUpperCase); - } - - /** - * Returns the analytics database JDBC catalog name. - * - * @return the analytics database JDBC catalog name. - */ - public String getAnalyticsDatabaseCatalog() { - return config.getProperty(ANALYTICS_DATABASE_CATALOG); - } - - /** - * Returns the analytics database JDBC driver filename. - * - * @return the analytics database JDBC driver filename. - */ - public String getAnalyticsDatabaseDriverFilename() { - return config.getProperty(ANALYTICS_DATABASE_DRIVER_FILENAME); - } - /** * Returns a set of dimension identifiers for which to skip building indexes for columns on * analytics tables. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettingsTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/setting/SqlBuilderSettingsTest.java similarity index 93% rename from dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettingsTest.java rename to dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/setting/SqlBuilderSettingsTest.java index 8a860b571a2..3b03b345b91 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/setting/AnalyticsTableSettingsTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/setting/SqlBuilderSettingsTest.java @@ -44,7 +44,7 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -class AnalyticsTableSettingsTest { +class SqlBuilderSettingsTest { @Mock private DhisConfigurationProvider config; @Mock private SystemSettingsService systemSettings; @@ -61,14 +61,6 @@ void testGetAndValidateInvalidDatabase() { assertThrows(IllegalArgumentException.class, () -> settings.getAndValidateDatabase("ORACLE")); } - @Test - void testGetAnalyticsDatabase() { - when(config.getProperty(ConfigurationKey.ANALYTICS_DATABASE)) - .thenReturn(ConfigurationKey.ANALYTICS_DATABASE.getDefaultValue()); - - assertEquals(Database.POSTGRESQL, settings.getAnalyticsDatabase()); - } - @Test void testGetSkipIndexDimensionsDefault() { when(config.getProperty(ConfigurationKey.ANALYTICS_TABLE_SKIP_INDEX)) diff --git a/dhis-2/dhis-support/dhis-support-sql/pom.xml b/dhis-2/dhis-support/dhis-support-sql/pom.xml new file mode 100644 index 00000000000..d26ebcf726a --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/pom.xml @@ -0,0 +1,107 @@ + + + 4.0.0 + + + org.hisp.dhis + dhis-support + 2.42-SNAPSHOT + + + dhis-support-sql + jar + DHIS SQL Building support + + ../../ + + + + + + + + org.hisp.dhis + dhis-support-commons + + + + org.hisp.dhis + dhis-support-external + + + + org.hisp.dhis + dhis-api + + + + org.springframework + spring-context + + + + org.springframework + spring-beans + + + + org.springframework + spring-jdbc + + + + javax.annotation + javax.annotation-api + + + + org.apache.commons + commons-text + + + + org.apache.commons + commons-lang3 + + + + org.apache.commons + commons-collections4 + + + + org.projectlombok + lombok + provided + + + + org.slf4j + slf4j-api + + + + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-inline + test + + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + + + + diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilderProvider.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java similarity index 86% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilderProvider.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java index 3f48b59a8cc..9f7aac27027 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilderProvider.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/AnalyticsSqlBuilderProvider.java @@ -25,11 +25,15 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.db.sql; +package org.hisp.dhis.db; import java.util.Objects; -import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.db.model.Database; +import org.hisp.dhis.db.setting.SqlBuilderSettings; +import org.hisp.dhis.db.sql.AnalyticsSqlBuilder; +import org.hisp.dhis.db.sql.ClickhouseAnalyticsSqlBuilder; +import org.hisp.dhis.db.sql.DorisAnalyticsSqlBuilder; +import org.hisp.dhis.db.sql.PostgresAnalyticsSqlBuilder; import org.hisp.dhis.external.conf.DhisConfigurationProvider; import org.springframework.stereotype.Service; @@ -38,7 +42,7 @@ public class AnalyticsSqlBuilderProvider { private final AnalyticsSqlBuilder analyticsSqlBuilder; - public AnalyticsSqlBuilderProvider(AnalyticsTableSettings config) { + public AnalyticsSqlBuilderProvider(SqlBuilderSettings config) { Objects.requireNonNull(config); this.analyticsSqlBuilder = getSqlBuilder(config); } @@ -59,7 +63,7 @@ public AnalyticsSqlBuilder getAnalyticsSqlBuilder() { * @param config the {@link DhisConfigurationProvider}. * @return a {@link AnalyticsSqlBuilder}. */ - private AnalyticsSqlBuilder getSqlBuilder(AnalyticsTableSettings config) { + private AnalyticsSqlBuilder getSqlBuilder(SqlBuilderSettings config) { Database database = config.getAnalyticsDatabase(); Objects.requireNonNull(database); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/SqlBuilderProvider.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/SqlBuilderProvider.java similarity index 88% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/SqlBuilderProvider.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/SqlBuilderProvider.java index 23d85a3b475..4be195a8503 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/SqlBuilderProvider.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/SqlBuilderProvider.java @@ -25,11 +25,15 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.db.sql; +package org.hisp.dhis.db; import java.util.Objects; -import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.db.model.Database; +import org.hisp.dhis.db.setting.SqlBuilderSettings; +import org.hisp.dhis.db.sql.ClickHouseSqlBuilder; +import org.hisp.dhis.db.sql.DorisSqlBuilder; +import org.hisp.dhis.db.sql.PostgreSqlBuilder; +import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.external.conf.DhisConfigurationProvider; import org.springframework.stereotype.Service; @@ -38,7 +42,7 @@ public class SqlBuilderProvider { private final SqlBuilder sqlBuilder; - public SqlBuilderProvider(AnalyticsTableSettings config) { + public SqlBuilderProvider(SqlBuilderSettings config) { Objects.requireNonNull(config); this.sqlBuilder = getSqlBuilder(config); } @@ -58,7 +62,7 @@ public SqlBuilder getSqlBuilder() { * @param config the {@link DhisConfigurationProvider}. * @return a {@link SqlBuilder}. */ - private SqlBuilder getSqlBuilder(AnalyticsTableSettings config) { + private SqlBuilder getSqlBuilder(SqlBuilderSettings config) { Database database = config.getAnalyticsDatabase(); String catalog = config.getAnalyticsDatabaseCatalog(); String driverFilename = config.getAnalyticsDatabaseDriverFilename(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/init/AnalyticsDatabaseInit.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java similarity index 96% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/init/AnalyticsDatabaseInit.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java index 8f07af3b5d8..1bce670ce47 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/init/AnalyticsDatabaseInit.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/init/AnalyticsDatabaseInit.java @@ -25,7 +25,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.analytics.table.init; +package org.hisp.dhis.db.init; import static org.hisp.dhis.db.sql.ClickHouseSqlBuilder.NAMED_COLLECTION; @@ -33,11 +33,11 @@ import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; +import org.hisp.dhis.db.SqlBuilderProvider; import org.hisp.dhis.db.model.Database; +import org.hisp.dhis.db.setting.SqlBuilderSettings; import org.hisp.dhis.db.sql.ClickHouseSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; -import org.hisp.dhis.db.sql.SqlBuilderProvider; import org.hisp.dhis.external.conf.ConfigurationKey; import org.hisp.dhis.external.conf.DhisConfigurationProvider; import org.springframework.beans.factory.annotation.Qualifier; @@ -64,7 +64,7 @@ public class AnalyticsDatabaseInit { private final DhisConfigurationProvider config; - private final AnalyticsTableSettings settings; + private final SqlBuilderSettings settings; @Qualifier("analyticsJdbcTemplate") private final JdbcTemplate jdbcTemplate; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Collation.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Collation.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Collation.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Collation.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Column.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Column.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Column.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Column.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/DataType.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/DataType.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/DataType.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/DataType.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Database.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Database.java similarity index 93% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Database.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Database.java index 57a73caf358..a8894553da5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Database.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Database.java @@ -27,10 +27,6 @@ */ package org.hisp.dhis.db.model; -import org.hisp.dhis.analytics.table.init.AnalyticsDatabaseInit; -import org.hisp.dhis.db.sql.SqlBuilder; -import org.hisp.dhis.db.sql.SqlBuilderProvider; - /** * Enumeration of database platforms. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Index.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Index.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Index.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Index.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/IndexFunction.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/IndexFunction.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/IndexFunction.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/IndexFunction.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/IndexType.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/IndexType.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/IndexType.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/IndexType.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Logged.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Logged.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Logged.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Logged.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Table.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Table.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Table.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/Table.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/TablePartition.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/TablePartition.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/TablePartition.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/TablePartition.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/constraint/Nullable.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/constraint/Nullable.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/constraint/Nullable.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/constraint/Nullable.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/constraint/Unique.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/constraint/Unique.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/constraint/Unique.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/model/constraint/Unique.java diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/setting/SqlBuilderSettings.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/setting/SqlBuilderSettings.java new file mode 100644 index 00000000000..a62bd5b8835 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/setting/SqlBuilderSettings.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.setting; + +import static org.hisp.dhis.commons.util.TextUtils.format; +import static org.hisp.dhis.db.model.Logged.LOGGED; +import static org.hisp.dhis.db.model.Logged.UNLOGGED; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_DATABASE; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_DATABASE_CATALOG; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_DATABASE_DRIVER_FILENAME; +import static org.hisp.dhis.external.conf.ConfigurationKey.ANALYTICS_TABLE_UNLOGGED; +import static org.hisp.dhis.util.ObjectUtils.isNull; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.db.model.Database; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.external.conf.DhisConfigurationProvider; +import org.springframework.stereotype.Component; + +/** + * Component responsible for exposing analytics table settings related to the SqlBuilder. The source + * of the settings are configuration files (dhis.conf) and system settings. + */ +@Component +@RequiredArgsConstructor +public class SqlBuilderSettings { + private final DhisConfigurationProvider config; + + /** + * Returns the setting indicating whether resource and analytics tables should be logged or + * unlogged. + * + * @return the {@link Logged} parameter. + */ + public Logged getTableLogged() { + if (config.isEnabled(ANALYTICS_TABLE_UNLOGGED)) { + return UNLOGGED; + } + + return LOGGED; + } + + /** + * Returns the analytics database JDBC catalog name. + * + * @return the analytics database JDBC catalog name. + */ + public String getAnalyticsDatabaseCatalog() { + return config.getProperty(ANALYTICS_DATABASE_CATALOG); + } + + /** + * Returns the configured analytics {@link Database}. The default is {@link Database#POSTGRESQL}. + * + * @return the analytics {@link Database}. + */ + public Database getAnalyticsDatabase() { + String value = config.getProperty(ANALYTICS_DATABASE); + String valueUpperCase = StringUtils.trimToEmpty(value).toUpperCase(); + return getAndValidateDatabase(valueUpperCase); + } + + /** + * Returns the analytics database JDBC driver filename. + * + * @return the analytics database JDBC driver filename. + */ + public String getAnalyticsDatabaseDriverFilename() { + return config.getProperty(ANALYTICS_DATABASE_DRIVER_FILENAME); + } + + /** + * Returns the {@link Database} matching the given value. + * + * @param value the string value. + * @return the {@link Database}. + * @throws IllegalArgumentException if the value does not match a valid option. + */ + Database getAndValidateDatabase(String value) { + Database database = EnumUtils.getEnum(Database.class, value); + + if (isNull(database)) { + String message = + format( + "Property '{}' has illegal value: '{}', allowed options: {}", + ANALYTICS_DATABASE.getKey(), + value, + StringUtils.join(Database.values(), ',')); + throw new IllegalArgumentException(message); + } + + return database; + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java similarity index 96% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java index a354b4ddaa9..24dfa411c05 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java @@ -187,7 +187,7 @@ protected String getDataTypeName(DataType dataType) { case JSONB -> dataTypeJson(); default -> throw new UnsupportedOperationException( - String.format("Unsuported data type: %s", dataType)); + String.format("Unsupported data type: %s", dataType)); }; } @@ -203,7 +203,7 @@ protected String getIndexFunctionName(IndexFunction indexFunction) { case LOWER -> indexFunctionLower(); default -> throw new UnsupportedOperationException( - String.format("Unsuported index function: %s", indexFunction)); + String.format("Unsupported index function: %s", indexFunction)); }; } @@ -220,7 +220,7 @@ protected String getIndexTypeName(IndexType indexType) { case GIN -> indexTypeGin(); default -> throw new UnsupportedOperationException( - String.format("Unsuported index type: %s", indexType)); + String.format("Unsupported index type: %s", indexType)); }; } @@ -255,7 +255,7 @@ protected String toIndexColumn(Index index, String column) { * Indicates that the feature or syntax is not supported by throwing an {@link * UnsupportedOperationException}. * - * @throws UnsupportedOperationException + * @throws UnsupportedOperationException if the feature or syntax is not supported. */ protected String notSupported() { throw new UnsupportedOperationException(); @@ -265,7 +265,7 @@ protected String notSupported() { * Converts the given collection to a comma-separated string, using the given mapping function to * convert each item in the collection to a string. * - * @param + * @param the type of the collection. * @param collection the {@link Collection}. * @param mapper the string mapping {@link Function}. * @return a comma-separated string. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AnalyticsSqlBuilder.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickhouseAnalyticsSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickhouseAnalyticsSqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickhouseAnalyticsSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickhouseAnalyticsSqlBuilder.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisAnalyticsSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisAnalyticsSqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisAnalyticsSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisAnalyticsSqlBuilder.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgresAnalyticsSqlBuilder.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java rename to dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/ColumnTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/ColumnTest.java new file mode 100644 index 00000000000..e1d2f48443c --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/ColumnTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.model; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hisp.dhis.db.model.constraint.Nullable; +import org.junit.jupiter.api.Test; + +class ColumnTest { + @Test + void testIsNotNull() { + Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL); + Column colB = new Column("value", DataType.DOUBLE, Nullable.NULL); + + assertTrue(colA.isNotNull()); + assertFalse(colB.isNotNull()); + } + + @Test + void testHasCollation() { + Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL, Collation.DEFAULT); + Column colB = new Column("ou", DataType.CHARACTER_11, Nullable.NOT_NULL, Collation.C); + Column colC = new Column("value", DataType.DOUBLE, Nullable.NULL); + + assertFalse(colA.hasCollation()); + assertTrue(colB.hasCollation()); + assertFalse(colC.hasCollation()); + } +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java new file mode 100644 index 00000000000..1c75b3c3cdd --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.model; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class DataTypeTest { + @Test + void testIsNumeric() { + assertTrue(DataType.BIGINT.isNumeric()); + assertFalse(DataType.CHARACTER_11.isNumeric()); + } + + @Test + void testIsBoolean() { + assertTrue(DataType.BOOLEAN.isBoolean()); + assertFalse(DataType.DOUBLE.isBoolean()); + } + + @Test + void testIsCharacter() { + assertTrue(DataType.VARCHAR_255.isCharacter()); + assertFalse(DataType.DECIMAL.isCharacter()); + } +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/IndexTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/IndexTest.java new file mode 100644 index 00000000000..c7d46a847a7 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/IndexTest.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.model; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.hisp.dhis.db.model.constraint.Unique; +import org.junit.jupiter.api.Test; + +class IndexTest { + @Test + void testIsUnique() { + Index indexA = + Index.builder() + .name("in_analytics_id") + .tableName("analytics") + .unique(Unique.UNIQUE) + .columns(List.of("id")) + .build(); + + Index indexB = + Index.builder() + .name("in_analytics_dx") + .tableName("analytics") + .columns(List.of("dx")) + .build(); + + assertTrue(indexA.isUnique()); + assertFalse(indexB.isUnique()); + } + + @Test + void testDefaults() { + Index.IndexBuilder builder = Index.builder(); + + Index index = builder.build(); + + assertNull(index.getName()); + assertNull(index.getTableName()); + assertNull(index.getCondition()); + assertNull(index.getFunction()); + assertNull(index.getColumns()); + assertNull(index.getSortOrder()); + assertSame(IndexType.BTREE, index.getIndexType()); + assertFalse(index.isUnique()); + } +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/TableTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/TableTest.java new file mode 100644 index 00000000000..bc57ee6db4c --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/model/TableTest.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.model; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.junit.jupiter.api.Test; + +class TableTest { + private final Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL); + private final Column colB = new Column("value", DataType.DOUBLE, Nullable.NULL); + + @Test + void testToStagingTable() { + assertEquals( + "analytics_rs_categorystructure_temp", Table.toStaging("analytics_rs_categorystructure")); + assertEquals("analytics_temp", Table.toStaging("analytics")); + } + + @Test + void testFromStagingTable() { + assertEquals( + "analytics_rs_categorystructure", Table.fromStaging("analytics_rs_categorystructure_temp")); + assertEquals("analytics", Table.fromStaging("analytics_temp")); + } + + @Test + void testIsUnlogged() { + List columns = List.of(colA, colB); + + Table tableA = new Table("analytics", columns, List.of(), Logged.UNLOGGED); + Table tableB = new Table("analytics", columns, List.of(), Logged.LOGGED); + + assertTrue(tableA.isUnlogged()); + assertFalse(tableB.isUnlogged()); + } + + @Test + void testHasColumns() { + Table table = new Table("analytics", List.of(colA, colB), List.of()); + + assertTrue(table.hasColumns()); + } + + @Test + void getFirstColumn() { + Table table = new Table("analytics", List.of(colA, colB), List.of()); + + assertEquals(colA, table.getFirstColumn()); + } + + @Test + void testHasPrimaryKey() { + Table tableA = new Table("analytics", List.of(colA, colB), List.of("dx")); + Table tableB = new Table("analytics", List.of(colA, colB), List.of()); + + assertTrue(tableA.hasPrimaryKey()); + assertFalse(tableB.hasPrimaryKey()); + assertEquals(List.of("dx"), tableA.getPrimaryKey()); + } + + @Test + void testGetFirstPrimaryKey() { + Table table = new Table("analytics", List.of(colA, colB), List.of("dx", "value")); + + assertEquals("dx", table.getFirstPrimaryKey()); + } + + @Test + void testHasSortKey() { + Table tableA = + new Table( + "analytics", + List.of(colA, colB), + List.of("dx", "value"), + List.of("dx"), + List.of(), + Logged.UNLOGGED); + Table tableB = new Table("analytics", List.of(colA, colB), List.of("dx", "value")); + + assertTrue(tableA.hasSortKey()); + assertFalse(tableB.hasSortKey()); + assertEquals(List.of("dx"), tableA.getSortKey()); + } + + @Test + void testSuccessfulValidation() { + List columns = List.of(colA); + List primaryKey = List.of(); + + assertDoesNotThrow(() -> new Table("analytics", columns, primaryKey)); + } + + @Test + void testNameValidation() { + List columns = List.of(colA); + List primaryKey = List.of(); + + assertThrows(NullPointerException.class, () -> new Table(null, columns, primaryKey)); + assertThrows(IllegalArgumentException.class, () -> new Table("", columns, primaryKey)); + } + + @Test + void testColumnsParentValidation() { + List columns = List.of(); + List primaryKey = List.of(); + List checks = List.of(); + + assertThrows(IllegalArgumentException.class, () -> new Table("analytics", columns, primaryKey)); + assertThrows( + IllegalArgumentException.class, + () -> new Table("analytics", columns, primaryKey, checks, Logged.UNLOGGED, null)); + } +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/setting/SqlBuilderSettingsTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/setting/SqlBuilderSettingsTest.java new file mode 100644 index 00000000000..05a953af039 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/setting/SqlBuilderSettingsTest.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.setting; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +import org.hisp.dhis.db.model.Database; +import org.hisp.dhis.external.conf.ConfigurationKey; +import org.hisp.dhis.external.conf.DhisConfigurationProvider; +import org.hisp.dhis.setting.SystemSettingsService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class SqlBuilderSettingsTest { + @Mock private DhisConfigurationProvider config; + + @Mock private SystemSettingsService systemSettings; + + @InjectMocks private SqlBuilderSettings settings; + + @Test + void testGetAndValidateDatabase() { + assertEquals(Database.POSTGRESQL, settings.getAndValidateDatabase("POSTGRESQL")); + } + + @Test + void testGetAndValidateInvalidDatabase() { + assertThrows(IllegalArgumentException.class, () -> settings.getAndValidateDatabase("ORACLE")); + } + + @Test + void testGetAnalyticsDatabase() { + when(config.getProperty(ConfigurationKey.ANALYTICS_DATABASE)) + .thenReturn(ConfigurationKey.ANALYTICS_DATABASE.getDefaultValue()); + + assertEquals(Database.POSTGRESQL, settings.getAnalyticsDatabase()); + } +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java new file mode 100644 index 00000000000..5087172105e --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.sql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.hisp.dhis.db.model.Collation; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.junit.jupiter.api.Test; + +class ClickHouseSqlBuilderTest { + private final ClickHouseSqlBuilder sqlBuilder = new ClickHouseSqlBuilder(); + + private Table getTableA() { + List columns = + List.of( + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), + new Column("created", DataType.TIMESTAMP), + new Column("user", DataType.JSONB), + new Column("value", DataType.DOUBLE)); + + List primaryKey = List.of("id"); + + return new Table("immunization", columns, primaryKey, Logged.LOGGED); + } + + private Table getTableB() { + List columns = + List.of( + new Column("id", DataType.INTEGER, Nullable.NOT_NULL), + new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), + new Column("bcg_doses", DataType.DOUBLE)); + + return new Table("vaccination", columns, List.of()); + } + + private Table getTableC() { + List columns = + List.of( + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("vitamin_a", DataType.BIGINT), + new Column("vitamin_d", DataType.BIGINT)); + + List primaryKey = List.of("id"); + + return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); + } + + private Table getTableD() { + List columns = + List.of( + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), + new Column("value", DataType.DOUBLE)); + + List sortKey = List.of("data", "period"); + + return new Table("immunization", columns, List.of(), sortKey, List.of(), Logged.LOGGED); + } + + // Data types + + @Test + void testDataTypes() { + assertEquals("Float64", sqlBuilder.dataTypeDouble()); + assertEquals("DateTime64(3)", sqlBuilder.dataTypeTimestamp()); + } + + // Index types + + @Test + void testIndexTypes() { + assertThrows(UnsupportedOperationException.class, () -> sqlBuilder.indexTypeBtree()); + } + + // Capabilities + + @Test + void testSupportsAnalyze() { + assertFalse(sqlBuilder.supportsAnalyze()); + } + + @Test + void testSupportsVacuum() { + assertFalse(sqlBuilder.supportsVacuum()); + } + + // Utilities + + @Test + void testQuote() { + assertEquals( + "\"Treated \"\"malaria\"\" at facility\"", + sqlBuilder.quote("Treated \"malaria\" at facility")); + assertEquals( + "\"\"\"Patients on \"\"treatment\"\" for TB\"\"\"", + sqlBuilder.quote("\"Patients on \"treatment\" for TB\"")); + assertEquals("\"quarterly\"", sqlBuilder.quote("quarterly")); + assertEquals("\"Fully immunized\"", sqlBuilder.quote("Fully immunized")); + } + + @Test + void testQuoteAlias() { + assertEquals( + "ax.\"Treated \"\"malaria\"\" at facility\"", + sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); + assertEquals( + "analytics.\"Patients on \"\"treatment\"\" for TB\"", + sqlBuilder.quote("analytics", "Patients on \"treatment\" for TB")); + assertEquals("analytics.\"quarterly\"", sqlBuilder.quote("analytics", "quarterly")); + assertEquals("dv.\"Fully immunized\"", sqlBuilder.quote("dv", "Fully immunized")); + } + + @Test + void testQuoteAx() { + assertEquals( + "ax.\"Treated \"\"malaria\"\" at facility\"", + sqlBuilder.quoteAx("Treated \"malaria\" at facility")); + assertEquals("ax.\"quarterly\"", sqlBuilder.quoteAx("quarterly")); + assertEquals("ax.\"Fully immunized\"", sqlBuilder.quoteAx("Fully immunized")); + } + + @Test + void testSingleQuote() { + assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); + assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); + assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); + } + + @Test + void testEscape() { + assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); + assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); + } + + @Test + void testSinqleQuotedCommaDelimited() { + assertEquals( + "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", + sqlBuilder.singleQuotedCommaDelimited( + List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); + assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); + assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); + assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); + } + + @Test + void testQualifyTable() { + assertEquals("postgresql(\"pg_dhis\", table='category')", sqlBuilder.qualifyTable("category")); + assertEquals( + "postgresql(\"pg_dhis\", table='categories_options')", + sqlBuilder.qualifyTable("categories_options")); + } + + @Test + void testDateTrunc() { + assertEquals( + "date_trunc('month', pe.startdate)", sqlBuilder.dateTrunc("month", "pe.startdate")); + } + + @Test + void testDifferenceInSeconds() { + assertEquals( + "(toUnixTimestamp(a.startdate) - toUnixTimestamp(b.enddate))", + sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); + assertEquals( + "(toUnixTimestamp(a.\"startdate\") - toUnixTimestamp(b.\"enddate\"))", + sqlBuilder.differenceInSeconds( + sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); + } + + @Test + void testRegexpMatch() { + assertEquals("match(value, 'test')", sqlBuilder.regexpMatch("value", "'test'")); + assertEquals("match(number, '\\d')", sqlBuilder.regexpMatch("number", "'\\d'")); + assertEquals("match(color, '^Blue$')", sqlBuilder.regexpMatch("color", "'^Blue$'")); + assertEquals("match(id, '[a-z]\\w+\\d{3}')", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); + } + + @Test + void testJsonExtract() { + assertEquals( + "JSONExtractString(value, 'D7m8vpzxHDJ')", sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); + } + + @Test + void testJsonExtractNested() { + assertEquals( + "JSONExtractString(eventdatavalues, 'D7m8vpzxHDJ.value')", + sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); + } + + // Statements + + @Test + void testCreateTableA() { + Table table = getTableA(); + + String expected = + """ + create table "immunization" ("id" Int64 not null,"data" String not null,\ + "period" String not null,"created" DateTime64(3) null,"user" JSON null,\ + "value" Float64 null) \ + engine = MergeTree() \ + order by ("id");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testCreateTableB() { + Table table = getTableB(); + + String expected = + """ + create table "vaccination" ("id" Int32 not null,\ + "facility_type" String null,"bcg_doses" Float64 null) \ + engine = MergeTree() \ + order by ("id");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testCreateTableC() { + Table table = getTableC(); + + String expected = + """ + create table "nutrition" ("id" Int64 not null,"vitamin_a" Int64 null,\ + "vitamin_d" Int64 null) \ + engine = MergeTree() order by ("id");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testCreateTableD() { + Table table = getTableD(); + + String expected = + """ + create table "immunization" ("id" Int64 not null,"data" String not null,\ + "period" String not null,"value" Float64 null) \ + engine = MergeTree() \ + order by ("data","period");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testRenameTable() { + Table table = getTableA(); + + String expected = "rename table \"immunization\" to \"vaccination\";"; + + assertEquals(expected, sqlBuilder.renameTable(table, "vaccination")); + } + + @Test + void testDropTableIfExists() { + Table table = getTableA(); + + String expected = "drop table if exists \"immunization\";"; + + assertEquals(expected, sqlBuilder.dropTableIfExists(table)); + } + + @Test + void testDropTableIfExistsString() { + String expected = "drop table if exists \"immunization\";"; + + assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); + } + + @Test + void testDropTableIfExistsCascade() { + Table table = getTableA(); + + String expected = "drop table if exists \"immunization\";"; + + assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); + } + + @Test + void testDropTableIfExistsCascadeString() { + String expected = "drop table if exists \"immunization\";"; + + assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); + } + + @Test + void testSwapTable() { + String expected = + """ + drop table if exists "vaccination"; \ + rename table "immunization" to "vaccination";"""; + + assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); + } + + @Test + void testTableExists() { + String expected = + """ + select t.name as table_name \ + from system.tables t \ + where t.database = 'default' \ + and t.name = 'immunization' \ + and engine not in ('View', 'Materialized View');"""; + + assertEquals(expected, sqlBuilder.tableExists("immunization")); + } + + @Test + void testCountRows() { + String expected = + """ + select count(*) as row_count from \"immunization\";"""; + + assertEquals(expected, sqlBuilder.countRows(getTableA())); + } + + // Named collection + + @Test + void testCreateNamedCollection() { + String expected = + """ + create named collection "pg_dhis" as """; + + assertTrue( + sqlBuilder + .createNamedCollection("pg_dhis", Map.of("host", "mydomain.org")) + .startsWith(expected)); + } + + @Test + void testDropNamedCollectionIfExists() { + assertEquals( + "drop named collection if exists \"pg_dhis\";", + sqlBuilder.dropNamedCollectionIfExists("pg_dhis")); + } +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java new file mode 100644 index 00000000000..ad372141c9b --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.sql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import org.hisp.dhis.db.model.Collation; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.junit.jupiter.api.Test; + +class DorisSqlBuilderTest { + private final SqlBuilder sqlBuilder = new DorisSqlBuilder("pg_dhis", "postgresql.jar"); + + private Table getTableA() { + List columns = + List.of( + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), + new Column("created", DataType.TIMESTAMP), + new Column("user", DataType.JSONB), + new Column("value", DataType.DOUBLE)); + + List primaryKey = List.of("id"); + + return new Table("immunization", columns, primaryKey, Logged.LOGGED); + } + + private Table getTableB() { + List columns = + List.of( + new Column("id", DataType.INTEGER, Nullable.NOT_NULL), + new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), + new Column("bcg_doses", DataType.DOUBLE)); + + return new Table("vaccination", columns, List.of(), Logged.UNLOGGED); + } + + private Table getTableC() { + List columns = + List.of( + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("vitamin_a", DataType.BIGINT), + new Column("vitamin_d", DataType.BIGINT)); + + List primaryKey = List.of("id"); + + return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); + } + + // Data types + + @Test + void testDataTypes() { + assertEquals("double", sqlBuilder.dataTypeDouble()); + assertEquals("datetime", sqlBuilder.dataTypeTimestamp()); + } + + // Index types + + @Test + void testIndexTypes() { + assertThrows(UnsupportedOperationException.class, () -> sqlBuilder.indexTypeBtree()); + } + + // Capabilities + + @Test + void testSupportsAnalyze() { + assertFalse(sqlBuilder.supportsAnalyze()); + } + + @Test + void testSupportsVacuum() { + assertFalse(sqlBuilder.supportsVacuum()); + } + + // Utilities + + @Test + void testQuote() { + assertEquals( + "`Treated \"malaria\" at facility`", sqlBuilder.quote("Treated \"malaria\" at facility")); + assertEquals( + "`Patients on ``treatment`` for TB`", sqlBuilder.quote("Patients on `treatment` for TB")); + assertEquals("`quarterly`", sqlBuilder.quote("quarterly")); + assertEquals("`Fully immunized`", sqlBuilder.quote("Fully immunized")); + } + + @Test + void testQuoteAlias() { + assertEquals( + "ax.`Treated \"malaria\" at facility`", + sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); + assertEquals( + "analytics.`Patients on ``treatment`` for TB`", + sqlBuilder.quote("analytics", "Patients on `treatment` for TB")); + assertEquals("analytics.`quarterly`", sqlBuilder.quote("analytics", "quarterly")); + assertEquals("dv.`Fully immunized`", sqlBuilder.quote("dv", "Fully immunized")); + } + + @Test + void testQuoteAx() { + assertEquals( + "ax.`Treated ``malaria`` at facility`", + sqlBuilder.quoteAx("Treated `malaria` at facility")); + assertEquals("ax.`quarterly`", sqlBuilder.quoteAx("quarterly")); + assertEquals("ax.`Fully immunized`", sqlBuilder.quoteAx("Fully immunized")); + } + + @Test + void testSingleQuote() { + assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); + assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); + assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); + } + + @Test + void testEscape() { + assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); + assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); + assertEquals("C:\\\\Downloads\\\\File.doc", sqlBuilder.escape("C:\\Downloads\\File.doc")); + } + + @Test + void testSinqleQuotedCommaDelimited() { + assertEquals( + "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", + sqlBuilder.singleQuotedCommaDelimited( + List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); + assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); + assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); + assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); + } + + @Test + void testQualifyTable() { + assertEquals("pg_dhis.public.`category`", sqlBuilder.qualifyTable("category")); + assertEquals( + "pg_dhis.public.`categories_options`", sqlBuilder.qualifyTable("categories_options")); + } + + @Test + void testDateTrunc() { + assertEquals( + "date_trunc(pe.startdate, 'month')", sqlBuilder.dateTrunc("month", "pe.startdate")); + } + + @Test + void testDifferenceInSeconds() { + assertEquals( + "(unix_timestamp(a.startdate) - unix_timestamp(b.enddate))", + sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); + assertEquals( + "(unix_timestamp(a.`startdate`) - unix_timestamp(b.`enddate`))", + sqlBuilder.differenceInSeconds( + sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); + } + + @Test + void testRegexpMatch() { + assertEquals("value regexp 'test'", sqlBuilder.regexpMatch("value", "'test'")); + assertEquals("number regexp '\\d'", sqlBuilder.regexpMatch("number", "'\\d'")); + assertEquals("color regexp '^Blue$'", sqlBuilder.regexpMatch("color", "'^Blue$'")); + assertEquals("id regexp '[a-z]\\w+\\d{3}'", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); + } + + @Test + void testJsonExtract() { + assertEquals( + "json_unquote(json_extract(value, '$.D7m8vpzxHDJ'))", + sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); + } + + @Test + void testJsonExtractNested() { + assertEquals( + "json_unquote(json_extract(eventdatavalues, '$.D7m8vpzxHDJ.value'))", + sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); + } + + // Statements + + @Test + void testCreateTableA() { + Table table = getTableA(); + + String expected = + """ + create table `immunization` (`id` bigint not null,\ + `data` char(11) not null,`period` varchar(50) not null,\ + `created` datetime null,`user` json null,`value` double null) \ + engine = olap \ + unique key (`id`) \ + distributed by hash(`id`) buckets 10 \ + properties ("replication_num" = "1");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testCreateTableB() { + Table table = getTableB(); + + String expected = + """ + create table `vaccination` (`id` int not null,\ + `facility_type` varchar(255) null,`bcg_doses` double null) \ + engine = olap \ + duplicate key (`id`) \ + distributed by hash(`id`) buckets 10 \ + properties ("replication_num" = "1");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + // void testCreateTableB() + + @Test + void testCreateTableC() { + Table table = getTableC(); + + String expected = + """ + create table `nutrition` (`id` bigint not null,\ + `vitamin_a` bigint null,`vitamin_d` bigint null) \ + engine = olap \ + unique key (`id`) \ + distributed by hash(`id`) buckets 10 \ + properties ("replication_num" = "1");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testCreateCatalog() { + String expected = + """ + create catalog `pg_dhis` \ + properties ( + "type" = "jdbc", \ + "user" = "dhis", \ + "password" = "kH7g", \ + "jdbc_url" = "jdbc:mysql://127.18.0.1:9030/dev?useUnicode=true&characterEncoding=UTF-8", \ + "driver_url" = "postgresql.jar", \ + "driver_class" = "org.postgresql.Driver" + );"""; + + assertEquals( + expected, + sqlBuilder.createCatalog( + "jdbc:mysql://127.18.0.1:9030/dev?useUnicode=true&characterEncoding=UTF-8", + "dhis", + "kH7g")); + } + + @Test + void testDropCatalogIfExists() { + String expected = "drop catalog if exists `pg_dhis`;"; + + assertEquals(expected, sqlBuilder.dropCatalogIfExists()); + } + + @Test + void testRenameTable() { + Table table = getTableA(); + + String expected = + """ + alter table `immunization` rename `immunization_main`;"""; + + assertEquals(expected, sqlBuilder.renameTable(table, "immunization_main")); + } + + @Test + void testDropTableIfExists() { + Table table = getTableA(); + + String expected = "drop table if exists `immunization`;"; + + assertEquals(expected, sqlBuilder.dropTableIfExists(table)); + } + + @Test + void testDropTableIfExistsString() { + String expected = "drop table if exists `immunization`;"; + + assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); + } + + @Test + void testDropTableIfExistsCascade() { + Table table = getTableA(); + + String expected = "drop table if exists `immunization`;"; + + assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); + } + + @Test + void testDropTableIfExistsCascadeString() { + String expected = "drop table if exists `immunization`;"; + + assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); + } + + @Test + void testSwapTable() { + String expected = + """ + drop table if exists `vaccination`; \ + alter table `immunization` rename `vaccination`;"""; + + assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); + } + + @Test + void testTableExists() { + String expected = + """ + select t.table_name from information_schema.tables t \ + where t.table_schema = 'public' and t.table_name = 'immunization';"""; + + assertEquals(expected, sqlBuilder.tableExists("immunization")); + } + + @Test + void testCountRows() { + String expected = + """ + select count(*) as row_count from `immunization`;"""; + + assertEquals(expected, sqlBuilder.countRows(getTableA())); + } +} diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java new file mode 100644 index 00000000000..2d621ae766e --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java @@ -0,0 +1,452 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.db.sql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.hisp.dhis.db.model.Collation; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Index; +import org.hisp.dhis.db.model.IndexFunction; +import org.hisp.dhis.db.model.IndexType; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.hisp.dhis.db.model.constraint.Unique; +import org.junit.jupiter.api.Test; + +class PostgreSqlBuilderTest { + private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); + + private Table getTableA() { + List columns = + List.of( + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), + new Column("created", DataType.TIMESTAMP), + new Column("user", DataType.JSONB), + new Column("value", DataType.DOUBLE)); + + List primaryKey = List.of("id"); + + return new Table("immunization", columns, primaryKey, Logged.LOGGED); + } + + private List getIndexesA() { + return List.of( + Index.builder() + .name("in_immunization_data") + .tableName("immunization") + .columns(List.of("data")) + .build(), + Index.builder() + .name("in_immunization_period_created") + .tableName("immunization") + .columns(List.of("period", "created")) + .build(), + Index.builder() + .name("in_immunization_user") + .tableName("immunization") + .indexType(IndexType.GIN) + .columns(List.of("user")) + .build(), + Index.builder() + .name("in_immunization_data_period") + .tableName("immunization") + .columns(List.of("data", "period")) + .function(IndexFunction.LOWER) + .build()); + } + + private Table getTableB() { + List columns = + List.of( + new Column("id", DataType.INTEGER, Nullable.NOT_NULL), + new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), + new Column("bcg_doses", DataType.DOUBLE)); + + List checks = List.of("\"id\">0", "\"bcg_doses\">0"); + + return new Table("vaccination", columns, List.of(), List.of(), checks, Logged.UNLOGGED); + } + + private Table getTableC() { + List columns = + List.of( + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("vitamin_a", DataType.BIGINT), + new Column("vitamin_d", DataType.BIGINT)); + + List primaryKey = List.of("id"); + + return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); + } + + // Data types + + @Test + void testDataTypes() { + assertEquals("double precision", sqlBuilder.dataTypeDouble()); + assertEquals("geometry", sqlBuilder.dataTypeGeometry()); + } + + // Index types + + @Test + void testIndexTypes() { + assertEquals("btree", sqlBuilder.indexTypeBtree()); + assertEquals("gist", sqlBuilder.indexTypeGist()); + assertEquals("gin", sqlBuilder.indexTypeGin()); + } + + // Capabilities + + @Test + void testSupportsAnalyze() { + assertTrue(sqlBuilder.supportsAnalyze()); + } + + @Test + void testSupportsVacuum() { + assertTrue(sqlBuilder.supportsVacuum()); + } + + // Utilities + + @Test + void testQuote() { + assertEquals( + "\"Treated \"\"malaria\"\" at facility\"", + sqlBuilder.quote("Treated \"malaria\" at facility")); + assertEquals("\"quarterly\"", sqlBuilder.quote("quarterly")); + assertEquals("\"Fully immunized\"", sqlBuilder.quote("Fully immunized")); + } + + @Test + void testQuoteAlias() { + assertEquals( + "ax.\"Treated \"\"malaria\"\" at facility\"", + sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); + assertEquals("analytics.\"quarterly\"", sqlBuilder.quote("analytics", "quarterly")); + assertEquals("dv.\"Fully immunized\"", sqlBuilder.quote("dv", "Fully immunized")); + } + + @Test + void testQuoteAx() { + assertEquals( + "ax.\"Treated \"\"malaria\"\" at facility\"", + sqlBuilder.quoteAx("Treated \"malaria\" at facility")); + assertEquals("ax.\"quarterly\"", sqlBuilder.quoteAx("quarterly")); + assertEquals("ax.\"Fully immunized\"", sqlBuilder.quoteAx("Fully immunized")); + } + + @Test + void testSingleQuote() { + assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); + assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); + assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); + } + + @Test + void testEscape() { + assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); + assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); + assertEquals("C:\\\\Downloads\\\\File.doc", sqlBuilder.escape("C:\\Downloads\\File.doc")); + } + + @Test + void testSinqleQuotedCommaDelimited() { + assertEquals( + "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", + sqlBuilder.singleQuotedCommaDelimited( + List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); + assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); + assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); + assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); + } + + @Test + void testQualifyTable() { + assertEquals("\"category\"", sqlBuilder.qualifyTable("category")); + assertEquals("\"categories_options\"", sqlBuilder.qualifyTable("categories_options")); + } + + @Test + void testDateTrunc() { + assertEquals( + "date_trunc('month', pe.startdate)", sqlBuilder.dateTrunc("month", "pe.startdate")); + } + + @Test + void testDifferenceInSeconds() { + assertEquals( + "extract(epoch from (a.startdate - b.enddate))", + sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); + assertEquals( + "extract(epoch from (a.\"startdate\" - b.\"enddate\"))", + sqlBuilder.differenceInSeconds( + sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); + } + + @Test + void testRegexpMatch() { + assertEquals("value ~* 'test'", sqlBuilder.regexpMatch("value", "'test'")); + assertEquals("number ~* '\\d'", sqlBuilder.regexpMatch("number", "'\\d'")); + assertEquals("color ~* '^Blue$'", sqlBuilder.regexpMatch("color", "'^Blue$'")); + assertEquals("id ~* '[a-z]\\w+\\d{3}'", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); + } + + @Test + void testJsonExtract() { + assertEquals("value ->> 'D7m8vpzxHDJ'", sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); + } + + @Test + void testJsonExtractNested() { + assertEquals( + "eventdatavalues #>> '{D7m8vpzxHDJ, value}'", + sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); + } + + // Statements + + @Test + void testCreateTableA() { + Table table = getTableA(); + + String expected = + """ + create table "immunization" ("id" bigint not null, "data" char(11) not null, \ + "period\" varchar(50) not null, "created" timestamp null, "user" jsonb null, \ + "value" double precision null, primary key ("id"));"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testCreateTableB() { + Table table = getTableB(); + + String expected = + """ + create unlogged table "vaccination" ("id" integer not null, \ + "facility_type" varchar(255) null collate "C", "bcg_doses" double precision null, \ + check("id">0), check("bcg_doses">0));"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testCreateTableC() { + Table table = getTableC(); + + String expected = + """ + create table "nutrition" ("id" bigint not null, "vitamin_a" bigint null, \ + "vitamin_d" bigint null, primary key ("id")) inherits ("vaccination");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + + @Test + void testAnalyzeTable() { + Table table = getTableA(); + + String expected = "analyze \"immunization\";"; + + assertEquals(expected, sqlBuilder.analyzeTable(table)); + } + + @Test + void testVacuumTable() { + Table table = getTableA(); + + String expected = "vacuum \"immunization\";"; + + assertEquals(expected, sqlBuilder.vacuumTable(table)); + } + + @Test + void testRenameTable() { + Table table = getTableA(); + + String expected = "alter table \"immunization\" rename to \"vaccination\";"; + + assertEquals(expected, sqlBuilder.renameTable(table, "vaccination")); + } + + @Test + void testDropTableIfExists() { + Table table = getTableA(); + + String expected = "drop table if exists \"immunization\";"; + + assertEquals(expected, sqlBuilder.dropTableIfExists(table)); + } + + @Test + void testDropTableIfExistsString() { + String expected = "drop table if exists \"immunization\";"; + + assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); + } + + @Test + void testDropTableIfExistsCascade() { + Table table = getTableA(); + + String expected = "drop table if exists \"immunization\" cascade;"; + + assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); + } + + @Test + void testDropTableIfExistsCascadeString() { + String expected = "drop table if exists \"immunization\" cascade;"; + + assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); + } + + @Test + void testSwapTable() { + String expected = + """ + drop table if exists "vaccination" cascade; \ + alter table "immunization" rename to "vaccination";"""; + + assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); + } + + @Test + void testSetParent() { + String expected = "alter table \"immunization\" inherit \"vaccination\";"; + + assertEquals(expected, sqlBuilder.setParentTable(getTableA(), "vaccination")); + } + + @Test + void testRemoveParent() { + String expected = "alter table \"immunization\" no inherit \"vaccination\";"; + + assertEquals(expected, sqlBuilder.removeParentTable(getTableA(), "vaccination")); + } + + @Test + void testSwapParentTable() { + String expected = + """ + alter table "immunization" no inherit "vaccination"; \ + alter table "immunization" inherit \"nutrition\";"""; + + assertEquals(expected, sqlBuilder.swapParentTable(getTableA(), "vaccination", "nutrition")); + } + + @Test + void testTableExists() { + String expected = + """ + select t.table_name from information_schema.tables t \ + where t.table_schema = 'public' and t.table_name = 'immunization';"""; + + assertEquals(expected, sqlBuilder.tableExists("immunization")); + } + + @Test + void testCountRows() { + String expected = + """ + select count(*) as row_count from "immunization";"""; + + assertEquals(expected, sqlBuilder.countRows(getTableA())); + } + + @Test + void testCreateIndexA() { + List indexes = getIndexesA(); + + String expected = + "create index \"in_immunization_data\" on \"immunization\" using btree(\"data\");"; + + assertEquals(expected, sqlBuilder.createIndex(indexes.get(0))); + } + + @Test + void testCreateIndexB() { + List indexes = getIndexesA(); + + String expected = + "create index \"in_immunization_period_created\" on \"immunization\" using btree(\"period\",\"created\");"; + + assertEquals(expected, sqlBuilder.createIndex(indexes.get(1))); + } + + @Test + void testCreateIndexC() { + List indexes = getIndexesA(); + + String expected = + "create index \"in_immunization_user\" on \"immunization\" using gin(\"user\");"; + + assertEquals(expected, sqlBuilder.createIndex(indexes.get(2))); + } + + @Test + void testCreateIndexD() { + List indexes = getIndexesA(); + + String expected = + "create index \"in_immunization_data_period\" on \"immunization\" using btree(lower(\"data\"),lower(\"period\"));"; + + assertEquals(expected, sqlBuilder.createIndex(indexes.get(3))); + } + + @Test + void testCreateIndexWithDescNullsLast() { + // given + String expected = + "create unique index \"index_a\" on \"table_a\" using btree(\"column_a\" desc nulls last);"; + Index index = + Index.builder() + .name("index_a") + .tableName("table_a") + .unique(Unique.UNIQUE) + .columns(List.of("column_a")) + .sortOrder("desc nulls last") + .build(); + + // when + String createIndexStmt = sqlBuilder.createIndex(index); + + // then + assertEquals(expected, createIndexStmt); + } +} diff --git a/dhis-2/dhis-support/pom.xml b/dhis-2/dhis-support/pom.xml index f6d2acdd1e9..5366126ee28 100644 --- a/dhis-2/dhis-support/pom.xml +++ b/dhis-2/dhis-support/pom.xml @@ -21,6 +21,7 @@ dhis-support-audit dhis-support-system dhis-support-jdbc + dhis-support-sql dhis-support-expression-parser dhis-support-artemis dhis-support-cache-invalidation diff --git a/dhis-2/dhis-test-integration/pom.xml b/dhis-2/dhis-test-integration/pom.xml index 727a012dfcc..b3bb826d975 100644 --- a/dhis-2/dhis-test-integration/pom.xml +++ b/dhis-2/dhis-test-integration/pom.xml @@ -123,6 +123,7 @@ dhis-support-jdbc compile + org.hisp.dhis dhis-support-system @@ -140,6 +141,11 @@ dhis-support-cache-invalidation test + + org.hisp.dhis + dhis-support-sql + test + org.hisp.dhis.rules rule-engine-jvm diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index 72a481453a6..2e71a8e9551 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -348,6 +348,11 @@ dhis-support-jdbc ${project.version} + + org.hisp.dhis + dhis-support-sql + ${project.version} + org.hisp.dhis dhis-support-hibernate From e47d0bdf3ed4027c5b216892dcee3dc3e230a206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Helge=20=C3=98verland?= Date: Mon, 2 Dec 2024 12:39:23 +0100 Subject: [PATCH 22/24] refactor: Centralize analytics attribute column code [DHIS2-16705] (#19361) --- .../table/AbstractEventJdbcTableManager.java | 129 ++++- .../JdbcEnrollmentAnalyticsTableManager.java | 68 +-- .../table/JdbcEventAnalyticsTableManager.java | 90 ---- ...dbcTrackedEntityAnalyticsTableManager.java | 2 - .../AbstractEventJdbcTableManagerTest.java | 35 ++ .../org/hisp/dhis/db/model/ColumnTest.java | 56 --- .../org/hisp/dhis/db/model/DataTypeTest.java | 53 -- .../org/hisp/dhis/db/model/IndexTest.java | 76 --- .../org/hisp/dhis/db/model/TableTest.java | 145 ------ .../dhis/db/sql/ClickHouseSqlBuilderTest.java | 381 --------------- .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 367 -------------- .../dhis/db/sql/PostgreSqlBuilderTest.java | 452 ------------------ 12 files changed, 155 insertions(+), 1699 deletions(-) delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java delete mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 0a60a2d5042..5807a308ae4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -27,13 +27,20 @@ */ package org.hisp.dhis.analytics.table; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; +import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; +import static org.hisp.dhis.db.model.DataType.GEOMETRY; +import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.partition.PartitionManager; +import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; import org.hisp.dhis.analytics.table.model.Skip; @@ -43,6 +50,8 @@ import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.util.TextUtils; import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.IndexType; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodDataProvider; @@ -86,6 +95,8 @@ public AbstractEventJdbcTableManager( sqlBuilder); } + public static final String OU_GEOMETRY_COL_SUFFIX = "_geom"; + public static final String OU_NAME_COL_SUFFIX = "_name"; protected final String getNumericClause() { @@ -148,17 +159,16 @@ private String getSelectExpression(ValueType valueType, String columnExpression, } else if (valueType.isInteger()) { return getCastExpression(columnExpression, NUMERIC_REGEXP, sqlBuilder.dataTypeBigInt()); } else if (valueType.isBoolean()) { - return "case when " - + columnExpression - + " = 'true' then 1 when " - + columnExpression - + " = 'false' then 0 else null end"; + return String.format( + "case when %1$s = 'true' then 1 when %1$s = 'false' then 0 else null end", + columnExpression); } else if (valueType.isDate()) { return getCastExpression(columnExpression, DATE_REGEXP, sqlBuilder.dataTypeTimestamp()); } else if (valueType.isGeo() && isSpatialSupport()) { - return "ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (" - + columnExpression - + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; + return String.format( + """ + ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (%s) || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')""", + columnExpression); } else if (valueType.isOrganisationUnit()) { String ouClause = isTea @@ -170,6 +180,22 @@ private String getSelectExpression(ValueType valueType, String columnExpression, } } + /** + * For numeric and date value types, returns a data filter clause for checking whether the value + * is valid according to the value type. For other value types, returns the empty string. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @return a data filter clause. + */ + protected String getDataFilterClause(TrackedEntityAttribute attribute) { + if (attribute.isNumericType()) { + return getNumericClause(); + } else if (attribute.isDateType()) { + return getDateClause(); + } + return EMPTY; + } + /** * Returns a cast expression which includes a value filter for the given value type. * @@ -178,7 +204,7 @@ private String getSelectExpression(ValueType valueType, String columnExpression, * @param dataType the SQL data type. * @return a cast and validate expression. */ - String getCastExpression(String columnExpression, String filterRegex, String dataType) { + protected String getCastExpression(String columnExpression, String filterRegex, String dataType) { String filter = sqlBuilder.regexpMatch(columnExpression, filterRegex); return String.format( "case when %s then cast(%s as %s) else null end", filter, columnExpression, dataType); @@ -219,27 +245,100 @@ protected void populateTableInternal(AnalyticsTablePartition partition, String f invokeTimeAndLog(sql, "Populating table: '{}'", tableName); } + /** + * Returns a list of columns based on the given attribute. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @return a list of {@link AnaylyticsTableColumn}. + */ + protected List getColumnForAttribute(TrackedEntityAttribute attribute) { + List columns = new ArrayList<>(); + + DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); + String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); + String dataFilterClause = getDataFilterClause(attribute); + String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); + + if (attribute.getValueType().isOrganisationUnit()) { + columns.addAll(getColumnForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); + } + + columns.add( + AnalyticsTableColumn.builder() + .name(attribute.getUid()) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(dataType) + .selectExpression(sql) + .skipIndex(skipIndex) + .build()); + + return columns; + } + + /** + * Returns a list of columns based on the given attribute. + * + * @param attribute the {@link TrackedEntityAttribute}. + * @param dataFilterClause the data filter clause. + * @return a list of {@link AnalyticsTableColumn}. + */ + private List getColumnForOrgUnitTrackedEntityAttribute( + TrackedEntityAttribute attribute, String dataFilterClause) { + List columns = new ArrayList<>(); + + String fromClause = + qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); + + if (isSpatialSupport()) { + String selectExpression = "ou.geometry " + fromClause; + String ouGeoSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + columns.add( + AnalyticsTableColumn.builder() + .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(GEOMETRY) + .selectExpression(ouGeoSql) + .indexType(IndexType.GIST) + .build()); + } + + String selectExpression = "ou.name " + fromClause; + String ouNameSql = getSelectSubquery(attribute, selectExpression, dataFilterClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) + .dimensionType(AnalyticsDimensionType.DYNAMIC) + .dataType(TEXT) + .selectExpression(ouNameSql) + .skipIndex(SKIP) + .build()); + + return columns; + } + /** * The select subquery statement. * * @param attribute the {@link TrackedEntityAttribute}. - * @param columnExpression the column expression. + * @param selectExpression the select expression. * @param dataFilterClause the data filter clause. * @return a select statement. */ - protected String getSelectSubquery( - TrackedEntityAttribute attribute, String columnExpression, String dataFilterClause) { + private String getSelectSubquery( + TrackedEntityAttribute attribute, String selectExpression, String dataFilterClause) { return replaceQualify( """ - (select ${columnExpression} from ${trackedentityattributevalue} \ + (select ${selectExpression} from ${trackedentityattributevalue} \ where trackedentityid=en.trackedentityid \ and trackedentityattributeid=${attributeId}${dataFilterClause})\ ${closingParentheses} as ${attributeUid}""", Map.of( - "columnExpression", columnExpression, + "selectExpression", selectExpression, "dataFilterClause", dataFilterClause, "attributeId", String.valueOf(attribute.getId()), - "closingParentheses", getClosingParentheses(columnExpression), + "closingParentheses", getClosingParentheses(selectExpression), "attributeUid", quote(attribute.getUid()))); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index f78f2aad73f..009b19645af 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -27,13 +27,10 @@ */ package org.hisp.dhis.analytics.table; -import static org.hisp.dhis.analytics.table.model.Skip.SKIP; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; -import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.util.DateUtils.toLongDate; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Map; @@ -41,17 +38,14 @@ import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; -import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; -import org.hisp.dhis.analytics.table.model.Skip; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.commons.collection.UniqueArrayList; import org.hisp.dhis.dataapproval.DataApprovalLevelService; -import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; @@ -60,7 +54,6 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.system.database.DatabaseInfoProvider; -import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -150,7 +143,7 @@ public void populateTable(AnalyticsTableUpdateParams params, AnalyticsTableParti String fromClause = replaceQualify( """ - \s from ${enrollment} en \ + \sfrom ${enrollment} en \ inner join ${program} pr on en.programid=pr.programid \ left join ${trackedentity} te on en.trackedentityid=te.trackedentityid and te.deleted = false \ left join ${organisationunit} registrationou on te.organisationunitid=registrationou.organisationunitid \ @@ -197,59 +190,10 @@ private List getColumns(Program program) { * @return a list of {@link AnalyticsTableColumn}. */ private List getTrackedEntityAttributeColumns(Program program) { - List columns = new ArrayList<>(); - - for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { - DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String dataClause = - attribute.isNumericType() - ? getNumericClause() - : attribute.isDateType() ? getDateClause() : ""; - String select = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); - - String sql = - replaceQualify( - """ - (select ${select} from ${trackedentityattributevalue} \ - where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}\ - ${dataClause})${closingParentheses} as ${attributeUid}""", - Map.of( - "select", - select, - "attributeId", - String.valueOf(attribute.getId()), - "dataClause", - dataClause, - "closingParentheses", - getClosingParentheses(select), - "attributeUid", - quote(attribute.getUid()))); - columns.add( - AnalyticsTableColumn.builder() - .name(attribute.getUid()) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(dataType) - .selectExpression(sql) - .skipIndex(skipIndex) - .build()); - - if (attribute.getValueType().isOrganisationUnit()) { - String fromTypeSql = "ou.name from organisationunit ou where ou.uid = (select value"; - String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - } - } - return columns; + return program.getNonConfidentialTrackedEntityAttributes().stream() + .map(this::getColumnForAttribute) + .flatMap(Collection::stream) + .toList(); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index aa63a4d5ed9..8c652952993 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -97,8 +97,6 @@ @Service("org.hisp.dhis.analytics.EventAnalyticsTableManager") public class JdbcEventAnalyticsTableManager extends AbstractEventJdbcTableManager { - public static final String OU_GEOMETRY_COL_SUFFIX = "_geom"; - static final String[] EXPORTABLE_EVENT_STATUSES = {"'COMPLETED'", "'ACTIVE'", "'SCHEDULE'"}; protected final List fixedColumns; @@ -583,38 +581,6 @@ private List getAttributeColumns(Program program) { return columns; } - /** - * Returns a list of columns based on the given attribute. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @param withLegendSet indicates whether the attribute has a legend set. - * @return a list of {@link AnaylyticsTableColumn}. - */ - private List getColumnForAttribute(TrackedEntityAttribute attribute) { - List columns = new ArrayList<>(); - - DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); - String dataFilterClause = getDataFilterClause(attribute); - String sql = getSelectSubquery(attribute, selectExpression, dataFilterClause); - Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); - - if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnsForOrgUnitTrackedEntityAttribute(attribute, dataFilterClause)); - } - - columns.add( - AnalyticsTableColumn.builder() - .name(attribute.getUid()) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(dataType) - .selectExpression(sql) - .skipIndex(skipIndex) - .build()); - - return columns; - } - /** * Returns a list of columns based on the given attribute with legend set. * @@ -657,48 +623,6 @@ private List getColumnForAttributeWithLegendSet( .toList(); } - /** - * Returns a list of columns based on the given attribute. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @param dataFilterClause the data filter clause. - * @return a list of {@link AnalyticsTableColumn}. - */ - private List getColumnsForOrgUnitTrackedEntityAttribute( - TrackedEntityAttribute attribute, String dataFilterClause) { - List columns = new ArrayList<>(); - - String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); - - if (isSpatialSupport()) { - String fromType = "ou.geometry " + fromClause; - String geoSql = getSelectSubquery(attribute, fromType, dataFilterClause); - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(GEOMETRY) - .selectExpression(geoSql) - .indexType(IndexType.GIST) - .build()); - } - - String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = getSelectSubquery(attribute, fromTypeSql, dataFilterClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((attribute.getUid() + OU_NAME_COL_SUFFIX)) - .dimensionType(AnalyticsDimensionType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - - return columns; - } - /** * Retyrns a select statement for the given select expression. * @@ -788,20 +712,6 @@ private String getDataFilterClause(DataElement dataElement) { return EMPTY; } - /** - * For numeric and date value types, returns a data filter clause for checking whether the value - * is valid according to the value type. For other value types, returns the empty string. - * - * @param attribute the {@link TrackedEntityAttribute}. - * @return a data filter clause. - */ - private String getDataFilterClause(TrackedEntityAttribute attribute) { - if (attribute.isNumericType()) { - return getNumericClause(); - } - return attribute.isDateType() ? getDateClause() : EMPTY; - } - /** * Returns a list of years for which data exist. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index 4071615a7e8..333602b5adf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -247,8 +247,6 @@ private Stream getAllTrackedEntityAttributes( /** * Returns the select clause, potentially with a cast statement, based on the given value type. - * (this method is an adapted version of {@link - * JdbcEventAnalyticsTableManager#getSelectExpression(ValueType, String)}) * * @param valueType the value type to represent as database column type. */ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java index 80cf1dbee0b..68decf6d520 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManagerTest.java @@ -28,10 +28,12 @@ package org.hisp.dhis.analytics.table; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -45,6 +47,11 @@ class AbstractEventJdbcTableManagerTest { @InjectMocks private JdbcEventAnalyticsTableManager manager; + @BeforeEach + public void beforeEach() { + manager.spatialSupport = true; + } + @Test void testGetCastExpression() { String expected = @@ -76,6 +83,19 @@ then cast(eventdatavalues #>> '{GieVkTxp4HH, value}' as double precision) \ assertEquals(expected, actual); } + @Test + void testGetSelectExpressionBoolean() { + String expected = + """ + case when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'true' then 1 when eventdatavalues #>> '{Xl3voRRcmpo, value}' = 'false' then 0 else null end"""; + + String actual = + manager.getSelectExpression( + ValueType.BOOLEAN, "eventdatavalues #>> '{Xl3voRRcmpo, value}'"); + + assertEquals(expected, actual); + } + @Test void testGetSelectExpressionDate() { String expected = @@ -101,4 +121,19 @@ void testGetSelectExpressionText() { assertEquals(expected, actual); } + + @Test + void testGetSelectExpressionGeometry() { + when(manager.isSpatialSupport()).thenReturn(Boolean.TRUE); + + String expected = + """ + ST_GeomFromGeoJSON('{"type":"Point", "coordinates":' || (eventdatavalues #>> '{C6bh7GevJfH, value}') || ', "crs":{"type":"name", "properties":{"name":"EPSG:4326"}}}')"""; + + String actual = + manager.getSelectExpression( + ValueType.GEOJSON, "eventdatavalues #>> '{C6bh7GevJfH, value}'"); + + assertEquals(expected, actual); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java deleted file mode 100644 index e1d2f48443c..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/ColumnTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class ColumnTest { - @Test - void testIsNotNull() { - Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL); - Column colB = new Column("value", DataType.DOUBLE, Nullable.NULL); - - assertTrue(colA.isNotNull()); - assertFalse(colB.isNotNull()); - } - - @Test - void testHasCollation() { - Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL, Collation.DEFAULT); - Column colB = new Column("ou", DataType.CHARACTER_11, Nullable.NOT_NULL, Collation.C); - Column colC = new Column("value", DataType.DOUBLE, Nullable.NULL); - - assertFalse(colA.hasCollation()); - assertTrue(colB.hasCollation()); - assertFalse(colC.hasCollation()); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java deleted file mode 100644 index 1c75b3c3cdd..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/DataTypeTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.jupiter.api.Test; - -class DataTypeTest { - @Test - void testIsNumeric() { - assertTrue(DataType.BIGINT.isNumeric()); - assertFalse(DataType.CHARACTER_11.isNumeric()); - } - - @Test - void testIsBoolean() { - assertTrue(DataType.BOOLEAN.isBoolean()); - assertFalse(DataType.DOUBLE.isBoolean()); - } - - @Test - void testIsCharacter() { - assertTrue(DataType.VARCHAR_255.isCharacter()); - assertFalse(DataType.DECIMAL.isCharacter()); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java deleted file mode 100644 index c7d46a847a7..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/IndexTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.hisp.dhis.db.model.constraint.Unique; -import org.junit.jupiter.api.Test; - -class IndexTest { - @Test - void testIsUnique() { - Index indexA = - Index.builder() - .name("in_analytics_id") - .tableName("analytics") - .unique(Unique.UNIQUE) - .columns(List.of("id")) - .build(); - - Index indexB = - Index.builder() - .name("in_analytics_dx") - .tableName("analytics") - .columns(List.of("dx")) - .build(); - - assertTrue(indexA.isUnique()); - assertFalse(indexB.isUnique()); - } - - @Test - void testDefaults() { - Index.IndexBuilder builder = Index.builder(); - - Index index = builder.build(); - - assertNull(index.getName()); - assertNull(index.getTableName()); - assertNull(index.getCondition()); - assertNull(index.getFunction()); - assertNull(index.getColumns()); - assertNull(index.getSortOrder()); - assertSame(IndexType.BTREE, index.getIndexType()); - assertFalse(index.isUnique()); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java deleted file mode 100644 index bc57ee6db4c..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/model/TableTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.model; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class TableTest { - private final Column colA = new Column("dx", DataType.CHARACTER_11, Nullable.NOT_NULL); - private final Column colB = new Column("value", DataType.DOUBLE, Nullable.NULL); - - @Test - void testToStagingTable() { - assertEquals( - "analytics_rs_categorystructure_temp", Table.toStaging("analytics_rs_categorystructure")); - assertEquals("analytics_temp", Table.toStaging("analytics")); - } - - @Test - void testFromStagingTable() { - assertEquals( - "analytics_rs_categorystructure", Table.fromStaging("analytics_rs_categorystructure_temp")); - assertEquals("analytics", Table.fromStaging("analytics_temp")); - } - - @Test - void testIsUnlogged() { - List columns = List.of(colA, colB); - - Table tableA = new Table("analytics", columns, List.of(), Logged.UNLOGGED); - Table tableB = new Table("analytics", columns, List.of(), Logged.LOGGED); - - assertTrue(tableA.isUnlogged()); - assertFalse(tableB.isUnlogged()); - } - - @Test - void testHasColumns() { - Table table = new Table("analytics", List.of(colA, colB), List.of()); - - assertTrue(table.hasColumns()); - } - - @Test - void getFirstColumn() { - Table table = new Table("analytics", List.of(colA, colB), List.of()); - - assertEquals(colA, table.getFirstColumn()); - } - - @Test - void testHasPrimaryKey() { - Table tableA = new Table("analytics", List.of(colA, colB), List.of("dx")); - Table tableB = new Table("analytics", List.of(colA, colB), List.of()); - - assertTrue(tableA.hasPrimaryKey()); - assertFalse(tableB.hasPrimaryKey()); - assertEquals(List.of("dx"), tableA.getPrimaryKey()); - } - - @Test - void testGetFirstPrimaryKey() { - Table table = new Table("analytics", List.of(colA, colB), List.of("dx", "value")); - - assertEquals("dx", table.getFirstPrimaryKey()); - } - - @Test - void testHasSortKey() { - Table tableA = - new Table( - "analytics", - List.of(colA, colB), - List.of("dx", "value"), - List.of("dx"), - List.of(), - Logged.UNLOGGED); - Table tableB = new Table("analytics", List.of(colA, colB), List.of("dx", "value")); - - assertTrue(tableA.hasSortKey()); - assertFalse(tableB.hasSortKey()); - assertEquals(List.of("dx"), tableA.getSortKey()); - } - - @Test - void testSuccessfulValidation() { - List columns = List.of(colA); - List primaryKey = List.of(); - - assertDoesNotThrow(() -> new Table("analytics", columns, primaryKey)); - } - - @Test - void testNameValidation() { - List columns = List.of(colA); - List primaryKey = List.of(); - - assertThrows(NullPointerException.class, () -> new Table(null, columns, primaryKey)); - assertThrows(IllegalArgumentException.class, () -> new Table("", columns, primaryKey)); - } - - @Test - void testColumnsParentValidation() { - List columns = List.of(); - List primaryKey = List.of(); - List checks = List.of(); - - assertThrows(IllegalArgumentException.class, () -> new Table("analytics", columns, primaryKey)); - assertThrows( - IllegalArgumentException.class, - () -> new Table("analytics", columns, primaryKey, checks, Logged.UNLOGGED, null)); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java deleted file mode 100644 index 5087172105e..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilderTest.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.sql; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import java.util.Map; -import org.hisp.dhis.db.model.Collation; -import org.hisp.dhis.db.model.Column; -import org.hisp.dhis.db.model.DataType; -import org.hisp.dhis.db.model.Logged; -import org.hisp.dhis.db.model.Table; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class ClickHouseSqlBuilderTest { - private final ClickHouseSqlBuilder sqlBuilder = new ClickHouseSqlBuilder(); - - private Table getTableA() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("created", DataType.TIMESTAMP), - new Column("user", DataType.JSONB), - new Column("value", DataType.DOUBLE)); - - List primaryKey = List.of("id"); - - return new Table("immunization", columns, primaryKey, Logged.LOGGED); - } - - private Table getTableB() { - List columns = - List.of( - new Column("id", DataType.INTEGER, Nullable.NOT_NULL), - new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), - new Column("bcg_doses", DataType.DOUBLE)); - - return new Table("vaccination", columns, List.of()); - } - - private Table getTableC() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("vitamin_a", DataType.BIGINT), - new Column("vitamin_d", DataType.BIGINT)); - - List primaryKey = List.of("id"); - - return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); - } - - private Table getTableD() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("value", DataType.DOUBLE)); - - List sortKey = List.of("data", "period"); - - return new Table("immunization", columns, List.of(), sortKey, List.of(), Logged.LOGGED); - } - - // Data types - - @Test - void testDataTypes() { - assertEquals("Float64", sqlBuilder.dataTypeDouble()); - assertEquals("DateTime64(3)", sqlBuilder.dataTypeTimestamp()); - } - - // Index types - - @Test - void testIndexTypes() { - assertThrows(UnsupportedOperationException.class, () -> sqlBuilder.indexTypeBtree()); - } - - // Capabilities - - @Test - void testSupportsAnalyze() { - assertFalse(sqlBuilder.supportsAnalyze()); - } - - @Test - void testSupportsVacuum() { - assertFalse(sqlBuilder.supportsVacuum()); - } - - // Utilities - - @Test - void testQuote() { - assertEquals( - "\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("Treated \"malaria\" at facility")); - assertEquals( - "\"\"\"Patients on \"\"treatment\"\" for TB\"\"\"", - sqlBuilder.quote("\"Patients on \"treatment\" for TB\"")); - assertEquals("\"quarterly\"", sqlBuilder.quote("quarterly")); - assertEquals("\"Fully immunized\"", sqlBuilder.quote("Fully immunized")); - } - - @Test - void testQuoteAlias() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); - assertEquals( - "analytics.\"Patients on \"\"treatment\"\" for TB\"", - sqlBuilder.quote("analytics", "Patients on \"treatment\" for TB")); - assertEquals("analytics.\"quarterly\"", sqlBuilder.quote("analytics", "quarterly")); - assertEquals("dv.\"Fully immunized\"", sqlBuilder.quote("dv", "Fully immunized")); - } - - @Test - void testQuoteAx() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quoteAx("Treated \"malaria\" at facility")); - assertEquals("ax.\"quarterly\"", sqlBuilder.quoteAx("quarterly")); - assertEquals("ax.\"Fully immunized\"", sqlBuilder.quoteAx("Fully immunized")); - } - - @Test - void testSingleQuote() { - assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); - assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); - assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); - } - - @Test - void testEscape() { - assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); - assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); - } - - @Test - void testSinqleQuotedCommaDelimited() { - assertEquals( - "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", - sqlBuilder.singleQuotedCommaDelimited( - List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); - assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); - } - - @Test - void testQualifyTable() { - assertEquals("postgresql(\"pg_dhis\", table='category')", sqlBuilder.qualifyTable("category")); - assertEquals( - "postgresql(\"pg_dhis\", table='categories_options')", - sqlBuilder.qualifyTable("categories_options")); - } - - @Test - void testDateTrunc() { - assertEquals( - "date_trunc('month', pe.startdate)", sqlBuilder.dateTrunc("month", "pe.startdate")); - } - - @Test - void testDifferenceInSeconds() { - assertEquals( - "(toUnixTimestamp(a.startdate) - toUnixTimestamp(b.enddate))", - sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); - assertEquals( - "(toUnixTimestamp(a.\"startdate\") - toUnixTimestamp(b.\"enddate\"))", - sqlBuilder.differenceInSeconds( - sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); - } - - @Test - void testRegexpMatch() { - assertEquals("match(value, 'test')", sqlBuilder.regexpMatch("value", "'test'")); - assertEquals("match(number, '\\d')", sqlBuilder.regexpMatch("number", "'\\d'")); - assertEquals("match(color, '^Blue$')", sqlBuilder.regexpMatch("color", "'^Blue$'")); - assertEquals("match(id, '[a-z]\\w+\\d{3}')", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); - } - - @Test - void testJsonExtract() { - assertEquals( - "JSONExtractString(value, 'D7m8vpzxHDJ')", sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); - } - - @Test - void testJsonExtractNested() { - assertEquals( - "JSONExtractString(eventdatavalues, 'D7m8vpzxHDJ.value')", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); - } - - // Statements - - @Test - void testCreateTableA() { - Table table = getTableA(); - - String expected = - """ - create table "immunization" ("id" Int64 not null,"data" String not null,\ - "period" String not null,"created" DateTime64(3) null,"user" JSON null,\ - "value" Float64 null) \ - engine = MergeTree() \ - order by ("id");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableB() { - Table table = getTableB(); - - String expected = - """ - create table "vaccination" ("id" Int32 not null,\ - "facility_type" String null,"bcg_doses" Float64 null) \ - engine = MergeTree() \ - order by ("id");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableC() { - Table table = getTableC(); - - String expected = - """ - create table "nutrition" ("id" Int64 not null,"vitamin_a" Int64 null,\ - "vitamin_d" Int64 null) \ - engine = MergeTree() order by ("id");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableD() { - Table table = getTableD(); - - String expected = - """ - create table "immunization" ("id" Int64 not null,"data" String not null,\ - "period" String not null,"value" Float64 null) \ - engine = MergeTree() \ - order by ("data","period");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testRenameTable() { - Table table = getTableA(); - - String expected = "rename table \"immunization\" to \"vaccination\";"; - - assertEquals(expected, sqlBuilder.renameTable(table, "vaccination")); - } - - @Test - void testDropTableIfExists() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists(table)); - } - - @Test - void testDropTableIfExistsString() { - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); - } - - @Test - void testDropTableIfExistsCascade() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); - } - - @Test - void testDropTableIfExistsCascadeString() { - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); - } - - @Test - void testSwapTable() { - String expected = - """ - drop table if exists "vaccination"; \ - rename table "immunization" to "vaccination";"""; - - assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); - } - - @Test - void testTableExists() { - String expected = - """ - select t.name as table_name \ - from system.tables t \ - where t.database = 'default' \ - and t.name = 'immunization' \ - and engine not in ('View', 'Materialized View');"""; - - assertEquals(expected, sqlBuilder.tableExists("immunization")); - } - - @Test - void testCountRows() { - String expected = - """ - select count(*) as row_count from \"immunization\";"""; - - assertEquals(expected, sqlBuilder.countRows(getTableA())); - } - - // Named collection - - @Test - void testCreateNamedCollection() { - String expected = - """ - create named collection "pg_dhis" as """; - - assertTrue( - sqlBuilder - .createNamedCollection("pg_dhis", Map.of("host", "mydomain.org")) - .startsWith(expected)); - } - - @Test - void testDropNamedCollectionIfExists() { - assertEquals( - "drop named collection if exists \"pg_dhis\";", - sqlBuilder.dropNamedCollectionIfExists("pg_dhis")); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java deleted file mode 100644 index ad372141c9b..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ /dev/null @@ -1,367 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.sql; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.List; -import org.hisp.dhis.db.model.Collation; -import org.hisp.dhis.db.model.Column; -import org.hisp.dhis.db.model.DataType; -import org.hisp.dhis.db.model.Logged; -import org.hisp.dhis.db.model.Table; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.junit.jupiter.api.Test; - -class DorisSqlBuilderTest { - private final SqlBuilder sqlBuilder = new DorisSqlBuilder("pg_dhis", "postgresql.jar"); - - private Table getTableA() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("created", DataType.TIMESTAMP), - new Column("user", DataType.JSONB), - new Column("value", DataType.DOUBLE)); - - List primaryKey = List.of("id"); - - return new Table("immunization", columns, primaryKey, Logged.LOGGED); - } - - private Table getTableB() { - List columns = - List.of( - new Column("id", DataType.INTEGER, Nullable.NOT_NULL), - new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), - new Column("bcg_doses", DataType.DOUBLE)); - - return new Table("vaccination", columns, List.of(), Logged.UNLOGGED); - } - - private Table getTableC() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("vitamin_a", DataType.BIGINT), - new Column("vitamin_d", DataType.BIGINT)); - - List primaryKey = List.of("id"); - - return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); - } - - // Data types - - @Test - void testDataTypes() { - assertEquals("double", sqlBuilder.dataTypeDouble()); - assertEquals("datetime", sqlBuilder.dataTypeTimestamp()); - } - - // Index types - - @Test - void testIndexTypes() { - assertThrows(UnsupportedOperationException.class, () -> sqlBuilder.indexTypeBtree()); - } - - // Capabilities - - @Test - void testSupportsAnalyze() { - assertFalse(sqlBuilder.supportsAnalyze()); - } - - @Test - void testSupportsVacuum() { - assertFalse(sqlBuilder.supportsVacuum()); - } - - // Utilities - - @Test - void testQuote() { - assertEquals( - "`Treated \"malaria\" at facility`", sqlBuilder.quote("Treated \"malaria\" at facility")); - assertEquals( - "`Patients on ``treatment`` for TB`", sqlBuilder.quote("Patients on `treatment` for TB")); - assertEquals("`quarterly`", sqlBuilder.quote("quarterly")); - assertEquals("`Fully immunized`", sqlBuilder.quote("Fully immunized")); - } - - @Test - void testQuoteAlias() { - assertEquals( - "ax.`Treated \"malaria\" at facility`", - sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); - assertEquals( - "analytics.`Patients on ``treatment`` for TB`", - sqlBuilder.quote("analytics", "Patients on `treatment` for TB")); - assertEquals("analytics.`quarterly`", sqlBuilder.quote("analytics", "quarterly")); - assertEquals("dv.`Fully immunized`", sqlBuilder.quote("dv", "Fully immunized")); - } - - @Test - void testQuoteAx() { - assertEquals( - "ax.`Treated ``malaria`` at facility`", - sqlBuilder.quoteAx("Treated `malaria` at facility")); - assertEquals("ax.`quarterly`", sqlBuilder.quoteAx("quarterly")); - assertEquals("ax.`Fully immunized`", sqlBuilder.quoteAx("Fully immunized")); - } - - @Test - void testSingleQuote() { - assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); - assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); - assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); - } - - @Test - void testEscape() { - assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); - assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); - assertEquals("C:\\\\Downloads\\\\File.doc", sqlBuilder.escape("C:\\Downloads\\File.doc")); - } - - @Test - void testSinqleQuotedCommaDelimited() { - assertEquals( - "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", - sqlBuilder.singleQuotedCommaDelimited( - List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); - assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); - } - - @Test - void testQualifyTable() { - assertEquals("pg_dhis.public.`category`", sqlBuilder.qualifyTable("category")); - assertEquals( - "pg_dhis.public.`categories_options`", sqlBuilder.qualifyTable("categories_options")); - } - - @Test - void testDateTrunc() { - assertEquals( - "date_trunc(pe.startdate, 'month')", sqlBuilder.dateTrunc("month", "pe.startdate")); - } - - @Test - void testDifferenceInSeconds() { - assertEquals( - "(unix_timestamp(a.startdate) - unix_timestamp(b.enddate))", - sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); - assertEquals( - "(unix_timestamp(a.`startdate`) - unix_timestamp(b.`enddate`))", - sqlBuilder.differenceInSeconds( - sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); - } - - @Test - void testRegexpMatch() { - assertEquals("value regexp 'test'", sqlBuilder.regexpMatch("value", "'test'")); - assertEquals("number regexp '\\d'", sqlBuilder.regexpMatch("number", "'\\d'")); - assertEquals("color regexp '^Blue$'", sqlBuilder.regexpMatch("color", "'^Blue$'")); - assertEquals("id regexp '[a-z]\\w+\\d{3}'", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); - } - - @Test - void testJsonExtract() { - assertEquals( - "json_unquote(json_extract(value, '$.D7m8vpzxHDJ'))", - sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); - } - - @Test - void testJsonExtractNested() { - assertEquals( - "json_unquote(json_extract(eventdatavalues, '$.D7m8vpzxHDJ.value'))", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); - } - - // Statements - - @Test - void testCreateTableA() { - Table table = getTableA(); - - String expected = - """ - create table `immunization` (`id` bigint not null,\ - `data` char(11) not null,`period` varchar(50) not null,\ - `created` datetime null,`user` json null,`value` double null) \ - engine = olap \ - unique key (`id`) \ - distributed by hash(`id`) buckets 10 \ - properties ("replication_num" = "1");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableB() { - Table table = getTableB(); - - String expected = - """ - create table `vaccination` (`id` int not null,\ - `facility_type` varchar(255) null,`bcg_doses` double null) \ - engine = olap \ - duplicate key (`id`) \ - distributed by hash(`id`) buckets 10 \ - properties ("replication_num" = "1");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - // void testCreateTableB() - - @Test - void testCreateTableC() { - Table table = getTableC(); - - String expected = - """ - create table `nutrition` (`id` bigint not null,\ - `vitamin_a` bigint null,`vitamin_d` bigint null) \ - engine = olap \ - unique key (`id`) \ - distributed by hash(`id`) buckets 10 \ - properties ("replication_num" = "1");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateCatalog() { - String expected = - """ - create catalog `pg_dhis` \ - properties ( - "type" = "jdbc", \ - "user" = "dhis", \ - "password" = "kH7g", \ - "jdbc_url" = "jdbc:mysql://127.18.0.1:9030/dev?useUnicode=true&characterEncoding=UTF-8", \ - "driver_url" = "postgresql.jar", \ - "driver_class" = "org.postgresql.Driver" - );"""; - - assertEquals( - expected, - sqlBuilder.createCatalog( - "jdbc:mysql://127.18.0.1:9030/dev?useUnicode=true&characterEncoding=UTF-8", - "dhis", - "kH7g")); - } - - @Test - void testDropCatalogIfExists() { - String expected = "drop catalog if exists `pg_dhis`;"; - - assertEquals(expected, sqlBuilder.dropCatalogIfExists()); - } - - @Test - void testRenameTable() { - Table table = getTableA(); - - String expected = - """ - alter table `immunization` rename `immunization_main`;"""; - - assertEquals(expected, sqlBuilder.renameTable(table, "immunization_main")); - } - - @Test - void testDropTableIfExists() { - Table table = getTableA(); - - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExists(table)); - } - - @Test - void testDropTableIfExistsString() { - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); - } - - @Test - void testDropTableIfExistsCascade() { - Table table = getTableA(); - - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); - } - - @Test - void testDropTableIfExistsCascadeString() { - String expected = "drop table if exists `immunization`;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); - } - - @Test - void testSwapTable() { - String expected = - """ - drop table if exists `vaccination`; \ - alter table `immunization` rename `vaccination`;"""; - - assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); - } - - @Test - void testTableExists() { - String expected = - """ - select t.table_name from information_schema.tables t \ - where t.table_schema = 'public' and t.table_name = 'immunization';"""; - - assertEquals(expected, sqlBuilder.tableExists("immunization")); - } - - @Test - void testCountRows() { - String expected = - """ - select count(*) as row_count from `immunization`;"""; - - assertEquals(expected, sqlBuilder.countRows(getTableA())); - } -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java deleted file mode 100644 index 2d621ae766e..00000000000 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/db/sql/PostgreSqlBuilderTest.java +++ /dev/null @@ -1,452 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.db.sql; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.hisp.dhis.db.model.Collation; -import org.hisp.dhis.db.model.Column; -import org.hisp.dhis.db.model.DataType; -import org.hisp.dhis.db.model.Index; -import org.hisp.dhis.db.model.IndexFunction; -import org.hisp.dhis.db.model.IndexType; -import org.hisp.dhis.db.model.Logged; -import org.hisp.dhis.db.model.Table; -import org.hisp.dhis.db.model.constraint.Nullable; -import org.hisp.dhis.db.model.constraint.Unique; -import org.junit.jupiter.api.Test; - -class PostgreSqlBuilderTest { - private final SqlBuilder sqlBuilder = new PostgreSqlBuilder(); - - private Table getTableA() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), - new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), - new Column("created", DataType.TIMESTAMP), - new Column("user", DataType.JSONB), - new Column("value", DataType.DOUBLE)); - - List primaryKey = List.of("id"); - - return new Table("immunization", columns, primaryKey, Logged.LOGGED); - } - - private List getIndexesA() { - return List.of( - Index.builder() - .name("in_immunization_data") - .tableName("immunization") - .columns(List.of("data")) - .build(), - Index.builder() - .name("in_immunization_period_created") - .tableName("immunization") - .columns(List.of("period", "created")) - .build(), - Index.builder() - .name("in_immunization_user") - .tableName("immunization") - .indexType(IndexType.GIN) - .columns(List.of("user")) - .build(), - Index.builder() - .name("in_immunization_data_period") - .tableName("immunization") - .columns(List.of("data", "period")) - .function(IndexFunction.LOWER) - .build()); - } - - private Table getTableB() { - List columns = - List.of( - new Column("id", DataType.INTEGER, Nullable.NOT_NULL), - new Column("facility_type", DataType.VARCHAR_255, Nullable.NULL, Collation.C), - new Column("bcg_doses", DataType.DOUBLE)); - - List checks = List.of("\"id\">0", "\"bcg_doses\">0"); - - return new Table("vaccination", columns, List.of(), List.of(), checks, Logged.UNLOGGED); - } - - private Table getTableC() { - List columns = - List.of( - new Column("id", DataType.BIGINT, Nullable.NOT_NULL), - new Column("vitamin_a", DataType.BIGINT), - new Column("vitamin_d", DataType.BIGINT)); - - List primaryKey = List.of("id"); - - return new Table("nutrition", columns, primaryKey, List.of(), Logged.LOGGED, getTableB()); - } - - // Data types - - @Test - void testDataTypes() { - assertEquals("double precision", sqlBuilder.dataTypeDouble()); - assertEquals("geometry", sqlBuilder.dataTypeGeometry()); - } - - // Index types - - @Test - void testIndexTypes() { - assertEquals("btree", sqlBuilder.indexTypeBtree()); - assertEquals("gist", sqlBuilder.indexTypeGist()); - assertEquals("gin", sqlBuilder.indexTypeGin()); - } - - // Capabilities - - @Test - void testSupportsAnalyze() { - assertTrue(sqlBuilder.supportsAnalyze()); - } - - @Test - void testSupportsVacuum() { - assertTrue(sqlBuilder.supportsVacuum()); - } - - // Utilities - - @Test - void testQuote() { - assertEquals( - "\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("Treated \"malaria\" at facility")); - assertEquals("\"quarterly\"", sqlBuilder.quote("quarterly")); - assertEquals("\"Fully immunized\"", sqlBuilder.quote("Fully immunized")); - } - - @Test - void testQuoteAlias() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quote("ax", "Treated \"malaria\" at facility")); - assertEquals("analytics.\"quarterly\"", sqlBuilder.quote("analytics", "quarterly")); - assertEquals("dv.\"Fully immunized\"", sqlBuilder.quote("dv", "Fully immunized")); - } - - @Test - void testQuoteAx() { - assertEquals( - "ax.\"Treated \"\"malaria\"\" at facility\"", - sqlBuilder.quoteAx("Treated \"malaria\" at facility")); - assertEquals("ax.\"quarterly\"", sqlBuilder.quoteAx("quarterly")); - assertEquals("ax.\"Fully immunized\"", sqlBuilder.quoteAx("Fully immunized")); - } - - @Test - void testSingleQuote() { - assertEquals("'jkhYg65ThbF'", sqlBuilder.singleQuote("jkhYg65ThbF")); - assertEquals("'Age ''<5'' years'", sqlBuilder.singleQuote("Age '<5' years")); - assertEquals("'Status \"not checked\"'", sqlBuilder.singleQuote("Status \"not checked\"")); - } - - @Test - void testEscape() { - assertEquals("Age group ''under 5'' years", sqlBuilder.escape("Age group 'under 5' years")); - assertEquals("Level ''high'' found", sqlBuilder.escape("Level 'high' found")); - assertEquals("C:\\\\Downloads\\\\File.doc", sqlBuilder.escape("C:\\Downloads\\File.doc")); - } - - @Test - void testSinqleQuotedCommaDelimited() { - assertEquals( - "'dmPbDBKwXyF', 'zMl4kciwJtz', 'q1Nqu1r1GTn'", - sqlBuilder.singleQuotedCommaDelimited( - List.of("dmPbDBKwXyF", "zMl4kciwJtz", "q1Nqu1r1GTn"))); - assertEquals("'1', '3', '5'", sqlBuilder.singleQuotedCommaDelimited(List.of("1", "3", "5"))); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(List.of())); - assertEquals("", sqlBuilder.singleQuotedCommaDelimited(null)); - } - - @Test - void testQualifyTable() { - assertEquals("\"category\"", sqlBuilder.qualifyTable("category")); - assertEquals("\"categories_options\"", sqlBuilder.qualifyTable("categories_options")); - } - - @Test - void testDateTrunc() { - assertEquals( - "date_trunc('month', pe.startdate)", sqlBuilder.dateTrunc("month", "pe.startdate")); - } - - @Test - void testDifferenceInSeconds() { - assertEquals( - "extract(epoch from (a.startdate - b.enddate))", - sqlBuilder.differenceInSeconds("a.startdate", "b.enddate")); - assertEquals( - "extract(epoch from (a.\"startdate\" - b.\"enddate\"))", - sqlBuilder.differenceInSeconds( - sqlBuilder.quote("a", "startdate"), sqlBuilder.quote("b", "enddate"))); - } - - @Test - void testRegexpMatch() { - assertEquals("value ~* 'test'", sqlBuilder.regexpMatch("value", "'test'")); - assertEquals("number ~* '\\d'", sqlBuilder.regexpMatch("number", "'\\d'")); - assertEquals("color ~* '^Blue$'", sqlBuilder.regexpMatch("color", "'^Blue$'")); - assertEquals("id ~* '[a-z]\\w+\\d{3}'", sqlBuilder.regexpMatch("id", "'[a-z]\\w+\\d{3}'")); - } - - @Test - void testJsonExtract() { - assertEquals("value ->> 'D7m8vpzxHDJ'", sqlBuilder.jsonExtract("value", "D7m8vpzxHDJ")); - } - - @Test - void testJsonExtractNested() { - assertEquals( - "eventdatavalues #>> '{D7m8vpzxHDJ, value}'", - sqlBuilder.jsonExtractNested("eventdatavalues", "D7m8vpzxHDJ", "value")); - } - - // Statements - - @Test - void testCreateTableA() { - Table table = getTableA(); - - String expected = - """ - create table "immunization" ("id" bigint not null, "data" char(11) not null, \ - "period\" varchar(50) not null, "created" timestamp null, "user" jsonb null, \ - "value" double precision null, primary key ("id"));"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableB() { - Table table = getTableB(); - - String expected = - """ - create unlogged table "vaccination" ("id" integer not null, \ - "facility_type" varchar(255) null collate "C", "bcg_doses" double precision null, \ - check("id">0), check("bcg_doses">0));"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testCreateTableC() { - Table table = getTableC(); - - String expected = - """ - create table "nutrition" ("id" bigint not null, "vitamin_a" bigint null, \ - "vitamin_d" bigint null, primary key ("id")) inherits ("vaccination");"""; - - assertEquals(expected, sqlBuilder.createTable(table)); - } - - @Test - void testAnalyzeTable() { - Table table = getTableA(); - - String expected = "analyze \"immunization\";"; - - assertEquals(expected, sqlBuilder.analyzeTable(table)); - } - - @Test - void testVacuumTable() { - Table table = getTableA(); - - String expected = "vacuum \"immunization\";"; - - assertEquals(expected, sqlBuilder.vacuumTable(table)); - } - - @Test - void testRenameTable() { - Table table = getTableA(); - - String expected = "alter table \"immunization\" rename to \"vaccination\";"; - - assertEquals(expected, sqlBuilder.renameTable(table, "vaccination")); - } - - @Test - void testDropTableIfExists() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists(table)); - } - - @Test - void testDropTableIfExistsString() { - String expected = "drop table if exists \"immunization\";"; - - assertEquals(expected, sqlBuilder.dropTableIfExists("immunization")); - } - - @Test - void testDropTableIfExistsCascade() { - Table table = getTableA(); - - String expected = "drop table if exists \"immunization\" cascade;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade(table)); - } - - @Test - void testDropTableIfExistsCascadeString() { - String expected = "drop table if exists \"immunization\" cascade;"; - - assertEquals(expected, sqlBuilder.dropTableIfExistsCascade("immunization")); - } - - @Test - void testSwapTable() { - String expected = - """ - drop table if exists "vaccination" cascade; \ - alter table "immunization" rename to "vaccination";"""; - - assertEquals(expected, sqlBuilder.swapTable(getTableA(), "vaccination")); - } - - @Test - void testSetParent() { - String expected = "alter table \"immunization\" inherit \"vaccination\";"; - - assertEquals(expected, sqlBuilder.setParentTable(getTableA(), "vaccination")); - } - - @Test - void testRemoveParent() { - String expected = "alter table \"immunization\" no inherit \"vaccination\";"; - - assertEquals(expected, sqlBuilder.removeParentTable(getTableA(), "vaccination")); - } - - @Test - void testSwapParentTable() { - String expected = - """ - alter table "immunization" no inherit "vaccination"; \ - alter table "immunization" inherit \"nutrition\";"""; - - assertEquals(expected, sqlBuilder.swapParentTable(getTableA(), "vaccination", "nutrition")); - } - - @Test - void testTableExists() { - String expected = - """ - select t.table_name from information_schema.tables t \ - where t.table_schema = 'public' and t.table_name = 'immunization';"""; - - assertEquals(expected, sqlBuilder.tableExists("immunization")); - } - - @Test - void testCountRows() { - String expected = - """ - select count(*) as row_count from "immunization";"""; - - assertEquals(expected, sqlBuilder.countRows(getTableA())); - } - - @Test - void testCreateIndexA() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_data\" on \"immunization\" using btree(\"data\");"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(0))); - } - - @Test - void testCreateIndexB() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_period_created\" on \"immunization\" using btree(\"period\",\"created\");"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(1))); - } - - @Test - void testCreateIndexC() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_user\" on \"immunization\" using gin(\"user\");"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(2))); - } - - @Test - void testCreateIndexD() { - List indexes = getIndexesA(); - - String expected = - "create index \"in_immunization_data_period\" on \"immunization\" using btree(lower(\"data\"),lower(\"period\"));"; - - assertEquals(expected, sqlBuilder.createIndex(indexes.get(3))); - } - - @Test - void testCreateIndexWithDescNullsLast() { - // given - String expected = - "create unique index \"index_a\" on \"table_a\" using btree(\"column_a\" desc nulls last);"; - Index index = - Index.builder() - .name("index_a") - .tableName("table_a") - .unique(Unique.UNIQUE) - .columns(List.of("column_a")) - .sortOrder("desc nulls last") - .build(); - - // when - String createIndexStmt = sqlBuilder.createIndex(index); - - // then - assertEquals(expected, createIndexStmt); - } -} From 40ac8f36551bd3fb69bfa65bc29d4d88a660f68c Mon Sep 17 00:00:00 2001 From: teleivo Date: Mon, 2 Dec 2024 15:25:37 +0100 Subject: [PATCH 23/24] chore: let COC.isDefault look at its own name only DHIS2-14968 (#19266) * chore: let COC.isDefault look at its own name only * test: fix code review comment * test: simplify setup --------- Co-authored-by: Jan Bernitt --- .../dhis/category/CategoryOptionCombo.java | 2 +- .../category/CategoryOptionComboTest.java | 2 +- .../dhis/category/DefaultCategoryService.java | 2 +- .../expression/ExpressionServiceTest.java | 14 +-- .../event/DataRelationsValidatorTest.java | 1 + .../tracker/export/MappingErrors.java | 95 +++++++++++-------- .../tracker/export/MappingErrorsTest.java | 8 +- 7 files changed, 65 insertions(+), 59 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryOptionCombo.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryOptionCombo.java index d728fd6b594..fcda8333a8f 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryOptionCombo.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/category/CategoryOptionCombo.java @@ -162,7 +162,7 @@ public void removeAllCategoryOptions() { @Override public boolean isDefault() { - return categoryCombo != null && DEFAULT_NAME.equals(categoryCombo.getName()); + return DEFAULT_NAME.equals(name); } /** diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/category/CategoryOptionComboTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/category/CategoryOptionComboTest.java index 30c0a609cd9..429ff8a02a9 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/category/CategoryOptionComboTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/category/CategoryOptionComboTest.java @@ -145,7 +145,7 @@ void hasDefault() { @Test void testIsDefault() { - categoryComboA.setName(CategoryCombo.DEFAULT_CATEGORY_COMBO_NAME); + optionComboA.setName(CategoryOptionCombo.DEFAULT_NAME); assertTrue(optionComboA.isDefault()); } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/category/DefaultCategoryService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/category/DefaultCategoryService.java index 6fd781feb03..7daabeaa101 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/category/DefaultCategoryService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/category/DefaultCategoryService.java @@ -580,7 +580,7 @@ public void generateDefaultDimension(UserDetails actingUser) { @Override @Transactional(readOnly = true) public CategoryOptionCombo getDefaultCategoryOptionCombo() { - return categoryOptionComboStore.getByName(CategoryCombo.DEFAULT_CATEGORY_COMBO_NAME); + return categoryOptionComboStore.getByName(CategoryOptionCombo.DEFAULT_NAME); } @Override diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java index b472290e850..f9bde0ece91 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java @@ -283,6 +283,7 @@ public void setUp() { coc = rnd.nextObject(CategoryOptionCombo.class); coc.setName(DEFAULT_CATEGORY_COMBO_NAME); + assertTrue(coc.isDefault(), "coc must be the default category option combo"); optionCombos.add(coc); @@ -873,21 +874,12 @@ void testGetExpressionDescription() { when(dimensionService.getDataDimensionalItemObject(getId(deA))).thenReturn(deA); description = target.getExpressionDescription(expressionM, INDICATOR_EXPRESSION); - assertThat( - description, - is(deA.getDisplayName() + "-" + deB.getDisplayName() + " " + coc.getDisplayName())); + assertThat(description, is(deA.getDisplayName() + "-" + deB.getDisplayName())); when(dimensionService.getDataDimensionalItemObject(getId(reportingRate))) .thenReturn(reportingRate); description = target.getExpressionDescription(expressionR, INDICATOR_EXPRESSION); - assertThat( - description, - is( - deB.getDisplayName() - + " " - + coc.getDisplayName() - + " + " - + reportingRate.getDisplayName())); + assertThat(description, is(deB.getDisplayName() + " + " + reportingRate.getDisplayName())); } @Test diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DataRelationsValidatorTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DataRelationsValidatorTest.java index 2dbc5376c3b..8d538548348 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DataRelationsValidatorTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/DataRelationsValidatorTest.java @@ -918,6 +918,7 @@ private CategoryCombo defaultCategoryCombo() { assertTrue(cc.isDefault(), "tests rely on this CC being the default one"); cc.setDataDimensionType(DataDimensionType.ATTRIBUTE); CategoryOptionCombo aoc = createCategoryOptionCombo(cc, co); + aoc.setName(CategoryOptionCombo.DEFAULT_NAME); assertTrue(aoc.isDefault(), "tests rely on this AOC being the default one"); cc.setOptionCombos(Sets.newHashSet(aoc)); return cc; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrors.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrors.java index 51161fe163e..11746fc67bf 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrors.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrors.java @@ -30,8 +30,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import lombok.RequiredArgsConstructor; import org.hisp.dhis.category.CategoryOption; import org.hisp.dhis.category.CategoryOptionCombo; @@ -51,7 +51,8 @@ public class MappingErrors { // metadata if such large numbers do not have identifiers for the requested idScheme. private static final int DISPLAY_MAX_UIDS = 20; - private final Map, Set> errors = new HashMap<>(); + private final Map, Set> errors = + new HashMap<>(); private final TrackerIdSchemeParams idSchemeParams; /** @@ -77,43 +78,25 @@ public String toString() { "Following metadata listed using their UIDs is missing identifiers for the requested" + " idScheme:\n"); - AtomicBoolean hasDefaultCategory = new AtomicBoolean(); - - errors.forEach( - (metadataClass, metadatas) -> { - if (metadatas.isEmpty()) { - return; - } - - result.append("\n"); - result.append(metadataClass.getSimpleName()); - result.append("["); - result.append(idSchemeParams.getByClass(metadataClass)); - result.append("]="); - - int count = 1; - int size = metadatas.size(); - for (IdentifiableObject metadata : metadatas) { - if (count > DISPLAY_MAX_UIDS) { - result.append("..."); - return; - } - - result.append(metadata.getUid()); - if (isDefaultCategory(metadata)) { - hasDefaultCategory.set(true); - result.append("(default)"); - } - if (count < size) { - result.append(","); - } - count++; - } - }); + boolean hasDefaultCategory = false; + + for (Entry, Set> entry : + errors.entrySet()) { + Set metadatas = entry.getValue(); + if (metadatas.isEmpty()) { + continue; + } + + appendMetadataPrefix(result, entry); + + if (appendUids(result, metadatas)) { + hasDefaultCategory = true; + } + } // default category option (combo) are guaranteed to have a UID, name and code but no attribute // values - if (hasDefaultCategory.get()) { + if (hasDefaultCategory) { result.append( """ @@ -124,11 +107,45 @@ Data linked to default category option (combo)s cannot be exported using\ return result.toString(); } + private void appendMetadataPrefix( + StringBuilder result, + Entry, Set> entry) { + Class metadataClass = entry.getKey(); + result.append("\n"); + result.append(metadataClass.getSimpleName()); + result.append("["); + result.append(idSchemeParams.getByClass(metadataClass)); + result.append("]="); + } + + private boolean appendUids(StringBuilder result, Set metadatas) { + int count = 1; + int size = metadatas.size(); + boolean hasDefaultCategory = false; + for (IdentifiableObject metadata : metadatas) { + if (count > DISPLAY_MAX_UIDS) { + result.append("..."); + continue; + } + + result.append(metadata.getUid()); + + if (isDefaultCategory(metadata)) { + hasDefaultCategory = true; + result.append("(default)"); + } + + if (count < size) { + result.append(","); + } + count++; + } + return hasDefaultCategory; + } + private static boolean isDefaultCategory(IdentifiableObject metadata) { return metadata instanceof CategoryOptionCombo categoryOptionCombo - && CategoryOptionCombo.DEFAULT_NAME.equals( - categoryOptionCombo.getName()) // CategoryOptionCombo.isDefault - // checks the CategoryCombo.name and we might not have mapped the CategoryCombo + && categoryOptionCombo.isDefault() || metadata instanceof CategoryOption categoryOption && categoryOption.isDefault(); } } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrorsTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrorsTest.java index b08d46cb9cf..76cdf2d4f6c 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrorsTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/MappingErrorsTest.java @@ -34,7 +34,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.hisp.dhis.category.CategoryCombo; import org.hisp.dhis.category.CategoryOption; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.common.CodeGenerator; @@ -101,13 +100,10 @@ void shouldReportErrors() { } @Test - void shouldReportDefaultCategoryOptionComboCannotBeExortedUsingIdSchemAttribute() { + void shouldReportDefaultCategoryOptionComboCannotBeExortedUsingIdSchemeAttribute() { CategoryOptionCombo defaultCategoryOptionCombo = new CategoryOptionCombo(); defaultCategoryOptionCombo.setName(CategoryOptionCombo.DEFAULT_NAME); defaultCategoryOptionCombo.setUid(CodeGenerator.generateUid()); - CategoryCombo defaultCategoryCombo = new CategoryCombo(); - defaultCategoryCombo.setName(CategoryOptionCombo.DEFAULT_NAME); - defaultCategoryOptionCombo.setCategoryCombo(defaultCategoryCombo); assertTrue( defaultCategoryOptionCombo.isDefault(), "this test needs the CategoryOptionCombo to be the default one"); @@ -136,7 +132,7 @@ void shouldReportDefaultCategoryOptionComboCannotBeExortedUsingIdSchemAttribute( } @Test - void shouldReportDefaultCategoryOptionCannotBeExortedUsingIdSchemAttribute() { + void shouldReportDefaultCategoryOptionCannotBeExortedUsingIdSchemeAttribute() { CategoryOption defaultCategoryOption = new CategoryOption(); defaultCategoryOption.setName(CategoryOption.DEFAULT_NAME); defaultCategoryOption.setUid(CodeGenerator.generateUid()); From 39b5511344023394c1099048e1e49659f15628bb Mon Sep 17 00:00:00 2001 From: Jan Bernitt Date: Mon, 2 Dec 2024 16:57:09 +0100 Subject: [PATCH 24/24] fix: SMS inbound and outbound lists stack overflow [DHIS2-18542] (#19364) --- .../dhis/webapi/controller/SmsInboundControllerTest.java | 8 ++++++++ .../dhis/webapi/controller/SmsOutboundControllerTest.java | 8 ++++++++ .../dhis/webapi/controller/sms/SmsInboundController.java | 4 ++-- .../dhis/webapi/controller/sms/SmsOutboundController.java | 5 +++-- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsInboundControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsInboundControllerTest.java index 1fad2629914..5d29fa4c17a 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsInboundControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsInboundControllerTest.java @@ -28,8 +28,10 @@ package org.hisp.dhis.webapi.controller; import static org.hisp.dhis.test.webapi.Assertions.assertWebMessage; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.hisp.dhis.http.HttpStatus; +import org.hisp.dhis.jsontree.JsonObject; import org.hisp.dhis.sms.incoming.IncomingSms; import org.hisp.dhis.sms.incoming.IncomingSmsService; import org.hisp.dhis.test.webapi.H2ControllerIntegrationTestBase; @@ -51,6 +53,12 @@ class SmsInboundControllerTest extends H2ControllerIntegrationTestBase { @Test void testGetInboundSMSMessage() { + JsonObject list = GET("/sms/inbound").content(); + assertEquals(0, list.getArray("inboundsmss").size()); + } + + @Test + void testGetInboundSMSMessage_Forbidden() { User guestUser = createUserWithAuth("guestuser", "NONE"); injectSecurityContextUser(guestUser); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsOutboundControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsOutboundControllerTest.java index 3d4157130a7..33eb7e10471 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsOutboundControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SmsOutboundControllerTest.java @@ -28,8 +28,10 @@ package org.hisp.dhis.webapi.controller; import static org.hisp.dhis.test.webapi.Assertions.assertWebMessage; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.hisp.dhis.http.HttpStatus; +import org.hisp.dhis.jsontree.JsonObject; import org.hisp.dhis.sms.outbound.OutboundSms; import org.hisp.dhis.sms.outbound.OutboundSmsService; import org.hisp.dhis.test.webapi.H2ControllerIntegrationTestBase; @@ -52,6 +54,12 @@ class SmsOutboundControllerTest extends H2ControllerIntegrationTestBase { @Test void testGetOutboundSMSMessage() { + JsonObject list = GET("/sms/outbound").content(); + assertEquals(0, list.getArray("outboundsmss").size()); + } + + @Test + void testGetOutboundSMSMessage_Forbidden() { User guestUser = createUserWithAuth("guestuser", "NONE"); injectSecurityContextUser(guestUser); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/sms/SmsInboundController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/sms/SmsInboundController.java index d73448e7f6b..fcc2c2e12b9 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/sms/SmsInboundController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/sms/SmsInboundController.java @@ -106,8 +106,8 @@ public class SmsInboundController extends AbstractCrudController