From fccc3326665842a263d9e72386f0c60d05fa160f Mon Sep 17 00:00:00 2001 From: Luc Gasser Date: Thu, 17 Oct 2024 17:17:47 +0200 Subject: [PATCH 1/2] Add datasetid column --- .../java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java b/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java index f1dbb53..53625ef 100644 --- a/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java +++ b/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java @@ -137,7 +137,7 @@ private Attribute parseNgsiLdAttribute(String key, JSONObject value, boolean fla addAttributeIfValid(subAttributes, subAttribute); } } else if (object instanceof JSONObject) { - Attribute subAttribute = parseNgsiLdSubAttribute(keyOne, value.getJSONObject(keyOne)); + Attribute subAttribute = parseNgsiLdSubAttribute(keyOne, value.getJSONObject(keyOne)); addAttributeIfValid(subAttributes, subAttribute); } else { logger.warn("Sub Attribute {} has unexpected value type: {}", keyOne, object.getClass()); @@ -150,6 +150,10 @@ private Attribute parseNgsiLdAttribute(String key, JSONObject value, boolean fla "parametername", "Property", "", "", "", "", key.toLowerCase(), false, null ); subAttributes.add(parameterName); + Attribute parameterDatasetId = new Attribute( + "datasetid", "Property", "", "", "", "", datasetId.toLowerCase(), false, null + ); + subAttributes.add(parameterDatasetId); return new Attribute(GENERIC_MEASURE, attrType, "", observedAt, createdAt, modifiedAt, attrValue, true, subAttributes); } else { return new Attribute(key.toLowerCase(), attrType, datasetId, observedAt, createdAt, modifiedAt, attrValue, !subAttributes.isEmpty(), subAttributes); From cf929a4ab9bb1d41a07c091311df94b0f5dc9c93 Mon Sep 17 00:00:00 2001 From: Luc Gasser Date: Fri, 18 Oct 2024 11:19:48 +0200 Subject: [PATCH 2/2] Handle datasetid addition in tests --- .../java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java | 5 ++++- .../io/nifi/processors/ngsild/TestPostgreSQLBackend.java | 3 ++- .../egm/io/nifi/processors/ngsild/utils/TestNGSIUtils.java | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java b/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java index 53625ef..aeb6b38 100644 --- a/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java +++ b/nifi-ngsild-postgresql-processors/src/main/java/egm/io/nifi/processors/ngsild/utils/NGSIUtils.java @@ -137,7 +137,7 @@ private Attribute parseNgsiLdAttribute(String key, JSONObject value, boolean fla addAttributeIfValid(subAttributes, subAttribute); } } else if (object instanceof JSONObject) { - Attribute subAttribute = parseNgsiLdSubAttribute(keyOne, value.getJSONObject(keyOne)); + Attribute subAttribute = parseNgsiLdSubAttribute(keyOne, value.getJSONObject(keyOne)); addAttributeIfValid(subAttributes, subAttribute); } else { logger.warn("Sub Attribute {} has unexpected value type: {}", keyOne, object.getClass()); @@ -150,6 +150,9 @@ private Attribute parseNgsiLdAttribute(String key, JSONObject value, boolean fla "parametername", "Property", "", "", "", "", key.toLowerCase(), false, null ); subAttributes.add(parameterName); + if (Objects.equals(datasetId, "")){ + datasetId = "default"; + } Attribute parameterDatasetId = new Attribute( "datasetid", "Property", "", "", "", "", datasetId.toLowerCase(), false, null ); diff --git a/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/TestPostgreSQLBackend.java b/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/TestPostgreSQLBackend.java index 6afe67d..9437d4b 100644 --- a/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/TestPostgreSQLBackend.java +++ b/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/TestPostgreSQLBackend.java @@ -37,12 +37,13 @@ public void testListOfFieldsWithFlattenedObservations() throws IOException { Map typedFields = postgreSQLBackend.listOfFields(entity, "urn:ngsi-ld:Dataset:", false, Collections.emptySet()); - assertEquals(26, typedFields.size()); + assertEquals(27, typedFields.size()); Set keys = typedFields.keySet(); assertTrue(keys.contains(GENERIC_MEASURE)); assertTrue(keys.contains(GENERIC_MEASURE + "_observedat")); assertTrue(keys.contains(GENERIC_MEASURE + "_unitcode")); assertTrue(keys.contains(GENERIC_MEASURE + "_parametername")); + assertTrue(keys.contains(GENERIC_MEASURE + "_datasetid")); } @ParameterizedTest diff --git a/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/utils/TestNGSIUtils.java b/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/utils/TestNGSIUtils.java index 0616ef5..4dc454d 100644 --- a/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/utils/TestNGSIUtils.java +++ b/nifi-ngsild-postgresql-processors/src/test/java/egm/io/nifi/processors/ngsild/utils/TestNGSIUtils.java @@ -68,10 +68,10 @@ public void testEntityWithFlattenObservations() throws IOException { // the entity under test has four observations assertEquals(4, measureAttributes.size()); Attribute measureAttribute = measureAttributes.get(0); - // each observation should have two sub-attributes: parametername and unitcode - assertEquals(2, measureAttribute.subAttrs.size()); + // each observation should have three sub-attributes: parametername, unitcode and datasetId + assertEquals(3, measureAttribute.subAttrs.size()); assertTrue(measureAttribute.subAttrs.stream().allMatch(attribute -> - attribute.getAttrName().equals("parametername") || attribute.getAttrName().equals("unitcode") + attribute.getAttrName().equals("parametername") || attribute.getAttrName().equals("unitcode") || attribute.getAttrName().equals("datasetid") )); } }