[6-dev] Fix address/location references on nested model types (finos#…

…3099) * Update DSL to version 9.15.1 and bundle to 11.16.2 * DSL update * Update bundle to version 11.17.1 * Clean up expectations * Update bundle version * Fix FIS mappings --------- Co-authored-by: Simon Cockx <[email protected]>
rosetta-models · Aug 27, 2024 · 86ff390 · 86ff390
1 parent 2f02f88
commit 86ff390
Show file tree

Hide file tree

Showing 17 changed files with 194 additions and 175 deletions.
diff --git a/RELEASE.md b/RELEASE.md
@@ -2,13 +2,17 @@
 
 _What is being released?_
 
-This release updates the `DSL` dependency. 
+This release updates the `ingest-test-framework` dependency.
 
 Version updates include:
-- `DSL` 9.15.0: patch for supporting tabulation of types with circular dependencies. For further details see DSL release notes: https://github.com/finos/rune-dsl/releases/tag/9.15.0
-- `DSL` 9.15.1: patch for missing generated Java files. For further details see DSL release notes: https://github.com/finos/rune-dsl/releases/tag/9.15.1
-- `DSL` 9.15.2: patch for missing Java meta classes. For further details see DSL release notes: https://github.com/finos/rune-dsl/releases/tag/9.15.2
+- `ingest-test-framework` 11.17.1: Add support for address/location references to be used on nested model types.
+
+FpML Java mapping code that sets address/location references have been updated accordingly.  There are no changes to the model. 
 
 _Review directions_
 
-The changes can be reviewed in PR: [#3097](https://github.com/finos/common-domain-model/pull/3097)
+The price referencing has been fixed in the following sample:
+
+- fpml-5-13 / products / interest-rate-derivatives / ird-ex37-zero-coupon-swap-known-amount-schedule.xml
+
+The changes can be reviewed in PR: [#3099](https://github.com/finos/common-domain-model/pull/3099)
diff --git a/pom.xml b/pom.xml
@@ -80,7 +80,7 @@
 
         <repoServerHost>s01.oss.sonatype.org</repoServerHost>
         <stagingTimeoutInMinutes>10</stagingTimeoutInMinutes>
-        <rosetta.bundle.version>11.17.0</rosetta.bundle.version>
+        <rosetta.bundle.version>11.17.1</rosetta.bundle.version>
         <rosetta.code-gen.version>${rosetta.bundle.version}</rosetta.code-gen.version>
         <rosetta.dsl.version>9.15.2</rosetta.dsl.version>
 

diff --git a/rosetta-source/src/main/java/cdm/event/workflow/processor/FISMapperMappingProcessor.java b/rosetta-source/src/main/java/cdm/event/workflow/processor/FISMapperMappingProcessor.java
@@ -205,7 +205,7 @@ private Multimap<String, MappingConsumer<TradeStateBuilder>> buildCommonMappings
                     .setReference(reference);
             return Arrays.asList(
                     new PathValue<>(pq.getModelPath().append(Path.parse("price[0].value.amount")), value),
-                    new PathValue<>(irp.getModelPath().append(Path.parse("rateSpecification.fixedRate.rateSchedule.price")), reference));
+                    new PathValue<>(irp.getModelPath().append(Path.parse("rateSpecification.fixedRate.rateSchedule.price.reference")), reference));
         });
 
         commonMappings.put("Loan_Value", (indexes, value, tradeState) -> {
@@ -224,7 +224,7 @@ private Multimap<String, MappingConsumer<TradeStateBuilder>> buildCommonMappings
                     .setReference(reference);
             return Arrays.asList(
                     new PathValue<>(pq.getModelPath().append(Path.parse("quantity[0].value.amount")), value),
-                    new PathValue<>(irp.getModelPath().append(Path.parse("priceQuantity.quantitySchedule")), reference));
+                    new PathValue<>(irp.getModelPath().append(Path.parse("priceQuantity.quantitySchedule.reference")), reference));
         });
 
         commonMappings.put("Loan_Value_Currency", (indexes, value, tradeState) -> {

diff --git a/...java/cdm/product/common/settlement/processor/FloatingRateCalculationMappingProcessor.java b/...java/cdm/product/common/settlement/processor/FloatingRateCalculationMappingProcessor.java
@@ -73,7 +73,7 @@ private void addPrice(Path subPath, PriceQuantityBuilder priceQuantityBuilder, M
         // price index must be incremented otherwise any references will break
         Path baseModelPath = toPath(getModelPath());
         Path amountModelPath = incrementPathElementIndex(baseModelPath, "price", emptyIfNull(priceQuantityBuilder.getPrice()).size());
-        updateMapping(mapping, amountModelPath);
+        addMapping(mapping.getXmlPath(), mapping.getXmlValue(), amountModelPath, mapping.getXmlValue());
         // add schedule (if exists)
         priceScheduleBuilder.setDatedValue(getSteps(mapping.getXmlPath().getParent(), amountModelPath.getParent()));
         // add to PriceQuantity
@@ -91,14 +91,6 @@ private UnitTypeBuilder toCurrencyUnitType(Path startsWithPath) {
                                 .build()));
     }
 
-    private void updateMapping(Mapping mapping, Path modelPath) {
-        mapping.setRosettaPath(modelPath);
-        // clear errors
-        mapping.setError(null);
-        mapping.setCondition(true);
-        mapping.setDuplicate(false);
-    }
-
     private List<DatedValue.DatedValueBuilder> getSteps(Path floorScheduleSynonymPath, Path priceScheduleModelPath) {
         List<DatedValue.DatedValueBuilder> steps = new ArrayList<>();
         int index = 0;
@@ -130,4 +122,8 @@ private Optional<DatedValue.DatedValueBuilder> getStep(Path floorScheduleSynonym
 
         return stepBuilder.hasData() ? Optional.of(stepBuilder) : Optional.empty();
     }
+
+    private void addMapping(Path xmlPath, Object xmlValue, Path modelPath, Object modelValue) {
+        getMappings().add(new Mapping(xmlPath, xmlValue, modelPath, modelValue, null, true, true, false));
+    }
 }
diff --git a/...rc/main/java/cdm/product/common/settlement/processor/NumberOfOptionsMappingProcessor.java b/...rc/main/java/cdm/product/common/settlement/processor/NumberOfOptionsMappingProcessor.java
@@ -4,12 +4,9 @@
 import cdm.base.math.NonNegativeQuantitySchedule;
 import cdm.base.math.UnitType;
 import cdm.product.common.settlement.PriceQuantity;
-import com.regnosys.rosetta.common.translation.MappingContext;
-import com.regnosys.rosetta.common.translation.MappingProcessor;
-import com.regnosys.rosetta.common.translation.MappingProcessorUtils;
-import com.regnosys.rosetta.common.translation.Path;
-import com.regnosys.rosetta.common.util.PathUtils;
+import com.regnosys.rosetta.common.translation.*;
 import com.rosetta.model.lib.RosettaModelObjectBuilder;
+import com.rosetta.model.lib.meta.Reference;
 import com.rosetta.model.lib.path.RosettaPath;
 
 import java.math.BigDecimal;
@@ -29,61 +26,80 @@
 @SuppressWarnings("unused")
 public class NumberOfOptionsMappingProcessor extends MappingProcessor {
 
-	private static final Path EQUITY_UNDERLIER_PATH = Path.parse("underlyer.singleUnderlyer.equity.instrumentId");
-	private static final Path INDEX_UNDERLIER_PATH = Path.parse("underlyer.singleUnderlyer.index.instrumentId");
-
-	public NumberOfOptionsMappingProcessor(RosettaPath modelPath, List<Path> synonymPaths, MappingContext context) {
-		super(modelPath, synonymPaths, context);
-	}
-
-	@Override
-	public void map(Path synonymPath, List<? extends RosettaModelObjectBuilder> builder, RosettaModelObjectBuilder parent) {
-		NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantity = NonNegativeQuantitySchedule.builder();
-		setAmountAndUnit(synonymPath, quantity, builder.size());
-		setMultiplierAndUnit(synonymPath, quantity);
-
-		if (quantity.hasData()) {
-			((PriceQuantity.PriceQuantityBuilder) parent).addQuantity(toReferencableQuantityBuilder(quantity));
-		}
-	}
-
-	private void setAmountAndUnit(Path synonymPath, NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantity, int index) {
-		Path baseModelPath = toPath(getModelPath());
-		Path mappedModelPath = incrementPathElementIndex(baseModelPath, "quantity", index);
-
-		MappingProcessorUtils.setValueAndUpdateMappings(synonymPath,
-				(xmlValue) -> quantity
-						.setValue(new BigDecimal(xmlValue))
-						.setUnit(UnitType.builder().setFinancialUnit(FinancialUnitEnum.CONTRACT)),
-				getMappings(),
-				PathUtils.toRosettaPath(mappedModelPath));
-	}
-
-	private void setMultiplierAndUnit(Path synonymPath, NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantity) {
-		Path parentSynonymPath = synonymPath.getParent();
-
-
-		setValueAndUpdateMappings(parentSynonymPath.addElement("optionEntitlement"),
-				(xmlValue) -> quantity.getOrCreateMultiplier().setValue(new BigDecimal(xmlValue)));
-
-		// bond option multiplier unit
-		setValueAndUpdateMappings(parentSynonymPath.addElement("entitlementCurrency"),
-				(xmlValue) -> quantity.getOrCreateMultiplier().setUnit(UnitType.builder().setCurrencyValue(xmlValue)));
-		setValueAndUpdateMappings(parentSynonymPath.addElement("entitlementCurrency").addElement("currencyScheme"),
-				(xmlValue) -> quantity.getOrCreateMultiplier().getOrCreateUnit().getOrCreateCurrency().getOrCreateMeta().setScheme(xmlValue));
-		// equity multiplier unit
-		if (pathExists(EQUITY_UNDERLIER_PATH)) {
-			quantity.getOrCreateMultiplier().setUnit(UnitType.builder().setFinancialUnit(FinancialUnitEnum.SHARE));
-		}
-		// index multiplier unit
-		if (pathExists(INDEX_UNDERLIER_PATH)) {
-			quantity.getOrCreateMultiplier().setUnit(UnitType.builder().setFinancialUnit(FinancialUnitEnum.INDEX_UNIT));
-		}
-	}
-
-	private boolean pathExists(Path endsWith) {
-		return getMappings().stream()
-				.filter(m -> m.getXmlPath().endsWith(endsWith))
-				.anyMatch(m -> m.getXmlValue() != null);
-	}
+    private static final Path EQUITY_UNDERLIER_PATH = Path.parse("underlyer.singleUnderlyer.equity.instrumentId");
+    private static final Path INDEX_UNDERLIER_PATH = Path.parse("underlyer.singleUnderlyer.index.instrumentId");
+
+    public NumberOfOptionsMappingProcessor(RosettaPath modelPath, List<Path> synonymPaths, MappingContext context) {
+        super(modelPath, synonymPaths, context);
+    }
+
+    @Override
+    public void map(Path synonymPath, List<? extends RosettaModelObjectBuilder> builder, RosettaModelObjectBuilder parent) {
+        NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantity = NonNegativeQuantitySchedule.builder();
+        setAmountAndUnit(synonymPath, quantity, builder.size());
+        setMultiplierAndUnit(synonymPath, quantity);
+
+        if (quantity.hasData()) {
+            ((PriceQuantity.PriceQuantityBuilder) parent).addQuantity(toReferencableQuantityBuilder(quantity));
+        }
+    }
+
+    private void setAmountAndUnit(Path synonymPath, NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantityBuilder, int index) {
+        Path baseModelPath = toPath(getModelPath());
+        Path mappedModelPath = incrementPathElementIndex(baseModelPath, "quantity", index);
+
+
+
+        MappingProcessorUtils.getNonNullMappedValue(synonymPath, getMappings()).ifPresent(xmlValue -> {
+            quantityBuilder
+                    .setValue(new BigDecimal(xmlValue))
+                    .setUnit(UnitType.builder().setFinancialUnit(FinancialUnitEnum.CONTRACT));
+            // add new mapping rather than updating, otherwise the referencing breaks
+            addMapping(synonymPath, xmlValue, mappedModelPath, xmlValue);
+            // clean up errors
+            updateReferenceMapping(synonymPath);
+        });
+    }
+
+    private void setMultiplierAndUnit(Path synonymPath, NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantity) {
+        Path parentSynonymPath = synonymPath.getParent();
+
+        setValueAndUpdateMappings(parentSynonymPath.addElement("optionEntitlement"),
+                (xmlValue) -> quantity.getOrCreateMultiplier().setValue(new BigDecimal(xmlValue)));
+
+        // bond option multiplier unit
+        setValueAndUpdateMappings(parentSynonymPath.addElement("entitlementCurrency"),
+                (xmlValue) -> quantity.getOrCreateMultiplier().setUnit(UnitType.builder().setCurrencyValue(xmlValue)));
+        setValueAndUpdateMappings(parentSynonymPath.addElement("entitlementCurrency").addElement("currencyScheme"),
+                (xmlValue) -> quantity.getOrCreateMultiplier().getOrCreateUnit().getOrCreateCurrency().getOrCreateMeta().setScheme(xmlValue));
+        // equity multiplier unit
+        if (pathExists(EQUITY_UNDERLIER_PATH)) {
+            quantity.getOrCreateMultiplier().setUnit(UnitType.builder().setFinancialUnit(FinancialUnitEnum.SHARE));
+        }
+        // index multiplier unit
+        if (pathExists(INDEX_UNDERLIER_PATH)) {
+            quantity.getOrCreateMultiplier().setUnit(UnitType.builder().setFinancialUnit(FinancialUnitEnum.INDEX_UNIT));
+        }
+    }
+
+    private boolean pathExists(Path endsWith) {
+        return getMappings().stream()
+                .filter(m -> m.getXmlPath().endsWith(endsWith))
+                .anyMatch(m -> m.getXmlValue() != null);
+    }
+
+    private void addMapping(Path xmlPath, Object xmlValue, Path modelPath, Object modelValue) {
+        getMappings().add(new Mapping(xmlPath, xmlValue, modelPath, modelValue, null, true, true, false));
+    }
+
+    private void updateReferenceMapping(Path synonymPath) {
+        getMappings().stream()
+                .filter(m -> m.getXmlPath().equals(synonymPath))
+                .filter(m -> m.getRosettaValue() instanceof Reference.ReferenceBuilder)
+                .filter(m -> m.getError() != null)
+                .forEach(m -> {
+                    m.setDuplicate(false);
+                    m.setError(null);
+                });
+    }
 }
diff --git a/...n/java/cdm/product/common/settlement/processor/TotalNotionalQuantityMappingProcessor.java b/...n/java/cdm/product/common/settlement/processor/TotalNotionalQuantityMappingProcessor.java
@@ -4,11 +4,7 @@
 import cdm.base.math.NonNegativeQuantitySchedule;
 import cdm.base.math.UnitType;
 import cdm.base.math.metafields.FieldWithMetaNonNegativeQuantitySchedule;
-import com.regnosys.rosetta.common.translation.MappingContext;
-import com.regnosys.rosetta.common.translation.MappingProcessor;
-import com.regnosys.rosetta.common.translation.MappingProcessorUtils;
-import com.regnosys.rosetta.common.translation.Path;
-import com.regnosys.rosetta.common.util.PathUtils;
+import com.regnosys.rosetta.common.translation.*;
 import com.rosetta.model.lib.RosettaModelObjectBuilder;
 import com.rosetta.model.lib.path.RosettaPath;
 
@@ -41,19 +37,20 @@ public void map(Path synonymPath, List<? extends RosettaModelObjectBuilder> buil
     }
 
     private Optional<NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder> getTotalNotionalQuantity(Path synonymPath, int index) {
-        NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantity = NonNegativeQuantitySchedule.builder();
+        NonNegativeQuantitySchedule.NonNegativeQuantityScheduleBuilder quantityBuilder = NonNegativeQuantitySchedule.builder();
 
         Path baseModelPath = toPath(getModelPath()).addElement("amount");
         Path mappedModelPath = incrementPathElementIndex(baseModelPath, "quantity", index);
 
-        MappingProcessorUtils.setValueAndUpdateMappings(synonymPath,
-                (xmlValue) -> quantity
-                        .setValue(new BigDecimal(xmlValue))
-                        .setUnit(UnitType.builder().setCapacityUnit(findCapacityUnitEnum(synonymPath.getParent()))),
-                getMappings(),
-                PathUtils.toRosettaPath(mappedModelPath));
-
-        return quantity.hasData() ? Optional.of(quantity) : Optional.empty();
+        MappingProcessorUtils.getNonNullMappedValue(synonymPath, getMappings()).ifPresent(xmlValue -> {
+            quantityBuilder
+                    .setValue(new BigDecimal(xmlValue))
+                    .setUnit(UnitType.builder().setCapacityUnit(findCapacityUnitEnum(synonymPath.getParent())));
+            // add new mapping rather than updating, otherwise the referencing breaks
+            addMapping(synonymPath, xmlValue, mappedModelPath, xmlValue);
+        });
+
+        return quantityBuilder.hasData() ? Optional.of(quantityBuilder) : Optional.empty();
     }
 
     private CapacityUnitEnum findCapacityUnitEnum(Path legSynonymPath) {
@@ -67,4 +64,8 @@ private Optional<CapacityUnitEnum> getCapacityUnitEnum(Path quantityUnitPath) {
         return getNonNullMappedValue(quantityUnitPath, getMappings())
                 .flatMap(xmlValue -> getSynonymToEnumMap().getEnumValueOptional(CapacityUnitEnum.class, xmlValue));
     }
+
+    private void addMapping(Path xmlPath, Object xmlValue, Path modelPath, Object modelValue) {
+        getMappings().add(new Mapping(xmlPath, xmlValue, modelPath, modelValue, null, true, true, false));
+    }
 }
diff --git a/rosetta-source/src/main/resources/cdm-sample-files/fis/expectations.json b/rosetta-source/src/main/resources/cdm-sample-files/fis/expectations.json
@@ -1,2 +1,2 @@
 [
-{"fileName":"cdm-sample-files/fis/isla1.xml","excludedPaths":0,"externalPaths":120,"outstandingMappings":10,"validationFailures":37,"qualificationExpectation":{"success":false,"qualifyResults":[{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"}],"qualifiableObjectCount":13}}]
+{"fileName":"cdm-sample-files/fis/isla1.xml","excludedPaths":0,"externalPaths":120,"outstandingMappings":19,"validationFailures":37,"qualificationExpectation":{"success":false,"qualifyResults":[{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"},{"qualifiedName":"SecuritiesFinance","qualifiedObjectClass":"cdm.product.template.EconomicTerms"}],"qualifiableObjectCount":13}}]
diff --git a/...tta-source/src/main/resources/mapping-analytics/fis-aggregated-mapping-failure-report.csv b/...tta-source/src/main/resources/mapping-analytics/fis-aggregated-mapping-failure-report.csv
@@ -1,5 +1,7 @@
-Mapping Coverage,91.67
+Mapping Coverage,84.17
 
 Input Path,Failed Mapping Count
 Activity.Loan_Value_Currency,5
+Activity.Loan_Value,5
+Activity.Activity_Rate,4
 Activity.Own_Cpty_Stock_Clearer_LEI,5