diff --git a/src/main/java/com/regnosys/rosetta/common/transform/PipelineModel.java b/src/main/java/com/regnosys/rosetta/common/transform/PipelineModel.java index 7eaadda7..5f993501 100644 --- a/src/main/java/com/regnosys/rosetta/common/transform/PipelineModel.java +++ b/src/main/java/com/regnosys/rosetta/common/transform/PipelineModel.java @@ -31,6 +31,7 @@ public class PipelineModel { private final String name; private final Transform transform; private final String upstreamPipelineId; + private final Serialisation inputSerialisation; private final Serialisation outputSerialisation; @JsonCreator @@ -38,11 +39,13 @@ public PipelineModel(@JsonProperty("id") String id, @JsonProperty("name") String name, @JsonProperty("transform") Transform transform, @JsonProperty("upstreamPipelineId") String upstreamPipelineId, + @JsonProperty("inputSerialisation") Serialisation inputSerialisation, @JsonProperty("outputSerialisation") Serialisation outputSerialisation) { this.id = id; this.name = name; this.transform = transform; this.upstreamPipelineId = upstreamPipelineId; + this.inputSerialisation = inputSerialisation; this.outputSerialisation = outputSerialisation; } @@ -62,6 +65,10 @@ public String getUpstreamPipelineId() { return upstreamPipelineId; } + public Serialisation getInputSerialisation() { + return inputSerialisation; + } + public Serialisation getOutputSerialisation() { return outputSerialisation; } @@ -69,14 +76,14 @@ public Serialisation getOutputSerialisation() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof PipelineModel)) return false; + if (o == null || getClass() != o.getClass()) return false; PipelineModel that = (PipelineModel) o; - return Objects.equals(getId(), that.getId()) && Objects.equals(getName(), that.getName()) && Objects.equals(getTransform(), that.getTransform()) && Objects.equals(getUpstreamPipelineId(), that.getUpstreamPipelineId()) && Objects.equals(getOutputSerialisation(), that.getOutputSerialisation()); + return Objects.equals(id, that.id) && Objects.equals(name, that.name) && Objects.equals(transform, that.transform) && Objects.equals(upstreamPipelineId, that.upstreamPipelineId) && Objects.equals(inputSerialisation, that.inputSerialisation) && Objects.equals(outputSerialisation, that.outputSerialisation); } @Override public int hashCode() { - return Objects.hash(getId(), getName(), getTransform(), getUpstreamPipelineId(), getOutputSerialisation()); + return Objects.hash(id, name, transform, upstreamPipelineId, inputSerialisation, outputSerialisation); } @Override @@ -86,6 +93,7 @@ public String toString() { ", name='" + name + '\'' + ", transform=" + transform + ", upstreamPipelineId='" + upstreamPipelineId + '\'' + + ", inputSerialisation=" + inputSerialisation + ", outputSerialisation=" + outputSerialisation + '}'; } diff --git a/src/main/java/com/regnosys/rosetta/common/transform/TestPackUtils.java b/src/main/java/com/regnosys/rosetta/common/transform/TestPackUtils.java index 1b63db1f..f6af7add 100644 --- a/src/main/java/com/regnosys/rosetta/common/transform/TestPackUtils.java +++ b/src/main/java/com/regnosys/rosetta/common/transform/TestPackUtils.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,20 +29,24 @@ import com.regnosys.rosetta.common.util.UrlUtils; import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; import java.io.UncheckedIOException; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; public class TestPackUtils { - public static final Path PROJECTION_PATH = Paths.get(TransformType.PROJECTION.getResourcePath()); + public static final Path PROJECTION_PATH = Paths.get(TransformType.PROJECTION.getResourcePath()); public static final Path PROJECTION_CONFIG_PATH_WITHOUT_ISO20022 = PROJECTION_PATH.resolve("config"); public static final Path REPORT_CONFIG_PATH = Paths.get(TransformType.REPORT.getResourcePath()).resolve("config"); + public static final Path INGEST_CONFIG_PATH = Paths.get(TransformType.TRANSLATE.getResourcePath()).resolve("config"); public static TestPackModel createTestPack(String testPackName, TransformType transformType, String formattedFunctionName, List sampleModels) { return new TestPackModel(createTestPackId(transformType, formattedFunctionName, testPackName), createPipelineId(transformType, formattedFunctionName), testPackName, sampleModels); @@ -56,16 +60,25 @@ private static String createPipelineId(TransformType transformType, String forma return String.format("pipeline-%s-%s", transformType.name().toLowerCase(), formattedFunctionName); } - public static PipelineModel createPipeline(TransformType transformType, String functionQualifiedName, String displayName, String formattedFunctionName, String inputType, String outputType, String upstreamPipelineId, PipelineModel.Serialisation outputSerialisation) { - return new PipelineModel(createPipelineId(transformType, formattedFunctionName), displayName, new PipelineModel.Transform(transformType, functionQualifiedName, inputType, outputType), upstreamPipelineId, outputSerialisation); + public static PipelineModel createPipeline(TransformType transformType, + String functionQualifiedName, + String displayName, + String formattedFunctionName, + String inputType, + String outputType, + String upstreamPipelineId, + PipelineModel.Serialisation inputSerialisation, + PipelineModel.Serialisation outputSerialisation) { + String pipelineId = createPipelineId(transformType, formattedFunctionName); + PipelineModel.Transform transform = new PipelineModel.Transform(transformType, functionQualifiedName, inputType, outputType); + return new PipelineModel(pipelineId, displayName, transform, upstreamPipelineId, inputSerialisation, outputSerialisation); } public static List getPipelineModels(Path resourcePath, ClassLoader classLoader, ObjectMapper jsonObjectMapper) { List pipelineFiles = findPaths(resourcePath, classLoader, "pipeline-.*\\.json"); - List pipelineModels = pipelineFiles.stream() + return pipelineFiles.stream() .map(url -> readFile(url, jsonObjectMapper, PipelineModel.class)) .collect(Collectors.toList()); - return pipelineModels; } public static PipelineModel getPipelineModel(List pipelineModels, String functionName) { @@ -89,11 +102,11 @@ public static List getTestPackModels(List testPack .collect(Collectors.toList()); } - public static Optional getObjectWriter(PipelineModel.Serialisation outputSerialisation) { - if (outputSerialisation != null && outputSerialisation.getFormat() == PipelineModel.Serialisation.Format.XML) { - URL xmlConfigPath = Resources.getResource(outputSerialisation.getConfigPath()); - try { - return Optional.of(RosettaObjectMapperCreator.forXML(xmlConfigPath.openStream()).create().writerWithDefaultPrettyPrinter()); + public static Optional getObjectMapper(PipelineModel.Serialisation serialisation) { + if (serialisation != null && serialisation.getFormat() == PipelineModel.Serialisation.Format.XML) { + URL xmlConfigPath = Objects.requireNonNull(Resources.getResource(serialisation.getConfigPath())); + try (InputStream inputStream = xmlConfigPath.openStream()) { + return Optional.of(RosettaObjectMapperCreator.forXML(inputStream).create()); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -102,6 +115,11 @@ public static Optional getObjectWriter(PipelineModel.Serialisation } } + public static Optional getObjectWriter(PipelineModel.Serialisation serialisation) { + return getObjectMapper(serialisation).map(ObjectMapper::writerWithDefaultPrettyPrinter); + } + + @Deprecated public static String getProjectionTestPackName(String reportId) { return "test-pack-projection-" + reportId + "-report-to-iso20022.*\\.json"; } @@ -112,7 +130,7 @@ public static String getReportTestPackName(String reportId) { public static List findPaths(Path basePath, ClassLoader classLoader, String fileName) { List expectations = ClassPathUtils - .findPathsFromClassPath(Arrays.asList(UrlUtils.toPortableString(basePath)), + .findPathsFromClassPath(Collections.singletonList(UrlUtils.toPortableString(basePath)), fileName, Optional.empty(), classLoader) @@ -123,10 +141,10 @@ public static List findPaths(Path basePath, ClassLoader classLoader, String } public static T readFile(URL u, ObjectMapper mapper, Class clazz) { - try { - return mapper.readValue(UrlUtils.openURL(u), clazz); - } catch (Exception e) { - throw new RuntimeException(e); + try (Reader src = UrlUtils.openURL(u)) { + return mapper.readValue(src, clazz); + } catch (IOException e) { + throw new UncheckedIOException(String.format("Failed to read url %s", u), e); } } } diff --git a/src/main/java/com/regnosys/rosetta/common/transform/TransformType.java b/src/main/java/com/regnosys/rosetta/common/transform/TransformType.java index 0e31c6a5..ac928f02 100644 --- a/src/main/java/com/regnosys/rosetta/common/transform/TransformType.java +++ b/src/main/java/com/regnosys/rosetta/common/transform/TransformType.java @@ -23,11 +23,11 @@ import java.util.function.Function; public enum TransformType { - PRE_TRANSLATE("translate/pre"), - TRANSLATE("translate", null), - TRANSLATE_1_5("ingestions", null), - POST_TRANSLATE("translate/post"), - + PRE_TRANSLATE("ingest/pre"), + TRANSLATE("ingest", null), + TRANSLATE_1_5("ingestions", null), // Legacy ingest + POST_TRANSLATE("ingest/post"), + PRE_ENRICH("enrich/pre"), ENRICH("enrich", null), POST_ENRICH("enrich/post"), diff --git a/src/test/java/com/regnosys/rosetta/common/serialisation/xml/XmlSerialisationTest.java b/src/test/java/com/regnosys/rosetta/common/serialisation/xml/XmlSerialisationTest.java index d98aa60c..b03a3dec 100644 --- a/src/test/java/com/regnosys/rosetta/common/serialisation/xml/XmlSerialisationTest.java +++ b/src/test/java/com/regnosys/rosetta/common/serialisation/xml/XmlSerialisationTest.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,30 +20,11 @@ * ============== */ -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.fasterxml.jackson.core.*; -import com.fasterxml.jackson.databind.*; -import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.databind.ser.BeanPropertyWriter; -import com.fasterxml.jackson.databind.util.NameTransformer; -import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; -import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.io.Resources; import com.regnosys.rosetta.common.serialisation.RosettaObjectMapperCreator; -import com.rosetta.model.lib.RosettaModelObject; -import com.rosetta.model.lib.RosettaModelObjectBuilder; -import com.rosetta.model.lib.annotations.RosettaDataType; -import com.rosetta.model.lib.meta.RosettaMetaData; -import com.rosetta.model.lib.path.RosettaPath; -import com.rosetta.model.lib.process.BuilderMerger; -import com.rosetta.model.lib.process.BuilderProcessor; -import com.rosetta.model.lib.process.Processor; import com.rosetta.test.*; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -56,12 +37,11 @@ import javax.xml.validation.Validator; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.math.BigDecimal; import java.net.URL; import java.nio.charset.StandardCharsets; import java.time.LocalTime; -import java.util.ArrayList; -import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -79,8 +59,10 @@ public XmlSerialisationTest() throws SAXException, IOException { xsdValidator = schema.newValidator(); // Create an XML mapper with the generated XML configuration based on the XSD schema - xmlMapper = RosettaObjectMapperCreator.forXML( - Resources.getResource("xml-serialisation/xml-config.json").openStream()).create(); + URL url = Resources.getResource("xml-serialisation/xml-config.json"); + try (InputStream inputStream = url.openStream()) { + xmlMapper = RosettaObjectMapperCreator.forXML(inputStream).create(); + } } @Test