Skip to content

Commit

Permalink
Update TransformType.TRANSLATE path and input serialisation (#441)
Browse files Browse the repository at this point in the history 
* Add TransformType.INGEST

* Support input xml de-serialisation

* Update exception handling

* Add INGEST_CONFIG_PATH

* Add comment

* Re-use TransformType.TRANSLATE for new ingest

* Add deprecated flag

* Fix warning

* Move comment

* Use auto-close

* Require not null
  • Loading branch information
@hugohills-regnosys
hugohills-regnosys authored Nov 18, 2024
1 parent 7daaf74 commit d019f7e
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 54 deletions.
14 changes: 11 additions & 3 deletions src/main/java/com/regnosys/rosetta/common/transform/PipelineModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,21 @@ public class PipelineModel {
private final String name;
private final Transform transform;
private final String upstreamPipelineId;
private final Serialisation inputSerialisation;
private final Serialisation outputSerialisation;

@JsonCreator
public PipelineModel(@JsonProperty("id") String id,
@JsonProperty("name") String name,
@JsonProperty("transform") Transform transform,
@JsonProperty("upstreamPipelineId") String upstreamPipelineId,
@JsonProperty("inputSerialisation") Serialisation inputSerialisation,
@JsonProperty("outputSerialisation") Serialisation outputSerialisation) {
this.id = id;
this.name = name;
this.transform = transform;
this.upstreamPipelineId = upstreamPipelineId;
this.inputSerialisation = inputSerialisation;
this.outputSerialisation = outputSerialisation;
}

Expand All @@ -62,21 +65,25 @@ public String getUpstreamPipelineId() {
return upstreamPipelineId;
}

public Serialisation getInputSerialisation() {
return inputSerialisation;
}

public Serialisation getOutputSerialisation() {
return outputSerialisation;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof PipelineModel)) return false;
if (o == null || getClass() != o.getClass()) return false;
PipelineModel that = (PipelineModel) o;
return Objects.equals(getId(), that.getId()) && Objects.equals(getName(), that.getName()) && Objects.equals(getTransform(), that.getTransform()) && Objects.equals(getUpstreamPipelineId(), that.getUpstreamPipelineId()) && Objects.equals(getOutputSerialisation(), that.getOutputSerialisation());
return Objects.equals(id, that.id) && Objects.equals(name, that.name) && Objects.equals(transform, that.transform) && Objects.equals(upstreamPipelineId, that.upstreamPipelineId) && Objects.equals(inputSerialisation, that.inputSerialisation) && Objects.equals(outputSerialisation, that.outputSerialisation);
}

@Override
public int hashCode() {
return Objects.hash(getId(), getName(), getTransform(), getUpstreamPipelineId(), getOutputSerialisation());
return Objects.hash(id, name, transform, upstreamPipelineId, inputSerialisation, outputSerialisation);
}

@Override
Expand All @@ -86,6 +93,7 @@ public String toString() {
", name='" + name + '\'' +
", transform=" + transform +
", upstreamPipelineId='" + upstreamPipelineId + '\'' +
", inputSerialisation=" + inputSerialisation +
", outputSerialisation=" + outputSerialisation +
'}';
}
Expand Down
54 changes: 36 additions & 18 deletions src/main/java/com/regnosys/rosetta/common/transform/TestPackUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -29,20 +29,24 @@
import com.regnosys.rosetta.common.util.UrlUtils;

import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

public class TestPackUtils {
public static final Path PROJECTION_PATH = Paths.get(TransformType.PROJECTION.getResourcePath());

public static final Path PROJECTION_PATH = Paths.get(TransformType.PROJECTION.getResourcePath());
public static final Path PROJECTION_CONFIG_PATH_WITHOUT_ISO20022 = PROJECTION_PATH.resolve("config");
public static final Path REPORT_CONFIG_PATH = Paths.get(TransformType.REPORT.getResourcePath()).resolve("config");
public static final Path INGEST_CONFIG_PATH = Paths.get(TransformType.TRANSLATE.getResourcePath()).resolve("config");

public static TestPackModel createTestPack(String testPackName, TransformType transformType, String formattedFunctionName, List<TestPackModel.SampleModel> sampleModels) {
return new TestPackModel(createTestPackId(transformType, formattedFunctionName, testPackName), createPipelineId(transformType, formattedFunctionName), testPackName, sampleModels);
Expand All @@ -56,16 +60,25 @@ private static String createPipelineId(TransformType transformType, String forma
return String.format("pipeline-%s-%s", transformType.name().toLowerCase(), formattedFunctionName);
}

public static PipelineModel createPipeline(TransformType transformType, String functionQualifiedName, String displayName, String formattedFunctionName, String inputType, String outputType, String upstreamPipelineId, PipelineModel.Serialisation outputSerialisation) {
return new PipelineModel(createPipelineId(transformType, formattedFunctionName), displayName, new PipelineModel.Transform(transformType, functionQualifiedName, inputType, outputType), upstreamPipelineId, outputSerialisation);
public static PipelineModel createPipeline(TransformType transformType,
String functionQualifiedName,
String displayName,
String formattedFunctionName,
String inputType,
String outputType,
String upstreamPipelineId,
PipelineModel.Serialisation inputSerialisation,
PipelineModel.Serialisation outputSerialisation) {
String pipelineId = createPipelineId(transformType, formattedFunctionName);
PipelineModel.Transform transform = new PipelineModel.Transform(transformType, functionQualifiedName, inputType, outputType);
return new PipelineModel(pipelineId, displayName, transform, upstreamPipelineId, inputSerialisation, outputSerialisation);
}

public static List<PipelineModel> getPipelineModels(Path resourcePath, ClassLoader classLoader, ObjectMapper jsonObjectMapper) {
List<URL> pipelineFiles = findPaths(resourcePath, classLoader, "pipeline-.*\\.json");
List<PipelineModel> pipelineModels = pipelineFiles.stream()
return pipelineFiles.stream()
.map(url -> readFile(url, jsonObjectMapper, PipelineModel.class))
.collect(Collectors.toList());
return pipelineModels;
}

public static PipelineModel getPipelineModel(List<PipelineModel> pipelineModels, String functionName) {
Expand All @@ -89,11 +102,11 @@ public static List<TestPackModel> getTestPackModels(List<TestPackModel> testPack
.collect(Collectors.toList());
}

public static Optional<ObjectWriter> getObjectWriter(PipelineModel.Serialisation outputSerialisation) {
if (outputSerialisation != null && outputSerialisation.getFormat() == PipelineModel.Serialisation.Format.XML) {
URL xmlConfigPath = Resources.getResource(outputSerialisation.getConfigPath());
try {
return Optional.of(RosettaObjectMapperCreator.forXML(xmlConfigPath.openStream()).create().writerWithDefaultPrettyPrinter());
public static Optional<ObjectMapper> getObjectMapper(PipelineModel.Serialisation serialisation) {
if (serialisation != null && serialisation.getFormat() == PipelineModel.Serialisation.Format.XML) {
URL xmlConfigPath = Objects.requireNonNull(Resources.getResource(serialisation.getConfigPath()));
try (InputStream inputStream = xmlConfigPath.openStream()) {
return Optional.of(RosettaObjectMapperCreator.forXML(inputStream).create());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
Expand All @@ -102,6 +115,11 @@ public static Optional<ObjectWriter> getObjectWriter(PipelineModel.Serialisation
}
}

public static Optional<ObjectWriter> getObjectWriter(PipelineModel.Serialisation serialisation) {
return getObjectMapper(serialisation).map(ObjectMapper::writerWithDefaultPrettyPrinter);
}

@Deprecated
public static String getProjectionTestPackName(String reportId) {
return "test-pack-projection-" + reportId + "-report-to-iso20022.*\\.json";
}
Expand All @@ -112,7 +130,7 @@ public static String getReportTestPackName(String reportId) {

public static List<URL> findPaths(Path basePath, ClassLoader classLoader, String fileName) {
List<URL> expectations = ClassPathUtils
.findPathsFromClassPath(Arrays.asList(UrlUtils.toPortableString(basePath)),
.findPathsFromClassPath(Collections.singletonList(UrlUtils.toPortableString(basePath)),
fileName,
Optional.empty(),
classLoader)
Expand All @@ -123,10 +141,10 @@ public static List<URL> findPaths(Path basePath, ClassLoader classLoader, String
}

public static <T> T readFile(URL u, ObjectMapper mapper, Class<T> clazz) {
try {
return mapper.readValue(UrlUtils.openURL(u), clazz);
} catch (Exception e) {
throw new RuntimeException(e);
try (Reader src = UrlUtils.openURL(u)) {
return mapper.readValue(src, clazz);
} catch (IOException e) {
throw new UncheckedIOException(String.format("Failed to read url %s", u), e);
}
}
}
10 changes: 5 additions & 5 deletions src/main/java/com/regnosys/rosetta/common/transform/TransformType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@
import java.util.function.Function;

public enum TransformType {
PRE_TRANSLATE("translate/pre"),
TRANSLATE("translate", null),
TRANSLATE_1_5("ingestions", null),
POST_TRANSLATE("translate/post"),

PRE_TRANSLATE("ingest/pre"),
TRANSLATE("ingest", null),
TRANSLATE_1_5("ingestions", null), // Legacy ingest
POST_TRANSLATE("ingest/post"),
PRE_ENRICH("enrich/pre"),
ENRICH("enrich", null),
POST_ENRICH("enrich/post"),
Expand Down
38 changes: 10 additions & 28 deletions src/test/java/com/regnosys/rosetta/common/serialisation/xml/XmlSerialisationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -20,30 +20,11 @@
* ==============
*/

import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.Module;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.BeanPropertyWriter;
import com.fasterxml.jackson.databind.util.NameTransformer;
import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.io.Resources;
import com.regnosys.rosetta.common.serialisation.RosettaObjectMapperCreator;
import com.rosetta.model.lib.RosettaModelObject;
import com.rosetta.model.lib.RosettaModelObjectBuilder;
import com.rosetta.model.lib.annotations.RosettaDataType;
import com.rosetta.model.lib.meta.RosettaMetaData;
import com.rosetta.model.lib.path.RosettaPath;
import com.rosetta.model.lib.process.BuilderMerger;
import com.rosetta.model.lib.process.BuilderProcessor;
import com.rosetta.model.lib.process.Processor;
import com.rosetta.test.*;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
Expand All @@ -56,12 +37,11 @@
import javax.xml.validation.Validator;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.LocalTime;
import java.util.ArrayList;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;

Expand All @@ -79,8 +59,10 @@ public XmlSerialisationTest() throws SAXException, IOException {
xsdValidator = schema.newValidator();

// Create an XML mapper with the generated XML configuration based on the XSD schema
xmlMapper = RosettaObjectMapperCreator.forXML(
Resources.getResource("xml-serialisation/xml-config.json").openStream()).create();
URL url = Resources.getResource("xml-serialisation/xml-config.json");
try (InputStream inputStream = url.openStream()) {
xmlMapper = RosettaObjectMapperCreator.forXML(inputStream).create();
}
}

@Test
Expand Down

0 comments on commit d019f7e

Please sign in to comment.