diff --git a/src/main/java/com/regnosys/testing/pipeline/FunctionNameHelper.java b/src/main/java/com/regnosys/testing/pipeline/FunctionNameHelper.java index c345738..8db0afb 100644 --- a/src/main/java/com/regnosys/testing/pipeline/FunctionNameHelper.java +++ b/src/main/java/com/regnosys/testing/pipeline/FunctionNameHelper.java @@ -33,9 +33,13 @@ public class FunctionNameHelper { - public String getInputType(Class function) { + public Class getInputClass(Class function) { Method functionMethod = getFuncMethod(function); - return functionMethod.getParameterTypes()[0].getName(); + return functionMethod.getParameterTypes()[0]; + } + + public String getInputType(Class function) { + return getInputClass(function).getName(); } public String getOutputType(Class function) { @@ -76,6 +80,7 @@ protected String readableId(Class function) { .orElse(function.getSimpleName()); String sanitise = simpleName + .replace("Ingest_", "") .replace("Report_", "") .replace("Function", "") .replace("Enrich_", "") diff --git a/src/main/java/com/regnosys/testing/pipeline/PipelineFilter.java b/src/main/java/com/regnosys/testing/pipeline/PipelineFilter.java index d7fa467..c277ef8 100644 --- a/src/main/java/com/regnosys/testing/pipeline/PipelineFilter.java +++ b/src/main/java/com/regnosys/testing/pipeline/PipelineFilter.java @@ -1,5 +1,25 @@ package com.regnosys.testing.pipeline; +/*- + * =============== + * Rune Testing + * =============== + * Copyright (C) 2022 - 2024 REGnosys + * =============== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =============== + */ + import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/src/main/java/com/regnosys/testing/pipeline/PipelineFunctionRunner.java b/src/main/java/com/regnosys/testing/pipeline/PipelineFunctionRunner.java index ee93242..6616998 100644 --- a/src/main/java/com/regnosys/testing/pipeline/PipelineFunctionRunner.java +++ b/src/main/java/com/regnosys/testing/pipeline/PipelineFunctionRunner.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,16 +41,12 @@ public class PipelineFunctionRunner { public Result run(PipelineModel pipelineModel, ImmutableMap, String> outputSchemaMap, Path inputPath) { TestPackFunctionRunner functionRunner = getFunctionRunner(pipelineModel, outputSchemaMap); - Pair run = functionRunner.run(inputPath); - return new Result(run.left(), run.right()); + Pair result = functionRunner.run(inputPath); + return new Result(result.left(), result.right()); } - private TestPackFunctionRunner getFunctionRunner(PipelineModel pipelineModel, ImmutableMap, String> outputSchemaMap) { - if (pipelineModel.getOutputSerialisation() != null) { - return provider.create(pipelineModel.getTransform(), pipelineModel.getOutputSerialisation(), outputSchemaMap, injector); - } else { - return provider.create(pipelineModel.getTransform(), injector); - } + private TestPackFunctionRunner getFunctionRunner(PipelineModel pipelineModel, ImmutableMap, String> schemaMap) { + return provider.create(pipelineModel.getTransform(), pipelineModel.getInputSerialisation(), pipelineModel.getOutputSerialisation(), schemaMap, injector); } public static class Result { diff --git a/src/main/java/com/regnosys/testing/pipeline/PipelineModelBuilder.java b/src/main/java/com/regnosys/testing/pipeline/PipelineModelBuilder.java index 48a57c8..2b6cf45 100644 --- a/src/main/java/com/regnosys/testing/pipeline/PipelineModelBuilder.java +++ b/src/main/java/com/regnosys/testing/pipeline/PipelineModelBuilder.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,15 +42,15 @@ public List createPipelineModels(PipelineTree pipelineTree) { } protected PipelineModel build(PipelineNode modelBuilder, PipelineTreeConfig config) { - String inputType = helper.getInputType(modelBuilder.getFunction()); String outputType = helper.getOutputType(modelBuilder.getFunction()); + String inputSerialisationConfigPath = config.getXmlConfigMap().get(helper.getInputClass(modelBuilder.getFunction())); String outputSerialisationConfigPath = config.getXmlConfigMap().get(helper.getFuncMethod(modelBuilder.getFunction()).getReturnType()); String name = helper.getName(modelBuilder.getFunction()); // assume XML for now. - PipelineModel.Serialisation outputSerialisation = outputSerialisationConfigPath == null ? null : - new PipelineModel.Serialisation(PipelineModel.Serialisation.Format.XML, outputSerialisationConfigPath); + PipelineModel.Serialisation inputSerialisation = getSerialisation(inputSerialisationConfigPath); + PipelineModel.Serialisation outputSerialisation = getSerialisation(outputSerialisationConfigPath); String pipelineId = modelBuilder.id(config.isStrictUniqueIds()); String upstreamPipelineId = modelBuilder.upstreamId(config.isStrictUniqueIds()); @@ -58,7 +58,13 @@ protected PipelineModel build(PipelineNode modelBuilder, PipelineTreeConfig conf return new PipelineModel(pipelineId, name, new PipelineModel.Transform(modelBuilder.getTransformType(), modelBuilder.getFunction().getName(), inputType, outputType), - upstreamPipelineId, outputSerialisation); + upstreamPipelineId, + inputSerialisation, + outputSerialisation); } + private PipelineModel.Serialisation getSerialisation(String xmlConfigPath) { + return xmlConfigPath == null ? null : + new PipelineModel.Serialisation(PipelineModel.Serialisation.Format.XML, xmlConfigPath); + } } diff --git a/src/main/java/com/regnosys/testing/pipeline/PipelineTestPackWriter.java b/src/main/java/com/regnosys/testing/pipeline/PipelineTestPackWriter.java index 6720493..37874ec 100644 --- a/src/main/java/com/regnosys/testing/pipeline/PipelineTestPackWriter.java +++ b/src/main/java/com/regnosys/testing/pipeline/PipelineTestPackWriter.java @@ -87,7 +87,7 @@ public void writeTestPacks(PipelineTreeConfig config) throws IOException { Path outputPath = resourcesPath.resolve(pipelineNode.getOutputPath(config.isStrictUniqueIds())); LOGGER.info("Output path {} ", outputPath); - List inputSamples = findAllJsonSamples(inputPath); + List inputSamples = findAllSamples(inputPath); Map> testPackToSamples = filterAndGroupingByTestPackId(resourcesPath, inputPath, inputSamples, config.getTestPackIdFilter()); @@ -108,14 +108,13 @@ public void writeTestPacks(PipelineTreeConfig config) throws IOException { } } - private List findAllJsonSamples(Path inputDir) throws IOException { + private List findAllSamples(Path inputDir) throws IOException { if (!Files.exists(inputDir)) { return List.of(); } try (Stream paths = Files.walk(inputDir)) { return paths.filter(Files::isRegularFile) .filter(Files::exists) - .filter(x -> x.getFileName().toString().endsWith(".json")) .collect(Collectors.toList()); } } @@ -135,8 +134,8 @@ private TestPackModel writeTestPackSamples(Path resourcesPath, Path inputPath, P Path outputSample = resourcesPath.relativize(outputDir.resolve(resourcesPath.relativize(inputPath).relativize(inputSample))); outputSample = outputSample.getParent().resolve(Path.of(updateFileExtensionBasedOnOutputFormat(pipeline, outputSample.toFile().getName()))); - PipelineFunctionRunner.Result run = pipelineFunctionRunner.run(pipeline, config.getXmlSchemaMap(), resourcesPath.resolve(inputSample)); - TestPackModel.SampleModel.Assertions assertions = run.getAssertions(); + PipelineFunctionRunner.Result result = pipelineFunctionRunner.run(pipeline, config.getXmlSchemaMap(), resourcesPath.resolve(inputSample)); + TestPackModel.SampleModel.Assertions assertions = result.getAssertions(); String baseFileName = getBaseFileName(inputSample.toUri().toURL()); String displayName = baseFileName.replace("-", " "); @@ -145,7 +144,7 @@ private TestPackModel writeTestPackSamples(Path resourcesPath, Path inputPath, P sampleModels.add(sampleModel); Files.createDirectories(resourcesPath.resolve(outputSample).getParent()); - Files.write(resourcesPath.resolve(outputSample), run.getSerialisedOutput().getBytes()); + Files.write(resourcesPath.resolve(outputSample), result.getSerialisedOutput().getBytes()); } List sortedSamples = sampleModels @@ -160,11 +159,12 @@ private TestPackModel writeTestPackSamples(Path resourcesPath, Path inputPath, P } private String updateFileExtensionBasedOnOutputFormat(PipelineModel pipelineModel, String fileName) { - if (pipelineModel.getOutputSerialisation() != null) { - String outputFormat = pipelineModel.getOutputSerialisation().getFormat().toString().toLowerCase(); - return fileName.substring(0, fileName.lastIndexOf(".")) + "." + outputFormat; - } - return fileName; + String outputFormat = Optional.ofNullable(pipelineModel.getOutputSerialisation()) + .map(PipelineModel.Serialisation::getFormat) + .map(PipelineModel.Serialisation.Format::toString) + .orElse("json") + .toLowerCase(); + return fileName.substring(0, fileName.lastIndexOf(".")) + "." + outputFormat; } private Map> filterAndGroupingByTestPackId(Path resourcesPath, Path inputPath, List inputSamples, Predicate testPackIdFilter) { diff --git a/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerImpl.java b/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerImpl.java index 6cb0642..dd7c744 100644 --- a/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerImpl.java +++ b/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerImpl.java @@ -21,6 +21,7 @@ */ import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.regnosys.rosetta.common.hashing.ReferenceConfig; import com.regnosys.rosetta.common.hashing.ReferenceResolverProcessStep; @@ -46,7 +47,7 @@ import static com.regnosys.rosetta.common.transform.TestPackModel.SampleModel.Assertions; import static com.regnosys.rosetta.common.transform.TestPackUtils.readFile; -import static com.regnosys.testing.testpack.TestPackFunctionRunnerProviderImpl.JSON_OBJECT_MAPPER; +import static com.regnosys.testing.transform.TransformTestExtension.ERROR_OUTPUT; public class TestPackFunctionRunnerImpl implements TestPackFunctionRunner { private static final Logger LOGGER = LoggerFactory.getLogger(TestPackFunctionRunnerImpl.class); @@ -56,6 +57,7 @@ public class TestPackFunctionRunnerImpl implement private final Class inputType; private final RosettaTypeValidator typeValidator; private final ReferenceConfig referenceConfig; + private final ObjectMapper inputObjectMapper; private final ObjectWriter outputObjectWriter; private final Validator xsdValidator; @@ -64,12 +66,14 @@ public TestPackFunctionRunnerImpl(Function function, Class inputType, RosettaTypeValidator typeValidator, ReferenceConfig referenceConfig, + ObjectMapper inputObjectMapper, ObjectWriter outputObjectWriter, Validator xsdValidator) { this.function = function; this.inputType = inputType; this.typeValidator = typeValidator; this.referenceConfig = referenceConfig; + this.inputObjectMapper = inputObjectMapper; this.outputObjectWriter = outputObjectWriter; this.xsdValidator = xsdValidator; } @@ -81,14 +85,14 @@ public Pair run(Path inputPath) { // TODO - fix this hack. Path inputPathFromRepositoryRoot = inputPath.isAbsolute() ? inputPath : ROSETTA_SOURCE_PATH.resolve(inputPath); URL inputFileUrl = inputPathFromRepositoryRoot.toUri().toURL(); - IN input = readFile(inputFileUrl, JSON_OBJECT_MAPPER, inputType); + IN input = readFile(inputFileUrl, inputObjectMapper, inputType); output = function.apply(resolveReferences(input)); } catch (MalformedURLException e) { LOGGER.error("Failed to load input path {}", inputPath, e); - return Pair.of(null, new Assertions(null, null, true)); + return Pair.of(ERROR_OUTPUT, new Assertions(null, null, true)); } catch (Exception e) { LOGGER.error("Exception occurred running sample creation", e); - return Pair.of(null, new Assertions(null, null, true)); + return Pair.of(ERROR_OUTPUT, new Assertions(null, null, true)); } String serialisedOutput; @@ -108,6 +112,7 @@ public Pair run(Path inputPath) { return Pair.of(serialisedOutput, assertions); } + @SuppressWarnings("unchecked") private T resolveReferences(T o) { RosettaModelObjectBuilder builder = o.toBuilder(); new ReferenceResolverProcessStep(referenceConfig).runProcessStep(o.getType(), builder); diff --git a/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProvider.java b/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProvider.java index 0d9fa99..2b2e0aa 100644 --- a/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProvider.java +++ b/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProvider.java @@ -30,7 +30,9 @@ @ImplementedBy(TestPackFunctionRunnerProviderImpl.class) public interface TestPackFunctionRunnerProvider { - TestPackFunctionRunner create(Transform transform, Injector injector); - - TestPackFunctionRunner create(Transform transform, Serialisation outputSerialisation, ImmutableMap, String> outputSchemaMap, Injector injector); + TestPackFunctionRunner create(Transform transform, + Serialisation inputSerialisation, + Serialisation outputSerialisation, + ImmutableMap, String> outputSchemaMap, + Injector injector); } diff --git a/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProviderImpl.java b/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProviderImpl.java index ba3edf4..47dca48 100644 --- a/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProviderImpl.java +++ b/src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProviderImpl.java @@ -29,6 +29,7 @@ import com.regnosys.rosetta.common.hashing.ReferenceConfig; import com.regnosys.rosetta.common.serialisation.RosettaObjectMapper; import com.regnosys.rosetta.common.transform.PipelineModel; +import com.regnosys.rosetta.common.transform.TestPackUtils; import com.regnosys.rosetta.common.validation.RosettaTypeValidator; import com.rosetta.model.lib.RosettaModelObject; import org.xml.sax.SAXException; @@ -44,12 +45,13 @@ import java.util.Optional; import java.util.function.Function; +import static com.regnosys.rosetta.common.transform.TestPackUtils.getObjectMapper; import static com.regnosys.rosetta.common.transform.TestPackUtils.getObjectWriter; public class TestPackFunctionRunnerProviderImpl implements TestPackFunctionRunnerProvider { - protected static final ObjectMapper JSON_OBJECT_MAPPER = RosettaObjectMapper.getNewRosettaObjectMapper(); - protected final static ObjectWriter JSON_OBJECT_WRITER = + private static final ObjectMapper JSON_OBJECT_MAPPER = RosettaObjectMapper.getNewRosettaObjectMapper(); + private final static ObjectWriter JSON_OBJECT_WRITER = JSON_OBJECT_MAPPER .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) .writerWithDefaultPrettyPrinter(); @@ -60,28 +62,39 @@ public class TestPackFunctionRunnerProviderImpl implements TestPackFunctionRunne ReferenceConfig referenceConfig; @Override - public TestPackFunctionRunner create(PipelineModel.Transform transform, Injector injector) { - Class inputType = toClass(transform.getInputType()); - Class functionType = toClass(transform.getFunction()); - return createTestPackFunctionRunner(functionType, inputType, injector, JSON_OBJECT_WRITER, null); - } - - @Override - public TestPackFunctionRunner create(PipelineModel.Transform transform, PipelineModel.Serialisation outputSerialisation, ImmutableMap, String> outputSchemaMap, Injector injector) { + public TestPackFunctionRunner create(PipelineModel.Transform transform, + PipelineModel.Serialisation inputSerialisation, + PipelineModel.Serialisation outputSerialisation, + ImmutableMap, String> schemaMap, + Injector injector) { Class inputType = toClass(transform.getInputType()); Class outputType = toClass(transform.getOutputType()); + // Input de-serialisation + ObjectMapper inputObjectMapper = Optional.ofNullable(inputSerialisation) + .flatMap(TestPackUtils::getObjectMapper) + .orElse(JSON_OBJECT_MAPPER); // Output serialisation - ObjectWriter outputObjectWriter = getObjectWriter(outputSerialisation).orElse(JSON_OBJECT_WRITER); + ObjectWriter outputObjectWriter = Optional.ofNullable(outputSerialisation) + .flatMap(TestPackUtils::getObjectWriter) + .orElse(JSON_OBJECT_WRITER); // XSD validation - Validator xsdValidator = getXsdValidator(outputType, outputSchemaMap); - return createTestPackFunctionRunner(toClass(transform.getFunction()), inputType, injector, outputObjectWriter, xsdValidator); + Validator xsdValidator = Optional.ofNullable(schemaMap) + .map(sm -> getXsdValidator(outputType, sm)) + .orElse(null); + return createTestPackFunctionRunner(toClass(transform.getFunction()), inputType, injector, inputObjectMapper, outputObjectWriter, xsdValidator); } - private TestPackFunctionRunner createTestPackFunctionRunner(Class functionType, Class inputType, Injector injector, ObjectWriter outputObjectWriter, Validator xsdValidator) { + private TestPackFunctionRunner createTestPackFunctionRunner(Class functionType, + Class inputType, + Injector injector, + ObjectMapper inputObjectMapper, + ObjectWriter outputObjectWriter, + Validator xsdValidator) { Function transformFunction = getTransformFunction(functionType, inputType, injector); - return new TestPackFunctionRunnerImpl<>(transformFunction, inputType, typeValidator, referenceConfig, outputObjectWriter, xsdValidator); + return new TestPackFunctionRunnerImpl<>(transformFunction, inputType, typeValidator, referenceConfig, inputObjectMapper, outputObjectWriter, xsdValidator); } + @SuppressWarnings("unchecked") private Class toClass(String name) { try { ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); @@ -113,7 +126,7 @@ private Function getTran private Validator getXsdValidator(Class functionType, ImmutableMap, String> outputSchemaMap) { URL schemaUrl = Optional.ofNullable(outputSchemaMap.get(functionType)) - .map(r -> Resources.getResource(r)) + .map(Resources::getResource) .orElse(null); if (schemaUrl == null) { return null; @@ -125,7 +138,7 @@ private Validator getXsdValidator(Class functionType, ImmutableMap, Schema schema = schemaFactory.newSchema(schemaUrl); return schema.newValidator(); } catch (SAXException e) { - throw new RuntimeException(String.format("Failed to create schema validator for {}", schemaUrl), e); + throw new RuntimeException(String.format("Failed to create schema validator for %s", schemaUrl), e); } } } diff --git a/src/main/java/com/regnosys/testing/transform/TransformTestExtension.java b/src/main/java/com/regnosys/testing/transform/TransformTestExtension.java index 5a35b43..58da4fc 100644 --- a/src/main/java/com/regnosys/testing/transform/TransformTestExtension.java +++ b/src/main/java/com/regnosys/testing/transform/TransformTestExtension.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -81,19 +81,23 @@ public class TransformTestExtension implements BeforeAllCallback, AfterAllCal JSON_OBJECT_MAPPER .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) .writerWithDefaultPrettyPrinter(); + // use empty string as error value for function output as it gets serialised + public static final String ERROR_OUTPUT = ""; private final Module runtimeModule; private final Path configPath; private final Class funcType; private Validator xsdValidator; - @Inject RosettaTypeValidator typeValidator; - @Inject ReferenceConfig referenceConfig; + @Inject + RosettaTypeValidator typeValidator; + @Inject + ReferenceConfig referenceConfig; private Multimap actualExpectation; private PipelineModel pipelineModel; private Injector injector; + private ObjectMapper inputObjectMapper; private ObjectWriter outputObjectWriter; - - + public TransformTestExtension(Module runtimeModule, Path configPath, Class funcType) { this.runtimeModule = runtimeModule; this.configPath = configPath; @@ -119,6 +123,7 @@ public void beforeAll(ExtensionContext context) { this.injector.injectMembers(this); ClassLoader classLoader = this.getClass().getClassLoader(); this.pipelineModel = getPipelineModel(getPipelineModels(configPath, classLoader, JSON_OBJECT_MAPPER), funcType.getName()); + this.inputObjectMapper = getObjectMapper(pipelineModel.getInputSerialisation()).orElse(JSON_OBJECT_MAPPER); this.outputObjectWriter = getObjectWriter(pipelineModel.getOutputSerialisation()).orElse(JSON_OBJECT_WRITER); this.actualExpectation = ArrayListMultimap.create(); } @@ -148,10 +153,12 @@ public void runT protected TransformTestResult getResult(TestPackModel.SampleModel sampleModel, Function function) { String inputFile = sampleModel.getInputPath(); URL inputFileUrl = getInputFileUrl(inputFile); + assertNotNull(inputFileUrl); + Class inputType = getInputType(); - IN input = readFile(inputFileUrl, JSON_OBJECT_MAPPER, inputType); - + try { + IN input = readFile(inputFileUrl, inputObjectMapper, inputType); IN resolvedInput = resolveReferences(input); OUT output = function.apply(resolvedInput); @@ -174,7 +181,7 @@ protected Transf } catch (Exception e) { LOGGER.error("Exception occurred running transform", e); TestPackModel.SampleModel.Assertions assertions = new TestPackModel.SampleModel.Assertions(null, null, true); - return new TransformTestResult(null, updateSampleModel(sampleModel, assertions)); + return new TransformTestResult(ERROR_OUTPUT, updateSampleModel(sampleModel, assertions)); } }