Merge pull request #191 from REGnosys/add-ingest-transform-type

Test pack creator - support input xml deserialisation

src/main/java/com/regnosys/testing/pipeline/FunctionNameHelper.java

@@ -33,9 +33,13 @@
 
 public class FunctionNameHelper {
 
-    public String getInputType(Class<? extends RosettaFunction> function) {
+    public Class<?> getInputClass(Class<? extends RosettaFunction> function) {
         Method functionMethod = getFuncMethod(function);
-        return functionMethod.getParameterTypes()[0].getName();
+        return functionMethod.getParameterTypes()[0];
+    }
+
+    public String getInputType(Class<? extends RosettaFunction> function) {
+        return getInputClass(function).getName();
     }
 
     public String getOutputType(Class<? extends RosettaFunction> function) {
@@ -76,6 +80,7 @@ protected String readableId(Class<? extends RosettaFunction> function) {
                 .orElse(function.getSimpleName());
 
         String sanitise = simpleName
+                .replace("Ingest_", "")
                 .replace("Report_", "")
                 .replace("Function", "")
                 .replace("Enrich_", "")

src/main/java/com/regnosys/testing/pipeline/PipelineFilter.java

@@ -1,5 +1,25 @@
 package com.regnosys.testing.pipeline;
 
+/*-
+ * ===============
+ * Rune Testing
+ * ===============
+ * Copyright (C) 2022 - 2024 REGnosys
+ * ===============
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ===============
+ */
+
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;

src/main/java/com/regnosys/testing/pipeline/PipelineFunctionRunner.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -41,16 +41,12 @@ public class PipelineFunctionRunner {
 
     public Result run(PipelineModel pipelineModel, ImmutableMap<Class<?>, String> outputSchemaMap, Path inputPath) {
         TestPackFunctionRunner functionRunner = getFunctionRunner(pipelineModel, outputSchemaMap);
-        Pair<String, TestPackModel.SampleModel.Assertions> run = functionRunner.run(inputPath);
-        return new Result(run.left(), run.right());
+        Pair<String, TestPackModel.SampleModel.Assertions> result = functionRunner.run(inputPath);
+        return new Result(result.left(), result.right());
     }
 
-    private TestPackFunctionRunner getFunctionRunner(PipelineModel pipelineModel, ImmutableMap<Class<?>, String> outputSchemaMap) {
-        if (pipelineModel.getOutputSerialisation() != null) {
-            return provider.create(pipelineModel.getTransform(), pipelineModel.getOutputSerialisation(), outputSchemaMap, injector);
-        } else {
-            return provider.create(pipelineModel.getTransform(), injector);
-        }
+    private TestPackFunctionRunner getFunctionRunner(PipelineModel pipelineModel, ImmutableMap<Class<?>, String> schemaMap) {
+        return provider.create(pipelineModel.getTransform(), pipelineModel.getInputSerialisation(), pipelineModel.getOutputSerialisation(), schemaMap, injector);
     }
 
     public static class Result {

src/main/java/com/regnosys/testing/pipeline/PipelineModelBuilder.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -42,23 +42,29 @@ public List<PipelineModel> createPipelineModels(PipelineTree pipelineTree) {
     }
 
     protected PipelineModel build(PipelineNode modelBuilder, PipelineTreeConfig config) {
-
         String inputType = helper.getInputType(modelBuilder.getFunction());
         String outputType = helper.getOutputType(modelBuilder.getFunction());
+        String inputSerialisationConfigPath = config.getXmlConfigMap().get(helper.getInputClass(modelBuilder.getFunction()));
         String outputSerialisationConfigPath = config.getXmlConfigMap().get(helper.getFuncMethod(modelBuilder.getFunction()).getReturnType());
         String name = helper.getName(modelBuilder.getFunction());
 
         // assume XML for now.
-        PipelineModel.Serialisation outputSerialisation = outputSerialisationConfigPath == null ? null :
-                new PipelineModel.Serialisation(PipelineModel.Serialisation.Format.XML, outputSerialisationConfigPath);
+        PipelineModel.Serialisation inputSerialisation = getSerialisation(inputSerialisationConfigPath);
+        PipelineModel.Serialisation outputSerialisation = getSerialisation(outputSerialisationConfigPath);
 
         String pipelineId = modelBuilder.id(config.isStrictUniqueIds());
         String upstreamPipelineId = modelBuilder.upstreamId(config.isStrictUniqueIds());
 
         return new PipelineModel(pipelineId,
                 name,
                 new PipelineModel.Transform(modelBuilder.getTransformType(), modelBuilder.getFunction().getName(), inputType, outputType),
-                upstreamPipelineId, outputSerialisation);
+                upstreamPipelineId,
+                inputSerialisation,
+                outputSerialisation);
     }
 
+    private PipelineModel.Serialisation getSerialisation(String xmlConfigPath) {
+        return xmlConfigPath == null ? null :
+                new PipelineModel.Serialisation(PipelineModel.Serialisation.Format.XML, xmlConfigPath);
+    }
 }

src/main/java/com/regnosys/testing/pipeline/PipelineTestPackWriter.java

@@ -87,7 +87,7 @@ public void writeTestPacks(PipelineTreeConfig config) throws IOException {
             Path outputPath = resourcesPath.resolve(pipelineNode.getOutputPath(config.isStrictUniqueIds()));
             LOGGER.info("Output path {} ", outputPath);
 
-            List<Path> inputSamples = findAllJsonSamples(inputPath);
+            List<Path> inputSamples = findAllSamples(inputPath);
 
             Map<String, List<Path>> testPackToSamples =
                     filterAndGroupingByTestPackId(resourcesPath, inputPath, inputSamples, config.getTestPackIdFilter());
@@ -108,14 +108,13 @@ public void writeTestPacks(PipelineTreeConfig config) throws IOException {
         }
     }
 
-    private List<Path> findAllJsonSamples(Path inputDir) throws IOException {
+    private List<Path> findAllSamples(Path inputDir) throws IOException {
         if (!Files.exists(inputDir)) {
             return List.of();
         }
         try (Stream<Path> paths = Files.walk(inputDir)) {
             return paths.filter(Files::isRegularFile)
                     .filter(Files::exists)
-                    .filter(x -> x.getFileName().toString().endsWith(".json"))
                     .collect(Collectors.toList());
         }
     }
@@ -135,8 +134,8 @@ private TestPackModel writeTestPackSamples(Path resourcesPath, Path inputPath, P
             Path outputSample = resourcesPath.relativize(outputDir.resolve(resourcesPath.relativize(inputPath).relativize(inputSample)));
             outputSample = outputSample.getParent().resolve(Path.of(updateFileExtensionBasedOnOutputFormat(pipeline, outputSample.toFile().getName())));
 
-            PipelineFunctionRunner.Result run = pipelineFunctionRunner.run(pipeline, config.getXmlSchemaMap(), resourcesPath.resolve(inputSample));
-            TestPackModel.SampleModel.Assertions assertions = run.getAssertions();
+            PipelineFunctionRunner.Result result = pipelineFunctionRunner.run(pipeline, config.getXmlSchemaMap(), resourcesPath.resolve(inputSample));
+            TestPackModel.SampleModel.Assertions assertions = result.getAssertions();
 
             String baseFileName = getBaseFileName(inputSample.toUri().toURL());
             String displayName = baseFileName.replace("-", " ");
@@ -145,7 +144,7 @@ private TestPackModel writeTestPackSamples(Path resourcesPath, Path inputPath, P
             sampleModels.add(sampleModel);
 
             Files.createDirectories(resourcesPath.resolve(outputSample).getParent());
-            Files.write(resourcesPath.resolve(outputSample), run.getSerialisedOutput().getBytes());
+            Files.write(resourcesPath.resolve(outputSample), result.getSerialisedOutput().getBytes());
         }
 
         List<TestPackModel.SampleModel> sortedSamples = sampleModels
@@ -160,11 +159,12 @@ private TestPackModel writeTestPackSamples(Path resourcesPath, Path inputPath, P
     }
 
     private String updateFileExtensionBasedOnOutputFormat(PipelineModel pipelineModel, String fileName) {
-        if (pipelineModel.getOutputSerialisation() != null) {
-            String outputFormat = pipelineModel.getOutputSerialisation().getFormat().toString().toLowerCase();
-            return fileName.substring(0, fileName.lastIndexOf(".")) + "." + outputFormat;
-        }
-        return fileName;
+        String outputFormat = Optional.ofNullable(pipelineModel.getOutputSerialisation())
+                .map(PipelineModel.Serialisation::getFormat)
+                .map(PipelineModel.Serialisation.Format::toString)
+                .orElse("json")
+                .toLowerCase();
+        return fileName.substring(0, fileName.lastIndexOf(".")) + "." + outputFormat;
     }
 
     private Map<String, List<Path>> filterAndGroupingByTestPackId(Path resourcesPath, Path inputPath, List<Path> inputSamples, Predicate<String> testPackIdFilter) {

src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerImpl.java

@@ -21,6 +21,7 @@
  */
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.regnosys.rosetta.common.hashing.ReferenceConfig;
 import com.regnosys.rosetta.common.hashing.ReferenceResolverProcessStep;
@@ -46,7 +47,7 @@
 
 import static com.regnosys.rosetta.common.transform.TestPackModel.SampleModel.Assertions;
 import static com.regnosys.rosetta.common.transform.TestPackUtils.readFile;
-import static com.regnosys.testing.testpack.TestPackFunctionRunnerProviderImpl.JSON_OBJECT_MAPPER;
+import static com.regnosys.testing.transform.TransformTestExtension.ERROR_OUTPUT;
 
 public class TestPackFunctionRunnerImpl<IN extends RosettaModelObject> implements TestPackFunctionRunner {
     private static final Logger LOGGER = LoggerFactory.getLogger(TestPackFunctionRunnerImpl.class);
@@ -56,6 +57,7 @@ public class TestPackFunctionRunnerImpl<IN extends RosettaModelObject> implement
     private final Class<IN> inputType;
     private final RosettaTypeValidator typeValidator;
     private final ReferenceConfig referenceConfig;
+    private final ObjectMapper inputObjectMapper;
     private final ObjectWriter outputObjectWriter;
     private final Validator xsdValidator;
 
@@ -64,12 +66,14 @@ public TestPackFunctionRunnerImpl(Function<IN, RosettaModelObject> function,
                                       Class<IN> inputType,
                                       RosettaTypeValidator typeValidator,
                                       ReferenceConfig referenceConfig,
+                                      ObjectMapper inputObjectMapper,
                                       ObjectWriter outputObjectWriter,
                                       Validator xsdValidator) {
         this.function = function;
         this.inputType = inputType;
         this.typeValidator = typeValidator;
         this.referenceConfig = referenceConfig;
+        this.inputObjectMapper = inputObjectMapper;
         this.outputObjectWriter = outputObjectWriter;
         this.xsdValidator = xsdValidator;
     }
@@ -81,14 +85,14 @@ public Pair<String, Assertions> run(Path inputPath) {
             // TODO - fix this hack.
             Path inputPathFromRepositoryRoot = inputPath.isAbsolute() ? inputPath : ROSETTA_SOURCE_PATH.resolve(inputPath);
             URL inputFileUrl = inputPathFromRepositoryRoot.toUri().toURL();
-            IN input = readFile(inputFileUrl, JSON_OBJECT_MAPPER, inputType);
+            IN input = readFile(inputFileUrl, inputObjectMapper, inputType);
             output = function.apply(resolveReferences(input));
         } catch (MalformedURLException e) {
             LOGGER.error("Failed to load input path {}", inputPath, e);
-            return Pair.of(null, new Assertions(null, null, true));
+            return Pair.of(ERROR_OUTPUT, new Assertions(null, null, true));
         } catch (Exception e) {
             LOGGER.error("Exception occurred running sample creation", e);
-            return Pair.of(null, new Assertions(null, null, true));
+            return Pair.of(ERROR_OUTPUT, new Assertions(null, null, true));
         }
 
         String serialisedOutput;
@@ -108,6 +112,7 @@ public Pair<String, Assertions> run(Path inputPath) {
         return Pair.of(serialisedOutput, assertions);
     }
 
+    @SuppressWarnings("unchecked")
     private <T extends RosettaModelObject> T resolveReferences(T o) {
         RosettaModelObjectBuilder builder = o.toBuilder();
         new ReferenceResolverProcessStep(referenceConfig).runProcessStep(o.getType(), builder);

src/main/java/com/regnosys/testing/testpack/TestPackFunctionRunnerProvider.java

@@ -30,7 +30,9 @@
 @ImplementedBy(TestPackFunctionRunnerProviderImpl.class)
 public interface TestPackFunctionRunnerProvider {
 
-    TestPackFunctionRunner create(Transform transform, Injector injector);
-
-    TestPackFunctionRunner create(Transform transform, Serialisation outputSerialisation, ImmutableMap<Class<?>, String> outputSchemaMap, Injector injector);
+    TestPackFunctionRunner create(Transform transform,
+                                  Serialisation inputSerialisation,
+                                  Serialisation outputSerialisation,
+                                  ImmutableMap<Class<?>, String> outputSchemaMap,
+                                  Injector injector);
 }