Skip to content

Commit

Permalink
Added test pack id filtering (#185)
Browse files Browse the repository at this point in the history
  • Loading branch information
@minesh-s-patel
minesh-s-patel authored Oct 23, 2024
1 parent e34d108 commit 1eeaf49
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 10 deletions.
24 changes: 15 additions & 9 deletions src/main/java/com/regnosys/testing/pipeline/PipelineTestPackWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -86,10 +87,10 @@ public void writeTestPacks(PipelineTreeConfig config) throws IOException {
Path outputPath = resourcesPath.resolve(pipelineNode.getOutputPath(config.isStrictUniqueIds()));
LOGGER.info("Output path {} ", outputPath);

List<Path> inputSamples = inputSamples(inputPath);
List<Path> inputSamples = findAllJsonSamples(inputPath);

Map<String, List<Path>> testPackToSamples =
groupingByTestPackId(resourcesPath, inputPath, inputSamples);
filterAndGroupingByTestPackId(resourcesPath, inputPath, inputSamples, config.getTestPackIdInclusionFilter());

Map<String, List<Path>> filteredTestPackToSamples = Optional.ofNullable(pipelineTestPackFilter)
.map(t -> filterTestPacks(pipelineNode, pipelineTestPackFilter, testPackToSamples)).orElse(testPackToSamples);
Expand All @@ -107,10 +108,14 @@ public void writeTestPacks(PipelineTreeConfig config) throws IOException {
}
}

private List<Path> inputSamples(Path inputDir) throws IOException {
private List<Path> findAllJsonSamples(Path inputDir) throws IOException {
if (!Files.exists(inputDir)) {
return List.of();
}
try (Stream<Path> paths = Files.walk(inputDir)) {
return paths.filter(Files::isRegularFile)
.filter(Files::exists)
.filter(x -> x.getFileName().toString().endsWith(".json"))
.collect(Collectors.toList());
}
}
Expand Down Expand Up @@ -154,30 +159,31 @@ private TestPackModel writeTestPackSamples(Path resourcesPath, Path inputPath, P
return new TestPackModel(String.format("test-pack-%s-%s-%s", transformType.name().toLowerCase(), pipelineIdSuffix, testPackId), pipelineId, testPackName, sortedSamples);
}

private String updateFileExtensionBasedOnOutputFormat(PipelineModel pipelineModel, String fileName){
if(pipelineModel.getOutputSerialisation() != null) {
private String updateFileExtensionBasedOnOutputFormat(PipelineModel pipelineModel, String fileName) {
if (pipelineModel.getOutputSerialisation() != null) {
String outputFormat = pipelineModel.getOutputSerialisation().getFormat().toString().toLowerCase();
return fileName.substring(0, fileName.lastIndexOf(".")) + "." + outputFormat;
}
return fileName;
}

private Map<String, List<Path>> groupingByTestPackId(Path resourcesPath, Path inputPath, List<Path> inputSamples) {
private Map<String, List<Path>> filterAndGroupingByTestPackId(Path resourcesPath, Path inputPath, List<Path> inputSamples, Predicate<String> testPackSampleInclusionFilter) {
return inputSamples.stream()
.map(resourcesPath::relativize)
.filter(path -> testPackSampleInclusionFilter.test(testPackId(resourcesPath, inputPath, path)))
.collect(Collectors.groupingBy(p -> testPackId(resourcesPath, inputPath, p)));
}

private String testPackId(Path resourcesPath, Path inputPath, Path samplePath) {
Path parent = samplePath.getParent();
Path relativePath = resourcesPath.relativize(inputPath).relativize(parent);
return relativePath.toString().replace(File.pathSeparatorChar, '-');
return relativePath.toString().replace(File.separatorChar, '-');
}

private @NotNull Map<String, List<Path>> filterTestPacks(PipelineNode pipelineNode, PipelineTestPackFilter pipelineTestPackFilter, Map<String, List<Path>> testPackToSamples) {
Map<String, List<Path>> filteredTestPackToSamples = testPackToSamples;
final Set<String> testPackSpecificFunctions = pipelineTestPackFilter.getTestPacksSpecificToFunctions().entries()
.stream().filter(entry -> entry.getValue() == pipelineNode.getFunction()) .map(Map.Entry::getKey)
final Set<String> testPackSpecificFunctions = pipelineTestPackFilter.getTestPacksSpecificToFunctions().entries()
.stream().filter(entry -> entry.getValue() == pipelineNode.getFunction()).map(Map.Entry::getKey)
.collect(Collectors.toSet());

if (pipelineTestPackFilter.getTestPacksSpecificToFunctions().containsValue(pipelineNode.getFunction())) {
Expand Down
13 changes: 12 additions & 1 deletion src/main/java/com/regnosys/testing/pipeline/PipelineTreeConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;
import java.util.stream.Collectors;

public class PipelineTreeConfig {
Expand All @@ -43,6 +44,7 @@ public class PipelineTreeConfig {
private final Multimap<Class<? extends RosettaFunction>, TransformFunction> conf = ArrayListMultimap.create();
private boolean strictUniqueIds;
private Path writePath;
private Predicate<String> testPackIdInclusionFilter = testPackId -> true;

public PipelineTreeConfig starting(TransformType transformType, Class<? extends RosettaFunction> function) {
starting.add(new TransformFunction(function, transformType));
Expand All @@ -59,7 +61,12 @@ public PipelineTreeConfig add(Class<? extends RosettaFunction> upstreamFunction,
conf.put(upstreamFunction, current);
return this;
}


public PipelineTreeConfig withTestPackIdInclusionFilter(Predicate<String> testPackIdInclusionFilter) {
this.testPackIdInclusionFilter = testPackIdInclusionFilter;
return this;
}

public PipelineTreeConfig withXmlConfigMap(ImmutableMap<Class<?>, String> xmlConfigMap) {
this.xmlConfigMap = xmlConfigMap;
return this;
Expand Down Expand Up @@ -99,6 +106,10 @@ public PipelineTreeConfig withTestPackFilter(PipelineTestPackFilter pipelineTest
this.pipelineTestPackFilter = pipelineTestPackFilter;
return this;
}

public Predicate<String> getTestPackIdInclusionFilter() {
return testPackIdInclusionFilter;
}

PipelineTestPackFilter getTestPackFilter() {
return pipelineTestPackFilter;
Expand Down
117 changes: 117 additions & 0 deletions src/test/java/com/regnosys/testing/pipeline/PipelineTestPackWriterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,123 @@ void writeTestPacksForTreeConfigWithFilter(@TempDir Path tempDir) throws Excepti
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-e-test-pack-d.json");
}


@Test
void writeSegregatedTestPacksWithFilter(@TempDir Path tempDir) throws Exception {
// Trade Report Pipeline, startClass == TradeInstruction, middleClass == TradeReport, endClass == TradeProject
PipelineTreeConfig tradeReportConf = new PipelineTreeConfig()
.withTestPackIdInclusionFilter(testPackId -> testPackId.startsWith("trade"))
.starting(TransformType.ENRICH, helper.startClass())
.add(helper.startClass(), TransformType.REPORT, helper.middleClass())
.add(helper.middleClass(), TransformType.PROJECTION, helper.endClass())
.strictUniqueIds().withWritePath(tempDir);

Path enrichInputPath = Files.createDirectories(tempDir.resolve(TransformType.ENRICH.getResourcePath()).resolve("input"));

Path tradeTestPack1Path = Files.createDirectories(enrichInputPath.resolve("trade").resolve("t-pack-1"));
Files.write(tradeTestPack1Path.resolve("t-1-1.json"), "{\"name\": \"1-1t\"}".getBytes());
Files.write(tradeTestPack1Path.resolve("t-1-2.json"), "{\"name\": \"1-2t\"}".getBytes());

// Valuation Report Pipeline, middleBClass == ValuationReport. endBClass == ValuationProject
PipelineTreeConfig valuationReportConf = new PipelineTreeConfig()
.withTestPackIdInclusionFilter(testPackId -> testPackId.startsWith("valuation"))
.starting(TransformType.REPORT, helper.middleBClass())
.add(helper.middleBClass(), TransformType.PROJECTION, helper.endBClass())
.strictUniqueIds().withWritePath(tempDir);

Path reportInputPath = Files.createDirectories(tempDir.resolve(TransformType.REPORT.getResourcePath()).resolve("input"));
Path valuationTestPack1Path = Files.createDirectories(reportInputPath.resolve("valuation").resolve("v-pack-1"));

Files.write(valuationTestPack1Path.resolve("v-1-1.json"), "{\"name\": \"1-1v\"}".getBytes());
Files.write(valuationTestPack1Path.resolve("v-1-2.json"), "{\"name\": \"1-2v\"}".getBytes());

pipelineTestPackWriter.writeTestPacks(tradeReportConf);
pipelineTestPackWriter.writeTestPacks(valuationReportConf);

assertFileExists(tempDir, "enrich/input/trade/t-pack-1/t-1-1.json");
assertFileExists(tempDir, "enrich/input/trade/t-pack-1/t-1-2.json");

assertFileExists(tempDir, "enrich/config/test-pack-enrich-start-trade-t-pack-1.json");
assertFileExists(tempDir, "enrich/output/start/trade/t-pack-1/t-1-1.json");
assertFileExists(tempDir, "enrich/output/start/trade/t-pack-1/t-1-2.json");

assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-start-middle-trade-t-pack-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle/trade/t-pack-1/t-1-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle/trade/t-pack-1/t-1-2.json");

assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-end-trade-t-pack-1.json");
assertFileExists(tempDir, "projection/output/start/middle/end/trade/t-pack-1/t-1-1.json");
assertFileExists(tempDir, "projection/output/start/middle/end/trade/t-pack-1/t-1-2.json");

assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-b-valuation-v-pack-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/middle-b/valuation/v-pack-1/v-1-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/middle-b/valuation/v-pack-1/v-1-2.json");

assertFileExists(tempDir, "projection/config/test-pack-projection-middle-b-end-b-valuation-v-pack-1.json");
assertFileExists(tempDir, "projection/output/middle-b/end-b/valuation/v-pack-1/v-1-1.json");
assertFileExists(tempDir, "projection/output/middle-b/end-b/valuation/v-pack-1/v-1-2.json");
}

@Test
void writeTestPacksForNestedTreeConfigNestedFolders(@TempDir Path tempDir) throws Exception {
Path inputPath = Files.createDirectories(tempDir.resolve(TransformType.ENRICH.getResourcePath()).resolve("input"));

Path testPack1Path = Files.createDirectories(inputPath.resolve("test-pack").resolve("1"));
Path testPack2Path = Files.createDirectories(inputPath.resolve("test-pack").resolve("2"));

Files.write(testPack1Path.resolve("sample-1-1.json"), "{\"name\": \"1-1\"}".getBytes());
Files.write(testPack1Path.resolve("sample-1-2.json"), "{\"name\": \"1-2\"}".getBytes());
Files.write(testPack2Path.resolve("sample-2-1.json"), "{\"name\": \"2-1\"}".getBytes());
Files.write(testPack2Path.resolve("sample-2-2.json"), "{\"name\": \"2-2\"}".getBytes());

PipelineTreeConfig chain = helper.createNestedTreeConfig().strictUniqueIds().withWritePath(tempDir);
pipelineTestPackWriter.writeTestPacks(chain);

assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-start-middle-a-test-pack-2.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-start-middle-b-test-pack-1.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-start-middle-b-test-pack-2.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-start-middle-a-test-pack-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-b/test-pack/2/sample-2-2.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-b/test-pack/2/sample-2-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-b/test-pack/1/sample-1-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-b/test-pack/1/sample-1-2.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-a/test-pack/2/sample-2-2.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-a/test-pack/2/sample-2-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-a/test-pack/1/sample-1-1.json");
assertFileExists(tempDir, "regulatory-reporting/output/start/middle-a/test-pack/1/sample-1-2.json");
assertFileExists(tempDir, "enrich/config/test-pack-enrich-start-test-pack-1.json");
assertFileExists(tempDir, "enrich/config/test-pack-enrich-start-test-pack-2.json");
assertFileExists(tempDir, "enrich/output/start/test-pack/2/sample-2-2.json");
assertFileExists(tempDir, "enrich/output/start/test-pack/2/sample-2-1.json");
assertFileExists(tempDir, "enrich/output/start/test-pack/1/sample-1-1.json");
assertFileExists(tempDir, "enrich/output/start/test-pack/1/sample-1-2.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-a-end-a-test-pack-1.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-b-end-b-test-pack-2.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-a-end-b-test-pack-2.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-b-end-a-test-pack-1.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-b-end-a-test-pack-2.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-a-end-b-test-pack-1.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-b-end-b-test-pack-1.json");
assertFileExists(tempDir, "projection/config/test-pack-projection-start-middle-a-end-a-test-pack-2.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-a/test-pack/2/sample-2-2.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-a/test-pack/2/sample-2-1.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-a/test-pack/1/sample-1-1.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-a/test-pack/1/sample-1-2.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-b/test-pack/2/sample-2-2.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-b/test-pack/2/sample-2-1.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-b/test-pack/1/sample-1-1.json");
assertFileExists(tempDir, "projection/output/start/middle-b/end-b/test-pack/1/sample-1-2.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-a/test-pack/2/sample-2-2.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-a/test-pack/2/sample-2-1.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-a/test-pack/1/sample-1-1.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-a/test-pack/1/sample-1-2.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-b/test-pack/2/sample-2-2.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-b/test-pack/2/sample-2-1.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-b/test-pack/1/sample-1-1.json");
assertFileExists(tempDir, "projection/output/start/middle-a/end-b/test-pack/1/sample-1-2.json");
}


private static void assertFileExists(Path tempDir, String fileName) {
assertTrue(Files.exists(tempDir.resolve(fileName)), String.format("File not found %s", fileName));
}
Expand Down

0 comments on commit 1eeaf49

Please sign in to comment.