Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into add-ingest-transform-…
Browse files Browse the repository at this point in the history 
…type
  • Loading branch information
@hugohills-regnosys
hugohills-regnosys committed Nov 18, 2024
2 parents f78330e + 3be9c8f commit e45b5ce
Show file tree
Hide file tree
Showing 6 changed files with 59 additions and 93 deletions.
39 changes: 39 additions & 0 deletions src/main/java/com/regnosys/testing/pipeline/PipelineFilter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package com.regnosys.testing.pipeline;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.BiPredicate;
import java.util.function.Predicate;

public class PipelineFilter implements Predicate<String> {

private final List<String> items;
private final BiPredicate<String, String> predicate;

private PipelineFilter(List<String> items, BiPredicate<String, String> predicate) {
this.items = items;
this.predicate = predicate;
}

public boolean test(String item) {
return items.stream().anyMatch(x -> predicate.test(item, x));
}

public static Predicate<String> startsWith(String... startsFilter) {
return new PipelineFilter(Arrays.asList(startsFilter), String::startsWith);
}

public static Predicate<String> contains(String... startsFilter) {
return new PipelineFilter(Arrays.asList(startsFilter), String::contains);
}

public static Predicate<String> equalsTo(String... startsFilter) {
return new PipelineFilter(Arrays.asList(startsFilter), String::equals);
}

public static Predicate<String> startsWith(List<String> startsFilterDefaults, String... startsFilter) {
Collections.addAll(startsFilterDefaults, startsFilter);
return new PipelineFilter(startsFilterDefaults, String::startsWith);
}
}
1 change: 1 addition & 0 deletions src/main/java/com/regnosys/testing/pipeline/PipelineTestPackFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.Collections;
import java.util.List;

@Deprecated
public class PipelineTestPackFilter {

public static PipelineTestPackFilter create() {
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/com/regnosys/testing/pipeline/PipelineTestPackWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public void writeTestPacks(PipelineTreeConfig config) throws IOException {
List<Path> inputSamples = findAllSamples(inputPath);

Map<String, List<Path>> testPackToSamples =
filterAndGroupingByTestPackId(resourcesPath, inputPath, inputSamples, config.getTestPackIdInclusionFilter());
filterAndGroupingByTestPackId(resourcesPath, inputPath, inputSamples, config.getTestPackIdFilter());

Map<String, List<Path>> filteredTestPackToSamples = Optional.ofNullable(pipelineTestPackFilter)
.map(t -> filterTestPacks(pipelineNode, pipelineTestPackFilter, testPackToSamples)).orElse(testPackToSamples);
Expand Down Expand Up @@ -167,10 +167,10 @@ private String updateFileExtensionBasedOnOutputFormat(PipelineModel pipelineMode
return fileName.substring(0, fileName.lastIndexOf(".")) + "." + outputFormat;
}

private Map<String, List<Path>> filterAndGroupingByTestPackId(Path resourcesPath, Path inputPath, List<Path> inputSamples, Predicate<String> testPackSampleInclusionFilter) {
private Map<String, List<Path>> filterAndGroupingByTestPackId(Path resourcesPath, Path inputPath, List<Path> inputSamples, Predicate<String> testPackIdFilter) {
return inputSamples.stream()
.map(resourcesPath::relativize)
.filter(path -> testPackSampleInclusionFilter.test(testPackId(resourcesPath, inputPath, path)))
.filter(path -> testPackIdFilter.test(testPackId(resourcesPath, inputPath, path)))
.collect(Collectors.groupingBy(p -> testPackId(resourcesPath, inputPath, p)));
}

Expand Down
10 changes: 5 additions & 5 deletions src/main/java/com/regnosys/testing/pipeline/PipelineTreeConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public class PipelineTreeConfig {
private final Multimap<Class<? extends RosettaFunction>, TransformFunction> conf = ArrayListMultimap.create();
private boolean strictUniqueIds;
private Path writePath;
private Predicate<String> testPackIdInclusionFilter = testPackId -> true;
private Predicate<String> testPackIdFilter = testPackId -> true;

public PipelineTreeConfig starting(TransformType transformType, Class<? extends RosettaFunction> function) {
starting.add(new TransformFunction(function, transformType));
Expand All @@ -62,8 +62,8 @@ public PipelineTreeConfig add(Class<? extends RosettaFunction> upstreamFunction,
return this;
}

public PipelineTreeConfig withTestPackIdInclusionFilter(Predicate<String> testPackIdInclusionFilter) {
this.testPackIdInclusionFilter = testPackIdInclusionFilter;
public PipelineTreeConfig withTestPackIdFilter(Predicate<String> testPackIdFilter) {
this.testPackIdFilter = testPackIdFilter;
return this;
}

Expand Down Expand Up @@ -107,8 +107,8 @@ public PipelineTreeConfig withTestPackFilter(PipelineTestPackFilter pipelineTest
return this;
}

public Predicate<String> getTestPackIdInclusionFilter() {
return testPackIdInclusionFilter;
public Predicate<String> getTestPackIdFilter() {
return testPackIdFilter;
}

PipelineTestPackFilter getTestPackFilter() {
Expand Down
59 changes: 0 additions & 59 deletions src/test/java/com/regnosys/testing/pipeline/PipelineTestHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import javax.inject.Inject;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;

public class PipelineTestHelper {

Expand All @@ -55,9 +54,6 @@ public class PipelineTestHelper {

private Path testPath;
private CompiledCode compiledCode;
private ImmutableMultimap<String, Class<?>> TEST_PACKS_RESTRICTED_FUNCTIONS;
private ImmutableMultimap<String, Class<?>> TEST_PACKS_SPECIFIC_TO_FUNCTIONS;
private ImmutableMultimap<Class<?>, String> FUNCTIONS_SPECIFIC_TO_TEST_PACKS;

static void setupInjector(Object caller) {
Injector injector = new RosettaTestingInjectorProvider().getInjector();
Expand All @@ -75,26 +71,6 @@ protected void configure() {
void init() throws Exception {
testPath = Path.of("src/test/resources/pipeline-test");
compiledCode = compileCode();
//For these test packs, only run these functions
TEST_PACKS_RESTRICTED_FUNCTIONS =
ImmutableMultimap.<String, Class<?>>builder()
.put("test-pack-a", middleAClass())
.put("test-pack-b", middleBClass())
.build();
//For these functions, only these test packs, do not include the test pack for other functions
TEST_PACKS_SPECIFIC_TO_FUNCTIONS =
ImmutableMultimap.<String, Class<?>>builder()
.put("test-pack-only-c", middleCClass())
.build();
//For these functions, only these test packs, but include the test packs for other functions
FUNCTIONS_SPECIFIC_TO_TEST_PACKS =
ImmutableMultimap.<Class<?>, String>builder()
.put(middleDClass(), "test-pack-d")
.build();
}

Path testPath() {
return testPath;
}

Class<RosettaFunction> endBClass() {
Expand Down Expand Up @@ -164,41 +140,6 @@ PipelineTreeConfig createTreeConfigWithoutStarting() {
.add(middleClass(), TransformType.PROJECTION, endClass());
}

PipelineTreeConfig createTreeConfigForFilter() {
return new PipelineTreeConfig()
.starting(TransformType.REPORT, middleAClass())
.starting(TransformType.REPORT, middleBClass())
.starting(TransformType.REPORT, middleCClass())
.starting(TransformType.REPORT, middleDClass())
.starting(TransformType.REPORT, middleEClass());
}
PipelineTreeConfig createTreeConfigWithFilter() {
PipelineTestPackFilter filter = PipelineTestPackFilter.create()
.withTestPacksRestrictedForFunctions(TEST_PACKS_RESTRICTED_FUNCTIONS)
.withTestPacksSpecificToFunctions(TEST_PACKS_SPECIFIC_TO_FUNCTIONS)
.withFunctionsSpecificToTestPacks(FUNCTIONS_SPECIFIC_TO_TEST_PACKS)
.withExcludedFunctionsFromTestPackGeneration(List.of(middleEClass()));
return createTreeConfigForFilter().withTestPackFilter(filter);
}

PipelineTreeConfig createTreeConfigWithTestPackFilter() {
PipelineTestPackFilter filter = PipelineTestPackFilter.create()
.withTestPacksRestrictedForFunctions(TEST_PACKS_RESTRICTED_FUNCTIONS);
return createTreeConfigForFilter().withTestPackFilter(filter);
}

PipelineTreeConfig createTreeConfigWithFunctionFilter() {
PipelineTestPackFilter filter = PipelineTestPackFilter.create()
.withTestPacksSpecificToFunctions(TEST_PACKS_SPECIFIC_TO_FUNCTIONS);
return createTreeConfigForFilter().withTestPackFilter(filter);
}

PipelineTreeConfig createTreeConfigWithExcludedFilter() {
PipelineTestPackFilter filter = PipelineTestPackFilter.create()
.withExcludedFunctionsFromTestPackGeneration(List.of(middleDClass()));
return createTreeConfigForFilter().withTestPackFilter(filter);
}

private CompiledCode compileCode() throws Exception {
CompiledCode compiledCode = modelHelper.generateAndCompileJava(Files.readString(testPath.resolve("rosetta/pipelines.rosetta")));

Expand Down
37 changes: 11 additions & 26 deletions src/test/java/com/regnosys/testing/pipeline/PipelineTestPackWriterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.function.Predicate;

import static com.regnosys.testing.pipeline.PipelineFilter.startsWith;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

Expand Down Expand Up @@ -221,53 +223,36 @@ void writeTestPacksForTreeConfigWithFilter(@TempDir Path tempDir) throws Excepti
Path testPackAPath = Files.createDirectories(inputPath.resolve("test-pack-a"));
Path testPackBPath = Files.createDirectories(inputPath.resolve("test-pack-b"));
Path testPackCPath = Files.createDirectories(inputPath.resolve("test-pack-only-c"));
Path testPackDPath = Files.createDirectories(inputPath.resolve("test-pack-d"));

Files.write(testPack1Path.resolve("sample-1-1.json"), "{\"name\": \"1-1\"}".getBytes());
Files.write(testPack2Path.resolve("sample-1-1.json"), "{\"name\": \"1-1\"}".getBytes());
Files.write(testPackAPath.resolve("sample-1-1.json"), "{\"name\": \"1-1\"}".getBytes());
Files.write(testPackBPath.resolve("sample-1-1.json"), "{\"name\": \"1-1\"}".getBytes());
Files.write(testPackCPath.resolve("sample-1-1.json"), "{\"name\": \"1-1\"}".getBytes());
Files.write(testPackDPath.resolve("sample-1-1.json"), "{\"name\": \"1-1\"}".getBytes());

PipelineTreeConfig chain = helper.createTreeConfigWithFilter().
strictUniqueIds().
withWritePath(tempDir);
PipelineTreeConfig chain = new PipelineTreeConfig()
.withTestPackIdFilter(startsWith("test-pack-a", "test-pack-1", "test-pack-2").
and(Predicate.not(startsWith("test-pack-b", "test-pack-only-c"))))
.starting(TransformType.REPORT, helper.middleAClass())
.add(helper.middleAClass(), TransformType.PROJECTION, helper.endAClass())
.strictUniqueIds()
.withWritePath(tempDir);
pipelineTestPackWriter.writeTestPacks(chain);

assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-a-test-pack-1.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-a-test-pack-2.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-a-test-pack-a.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-a-test-pack-d.json");

assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-a-test-pack-b.json");
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-a-test-pack-only-c.json");

assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-b-test-pack-1.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-b-test-pack-2.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-b-test-pack-b.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-b-test-pack-d.json");

assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-b-test-pack-a.json");
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-b-test-pack-only-c.json");

assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-c-test-pack-only-c.json");
assertFileExists(tempDir, "regulatory-reporting/config/test-pack-report-middle-d-test-pack-d.json");

assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-e-test-pack-1.json");
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-e-test-pack-2.json");
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-e-test-pack-a.json");
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-e-test-pack-b.json");
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-e-test-pack-only-c.json");
assertFileDoesNotExist(tempDir, "regulatory-reporting/config/test-pack-report-middle-e-test-pack-d.json");
}


@Test
void writeSegregatedTestPacksWithFilter(@TempDir Path tempDir) throws Exception {
// Trade Report Pipeline, startClass == TradeInstruction, middleClass == TradeReport, endClass == TradeProject
PipelineTreeConfig tradeReportConf = new PipelineTreeConfig()
.withTestPackIdInclusionFilter(testPackId -> testPackId.startsWith("trade"))
.withTestPackIdFilter(startsWith("trade"))
.starting(TransformType.ENRICH, helper.startClass())
.add(helper.startClass(), TransformType.REPORT, helper.middleClass())
.add(helper.middleClass(), TransformType.PROJECTION, helper.endClass())
Expand All @@ -281,7 +266,7 @@ void writeSegregatedTestPacksWithFilter(@TempDir Path tempDir) throws Exception

// Valuation Report Pipeline, middleBClass == ValuationReport. endBClass == ValuationProject
PipelineTreeConfig valuationReportConf = new PipelineTreeConfig()
.withTestPackIdInclusionFilter(testPackId -> testPackId.startsWith("valuation"))
.withTestPackIdFilter(startsWith("valuation"))
.starting(TransformType.REPORT, helper.middleBClass())
.add(helper.middleBClass(), TransformType.PROJECTION, helper.endBClass())
.strictUniqueIds().withWritePath(tempDir);
Expand Down

0 comments on commit e45b5ce

Please sign in to comment.