Skip to content

Commit

Permalink
Merge pull request #50 from ao508/generic-comparator
Browse files Browse the repository at this point in the history 
Added generic JSON comparator.
  • Loading branch information
@ao508
ao508 authored Mar 12, 2024
2 parents 46c9375 + 85438a7 commit 2366ca3
Show file tree
Hide file tree
Showing 23 changed files with 294 additions and 43 deletions.
1 change: 1 addition & 0 deletions src/main/java/org/mskcc/smile/commons/JsonComparator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ public interface JsonComparator {
Boolean isConsistent(String referenceJson, String targetJson) throws Exception;
Boolean isConsistent(String referenceJson, String targetJson, String[] ignoredFields,
String comparisonType) throws Exception;
Boolean isConsistentGenericComparison(String referenceJson, String targetJson) throws Exception;
}
58 changes: 56 additions & 2 deletions src/main/java/org/mskcc/smile/commons/impl/JsonComparatorImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ public class JsonComparatorImpl implements JsonComparator {
"sampleAliases",
"genePanel",
"additionalProperties",
"cmoInfoIgoId"};
"cmoInfoIgoId",
"date"};

public final String[] IGO_ACCEPTED_FIELDS = new String[]{
//RequestMetadata fields
Expand Down Expand Up @@ -89,6 +90,11 @@ public class JsonComparatorImpl implements JsonComparator {
"runs"
};

public final String[] GENERIC_IGNORED_FIELDS = new String[]{
"date",
"samples"
};

private final Map<String, String> STD_IGO_REQUEST_JSON_PROPS_MAP =
initStandardizedIgoRequestJsonPropsMap();
private final Map<String, String> STD_IGO_SAMPLE_JSON_PROPS_MAP =
Expand Down Expand Up @@ -185,6 +191,17 @@ public Boolean isConsistent(String referenceJson, String targetJson, String[] ig
return consistencyCheckStatus;
}

@Override
public Boolean isConsistentGenericComparison(String referenceJson, String targetJson) throws Exception {
Boolean consistencyCheckStatus = Boolean.TRUE;
String filteredReferenceJson = filterGenericJson(referenceJson);
String filteredTargetJson = filterGenericJson(targetJson);
if (!isMatchingJsons(filteredReferenceJson, filteredTargetJson)) {
consistencyCheckStatus = Boolean.FALSE;
}
return consistencyCheckStatus;
}

private Boolean isConsistentSampleMetadata(String referenceJson, String targetJson,
String comparisonType) throws JsonProcessingException {
return isConsistentSampleMetadata(mapper.readTree(referenceJson),
Expand Down Expand Up @@ -325,6 +342,19 @@ private String standardizeAndFilterRequestJson(String jsonString, String[] ignor
return mapper.writeValueAsString(stdFilteredJsonNode);
}

/**
* Given an input jsonString, returns a JSON with (1) generic fields ignored and (2) null
* or empty values removed.
* @param jsonString
* @return String
* @throws JsonProcessingException
*/
private String filterGenericJson(String jsonString) throws JsonProcessingException {
JsonNode unfilteredJsonNode = mapper.readTree(jsonString);
JsonNode filteredJsonNode = filterGenericJsonNode((ObjectNode) unfilteredJsonNode);
return mapper.writeValueAsString(filteredJsonNode);
}

/**
* Given an input jsonString and an array of ignoredFields, returns a JSON
* with (1) the fields to ignore removed, (2) json fields with null or empty values
Expand Down Expand Up @@ -410,6 +440,30 @@ private Boolean isMatchingJsons(String referenceJson, String targetJson) {
return Boolean.TRUE;
}

private JsonNode filterGenericJsonNode(ObjectNode node) throws JsonProcessingException {
List<String> fieldsToRemove = new ArrayList<>();
fieldsToRemove.addAll(Arrays.asList(GENERIC_IGNORED_FIELDS));

Iterator<String> itr = node.fieldNames();
while (itr.hasNext()) {
String field = itr.next();
String value = node.get(field).toString();

if (Strings.isNullOrEmpty(value) || value.equalsIgnoreCase("null")
|| value.equalsIgnoreCase("[]")) {
fieldsToRemove.add(field);
}
}
// remove compiled fields to remove list from input node
// and return cleaned up node
if (!fieldsToRemove.isEmpty()) {
for (String field : fieldsToRemove) {
node.remove(field);
}
}
return node;
}

/**
* Returns a JsonNode with the given json properties matching 'ignoredFields'
* removed from the node as well as any properties with null or empty values.
Expand Down Expand Up @@ -502,7 +556,7 @@ private JsonNode filterArrayNodeChildrenByList(String parentNode, String compari
}
return mapper.readTree(mapper.writeValueAsString(filteredParentNode));
}

/**
* Given a parent node as a string, returns a filtered JsonNode.
* This is special case handling specific to 'status' and other
Expand Down
84 changes: 50 additions & 34 deletions src/test/java/org/mskcc/smile/commons/JsonComparatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ private void initRequestJsonDataIdMap() {
}

@Autowired
private Map<String, MockJsonTestData> mockedRequestJsonDataMap;
private Map<String, MockJsonTestData> mockedJsonDataMap;

/**
* Simple test to assert failure when request ids are different.
Expand All @@ -57,11 +57,11 @@ public void testConsistencyCheckFailure() throws Exception {
*/
@Test
public void testMockedRequestJsonDataLoading() {
Assert.assertNotNull(mockedRequestJsonDataMap);
Assert.assertNotNull(mockedJsonDataMap);

for (Map.Entry<String, String> entry : requestJsonDataIdMap.entrySet()) {
Assert.assertTrue(mockedRequestJsonDataMap.containsKey(entry.getKey()));
Assert.assertTrue(mockedRequestJsonDataMap.containsKey(entry.getValue()));
Assert.assertTrue(mockedJsonDataMap.containsKey(entry.getKey()));
Assert.assertTrue(mockedJsonDataMap.containsKey(entry.getValue()));
}
}

Expand All @@ -76,8 +76,8 @@ public void testAllRequestJsonsForConsistency() throws Exception {
for (Map.Entry<String, String> entry : requestJsonDataIdMap.entrySet()) {
String incomingRequestId = entry.getKey();
String publishedRequestId = entry.getValue();
MockJsonTestData incomingRequest = mockedRequestJsonDataMap.get(incomingRequestId);
MockJsonTestData publishedRequest = mockedRequestJsonDataMap.get(publishedRequestId);
MockJsonTestData incomingRequest = mockedJsonDataMap.get(incomingRequestId);
MockJsonTestData publishedRequest = mockedJsonDataMap.get(publishedRequestId);

try {
Boolean consistencyCheckStatus = jsonComparator.isConsistent(
Expand All @@ -103,9 +103,9 @@ public void testAllRequestJsonsForConsistency() throws Exception {
@Test
public void testNullJsonFieldHandlingInPublishedRequest() throws Exception {
MockJsonTestData incomingRequest =
mockedRequestJsonDataMap.get("mockIncomingRequest1JsonDataWith2T2N");
mockedJsonDataMap.get("mockIncomingRequest1JsonDataWith2T2N");
MockJsonTestData publishedRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonNullValues");
mockedJsonDataMap.get("mockPublishedRequest1JsonNullValues");
Boolean consistencyCheckStatus = jsonComparator.isConsistent(
incomingRequest.getJsonString(), publishedRequest.getJsonString());
Assert.assertFalse(consistencyCheckStatus);
Expand All @@ -121,9 +121,9 @@ public void testNullJsonFieldHandlingInPublishedRequest() throws Exception {
@Test
public void testNullOrEmptyJsonFieldHandlingInIncomingAndPublishedRequest() throws Exception {
MockJsonTestData incomingRequest =
mockedRequestJsonDataMap.get("mockIncomingRequest4JsonNullOrEmptyValues");
mockedJsonDataMap.get("mockIncomingRequest4JsonNullOrEmptyValues");
MockJsonTestData publishedRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest4JsonNullOrEmptyValues");
mockedJsonDataMap.get("mockPublishedRequest4JsonNullOrEmptyValues");
Boolean consistencyCheckStatus = jsonComparator.isConsistent(
incomingRequest.getJsonString(), publishedRequest.getJsonString());
Assert.assertTrue(consistencyCheckStatus);
Expand All @@ -136,10 +136,10 @@ public void testNullOrEmptyJsonFieldHandlingInIncomingAndPublishedRequest() thro
@Test
public void testPublishedSampleAliasesIgnored() throws Exception {
MockJsonTestData incomingRequest =
mockedRequestJsonDataMap.get("mockIncomingRequest1JsonDataWith2T2N");
mockedJsonDataMap.get("mockIncomingRequest1JsonDataWith2T2N");
Assert.assertFalse(incomingRequest.getJsonString().contains("sampleAliases"));
MockJsonTestData publishedRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
mockedJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
Assert.assertTrue(publishedRequest.getJsonString().contains("sampleAliases"));
Boolean consistencyCheckStatus = jsonComparator.isConsistent(
incomingRequest.getJsonString(), publishedRequest.getJsonString());
Expand All @@ -153,9 +153,9 @@ public void testPublishedSampleAliasesIgnored() throws Exception {
@Test
public void testSubSampleMetadataJsonUpdates() throws Exception {
MockJsonTestData referenceRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonDataWithLibUpdates");
mockedJsonDataMap.get("mockPublishedRequest1JsonDataWithLibUpdates");
MockJsonTestData targetRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
mockedJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
Boolean consistencyCheckStatus = jsonComparator.isConsistent(
referenceRequest.getJsonString(), targetRequest.getJsonString());
Assert.assertFalse(consistencyCheckStatus);
Expand All @@ -169,9 +169,9 @@ public void testSubSampleMetadataJsonUpdates() throws Exception {
public void testUpdatesComparatorSampleMetadata() throws Exception {
// create json with sample-level metadata and another with updates
MockJsonTestData referenceRequest =
mockedRequestJsonDataMap.get("mockUpdatedPublishedSampleMetadata");
mockedJsonDataMap.get("mockUpdatedPublishedSampleMetadata");
MockJsonTestData targetRequest =
mockedRequestJsonDataMap.get("mockUpdatedPublishedSampleMetadataWithInvalidUpdates");
mockedJsonDataMap.get("mockUpdatedPublishedSampleMetadataWithInvalidUpdates");
Boolean consistencyCheckStatus = jsonComparator.isConsistentByIgoProperties(
referenceRequest.getJsonString(), targetRequest.getJsonString());
Assert.assertTrue(consistencyCheckStatus);
Expand All @@ -185,9 +185,9 @@ public void testUpdatesComparatorSampleMetadata() throws Exception {
public void testInvalidRunsUpdatesComparatorSampleMetadata() throws Exception {
// create json with sample-level metadata and another with updates
MockJsonTestData referenceRequest =
mockedRequestJsonDataMap.get("mockUpdatedPublishedSampleMetadata");
mockedJsonDataMap.get("mockUpdatedPublishedSampleMetadata");
MockJsonTestData targetRequest =
mockedRequestJsonDataMap.get("mockUpdatedPublishedSampleMetadataWithInvalidRunsUpdates");
mockedJsonDataMap.get("mockUpdatedPublishedSampleMetadataWithInvalidRunsUpdates");
Boolean consistencyCheckStatus = jsonComparator.isConsistentByIgoProperties(
referenceRequest.getJsonString(), targetRequest.getJsonString());
Assert.assertFalse(consistencyCheckStatus);
Expand All @@ -196,9 +196,9 @@ public void testInvalidRunsUpdatesComparatorSampleMetadata() throws Exception {
@Test
public void testValidUpdatesComparatorWholeRequest() throws Exception {
MockJsonTestData referenceRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonDataWithLibUpdates");
mockedJsonDataMap.get("mockPublishedRequest1JsonDataWithLibUpdates");
MockJsonTestData targetRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
mockedJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
Boolean consistencyCheckStatus = jsonComparator.isConsistentByIgoProperties(
referenceRequest.getJsonString(), targetRequest.getJsonString());
Assert.assertFalse(consistencyCheckStatus);
Expand All @@ -207,9 +207,9 @@ public void testValidUpdatesComparatorWholeRequest() throws Exception {
@Test
public void testInvalidUpdatesComparatorWholeRequest() throws Exception {
MockJsonTestData referenceRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonDataWithInvalidUpdates");
mockedJsonDataMap.get("mockPublishedRequest1JsonDataWithInvalidUpdates");
MockJsonTestData targetRequest =
mockedRequestJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
mockedJsonDataMap.get("mockPublishedRequest1JsonDataWith2T2N");
Boolean consistencyCheckStatus = jsonComparator.isConsistentByIgoProperties(
referenceRequest.getJsonString(), targetRequest.getJsonString());
Assert.assertTrue(consistencyCheckStatus);
Expand All @@ -221,8 +221,8 @@ public void testInvalidUpdatesComparatorWholeRequest() throws Exception {
* @throws Exception
*/
@Test public void testSampleHasIgoUpdates() throws Exception {
MockJsonTestData refSample = mockedRequestJsonDataMap.get("samplePreUpdate");
MockJsonTestData igoUpdateSample = mockedRequestJsonDataMap.get("sampleLimsUpdatesOnly");
MockJsonTestData refSample = mockedJsonDataMap.get("samplePreUpdate");
MockJsonTestData igoUpdateSample = mockedJsonDataMap.get("sampleLimsUpdatesOnly");
Boolean igoUpdatesOnlyCheck = jsonComparator.isConsistentByIgoProperties(
refSample.getJsonString(), igoUpdateSample.getJsonString());
Assert.assertFalse(igoUpdatesOnlyCheck);
Expand All @@ -233,8 +233,8 @@ public void testInvalidUpdatesComparatorWholeRequest() throws Exception {
* @throws Exception
*/
@Test public void testSampleHasNonIgoUpdates() throws Exception {
MockJsonTestData refSample = mockedRequestJsonDataMap.get("samplePreUpdate");
MockJsonTestData nonIgoUpdateSample = mockedRequestJsonDataMap.get("sampleNonLimsUpdates");
MockJsonTestData refSample = mockedJsonDataMap.get("samplePreUpdate");
MockJsonTestData nonIgoUpdateSample = mockedJsonDataMap.get("sampleNonLimsUpdates");
Boolean nonIgoUpdatesCheck2 = jsonComparator.isConsistentByIgoProperties(
refSample.getJsonString(), nonIgoUpdateSample.getJsonString());
Assert.assertTrue(nonIgoUpdatesCheck2);
Expand All @@ -245,8 +245,8 @@ public void testInvalidUpdatesComparatorWholeRequest() throws Exception {
* @throws Exception
*/
@Test public void testSampleHasMixedIgoAndNonIgoUpdates() throws Exception {
MockJsonTestData refSample = mockedRequestJsonDataMap.get("samplePreUpdate");
MockJsonTestData mixedUpdatesSample = mockedRequestJsonDataMap.get("sampleMixedUpdates");
MockJsonTestData refSample = mockedJsonDataMap.get("samplePreUpdate");
MockJsonTestData mixedUpdatesSample = mockedJsonDataMap.get("sampleMixedUpdates");
Boolean mixedUpdatesSampleCheck1 = jsonComparator.isConsistent(
refSample.getJsonString(), mixedUpdatesSample.getJsonString());
Assert.assertFalse(mixedUpdatesSampleCheck1);
Expand All @@ -260,8 +260,8 @@ public void testInvalidUpdatesComparatorWholeRequest() throws Exception {
* @throws Exception
*/
@Test public void testCompareSampleStatus() throws Exception {
MockJsonTestData refJson = mockedRequestJsonDataMap.get("mockIncomingRequest1JsonWithSampleStatus");
MockJsonTestData tarJson = mockedRequestJsonDataMap.get("mockPublishingRequest1JsonWithSampleStatus");
MockJsonTestData refJson = mockedJsonDataMap.get("mockIncomingRequest1JsonWithSampleStatus");
MockJsonTestData tarJson = mockedJsonDataMap.get("mockPublishingRequest1JsonWithSampleStatus");
Boolean compareSampleStatus = jsonComparator.isConsistent(
refJson.getJsonString(), tarJson.getJsonString());
Assert.assertTrue(compareSampleStatus);
Expand All @@ -272,9 +272,9 @@ public void testInvalidUpdatesComparatorWholeRequest() throws Exception {
* @throws Exception
*/
@Test public void testCompareInvalidSampleStatus() throws Exception {
MockJsonTestData refJson = mockedRequestJsonDataMap.get(
MockJsonTestData refJson = mockedJsonDataMap.get(
"mockIncomingRequest1JsonWithSampleStatus");
MockJsonTestData tarJson = mockedRequestJsonDataMap.get(
MockJsonTestData tarJson = mockedJsonDataMap.get(
"mockPublishingRequest1JsonInvalidSampleStatus");
Boolean compareSampleStatus = jsonComparator.isConsistent(
refJson.getJsonString(), tarJson.getJsonString());
Expand All @@ -286,15 +286,31 @@ public void testInvalidUpdatesComparatorWholeRequest() throws Exception {
* @throws Exception
*/
@Test public void testCompareMissingSampleStatus() throws Exception {
MockJsonTestData refJson = mockedRequestJsonDataMap.get(
MockJsonTestData refJson = mockedJsonDataMap.get(
"mockIncomingRequest1JsonWithSampleStatus");
MockJsonTestData tarJson = mockedRequestJsonDataMap.get(
MockJsonTestData tarJson = mockedJsonDataMap.get(
"mockPublishingRequest1JsonMissingSampleStatus");
Boolean compareSampleStatus = jsonComparator.isConsistent(
refJson.getJsonString(), tarJson.getJsonString());
Assert.assertFalse(compareSampleStatus);
}

/**
* Tests that the comparator can detect changes in generic jsons that do not undergo
* as much filtering and standardization as request jsons or sample jsons.
* @throws Exception
*/
@Test
public void testUpdateCohortCompleteData() throws Exception {
MockJsonTestData refJson = mockedJsonDataMap.get(
"mockCohortCompleteCCSPPPQQQQ");
MockJsonTestData tarJson = mockedJsonDataMap.get(
"mockCohortCompleteCCSPPPQQQQUpdated");
Boolean isConsistent = jsonComparator.isConsistentGenericComparison(refJson.getJsonString(),
tarJson.getJsonString());
Assert.assertFalse(isConsistent);
}

private String getErrorMessage(Map<String, String> errorsMap) {
StringBuilder builder = new StringBuilder();
builder.append("\nConsistencyCheckerUtil failures summary:\n");
Expand Down
12 changes: 6 additions & 6 deletions src/test/java/org/mskcc/smile/commons/config/MockDataConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,16 @@ public JsonComparator jsonComparator() {
return jsonComparator;
}

private Map<String, MockJsonTestData> mockedRequestJsonDataMap;
private Map<String, MockJsonTestData> mockedJsonDataMap;

/**
* Generates a mocked request json data map.
* @return Map
* @throws IOException
*/
@Bean(name = "mockedRequestJsonDataMap")
public Map<String, MockJsonTestData> mockedRequestJsonDataMap() throws IOException {
this.mockedRequestJsonDataMap = new HashMap<>();
@Bean(name = "mockedJsonDataMap")
public Map<String, MockJsonTestData> mockedJsonDataMap() throws IOException {
this.mockedJsonDataMap = new HashMap<>();
ClassPathResource jsonDataDetailsResource =
new ClassPathResource(MOCKED_REQUEST_DATA_DETAILS_FILEPATH);
BufferedReader reader = new BufferedReader(new FileReader(jsonDataDetailsResource.getFile()));
Expand All @@ -77,11 +77,11 @@ public Map<String, MockJsonTestData> mockedRequestJsonDataMap() throws IOExcepti
String identifier = data[columns.indexOf("identifier")];
String filepath = data[columns.indexOf("filepath")];
String description = data[columns.indexOf("description")];
mockedRequestJsonDataMap.put(identifier,
mockedJsonDataMap.put(identifier,
createMockJsonTestData(identifier, filepath, description));
}
reader.close();
return mockedRequestJsonDataMap;
return mockedJsonDataMap;
}

private MockJsonTestData createMockJsonTestData(String identifier,
Expand Down
Loading

0 comments on commit 2366ca3

Please sign in to comment.