Skip to content

Commit

Permalink
Show invariant test results in the diff (#646)
Browse files Browse the repository at this point in the history
  • Loading branch information
@eggrobin
eggrobin authored Jan 12, 2024
1 parent 8cb8c74 commit 210204f
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 33 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/cli-build-instructions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ jobs:
- name: Run command - Build and Test
run: |
cd unicodetools/mine/src
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DEMIT_GITHUB_ERRORS
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Expand Down
134 changes: 102 additions & 32 deletions unicodetools/src/main/java/org/unicode/text/UCD/TestUnicodeInvariants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ public class TestUnicodeInvariants {
private static int showRangeLimit = 20;
static boolean doHtml = true;
public static final String DEFAULT_FILE = "UnicodeInvariantTest.txt";
public static final HTMLTabber htmlTabber = new Tabber.HTMLTabber();
public static final boolean EMIT_GITHUB_ERRORS =
System.getProperty("EMIT_GITHUB_ERRORS") != null;

private static final int
// HELP1 = 0,
Expand Down Expand Up @@ -171,8 +174,6 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
out3.write('\uFEFF'); // BOM
}
try (final BufferedReader in = getInputReader(inputFile)) {
final HTMLTabber tabber = new Tabber.HTMLTabber();

errorLister =
new BagFormatter()
.setMergeRanges(doRange)
Expand All @@ -183,7 +184,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
.setFixName(toHTML);
errorLister.setShowTotal(false);
if (doHtml) {
errorLister.setTabber(tabber);
errorLister.setTabber(htmlTabber);
}

showLister =
Expand All @@ -198,7 +199,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
showLister.setValueSource(LATEST_PROPS.getProperty("script"));
}
if (doHtml) {
showLister.setTabber(tabber);
showLister.setTabber(htmlTabber);
}

// symbolTable = new ChainedSymbolTable();
Expand All @@ -207,7 +208,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
// ToolUnicodePropertySource.make(UCD.lastVersion).getSymbolTable("\u00D7"),
//
// ToolUnicodePropertySource.make(Default.ucdVersion()).getSymbolTable("")});
while (true) {
for (int lineNumber = 1; ; ++lineNumber) {
String line = in.readLine();
if (line == null) {
break;
Expand All @@ -230,24 +231,24 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
} else if (line.startsWith("Let")) {
letLine(pp, line);
} else if (line.startsWith("In")) {
inLine(pp, line);
inLine(pp, line, lineNumber);
} else if (line.startsWith("ShowScript")) {
showScript = true;
} else if (line.startsWith("HideScript")) {
showScript = false;
} else if (line.startsWith("Map")) {
testMapLine(line, pp);
testMapLine(line, pp, lineNumber);
} else if (line.startsWith("ShowMap")) {
showMapLine(line, pp);
} else if (line.startsWith("Show")) {
showLine(line, pp);
} else if (line.startsWith("EquivalencesOf")) {
equivalencesLine(line, pp);
equivalencesLine(line, pp, lineNumber);
} else {
testLine(line, pp);
testLine(line, pp, lineNumber);
}
} catch (final Exception e) {
parseErrorCount = parseError(parseErrorCount, line, e);
parseErrorCount = parseError(parseErrorCount, line, e, lineNumber);
continue;
}
}
Expand Down Expand Up @@ -276,7 +277,8 @@ static class PropertyComparison {
UnicodeProperty property2;
}

private static void equivalencesLine(String line, ParsePosition pp) throws ParseException {
private static void equivalencesLine(String line, ParsePosition pp, int lineNumber)
throws ParseException {
pp.setIndex("EquivalencesOf".length());
final UnicodeSet domain = new UnicodeSet(line, pp, symbolTable);
final var leftProperty = CompoundProperty.of(LATEST_PROPS, line, pp);
Expand Down Expand Up @@ -435,16 +437,26 @@ private static void equivalencesLine(String line, ParsePosition pp) throws Parse
++testFailureCount;
printErrorLine("Test Failure", Side.START, testFailureCount);
}
final List<String> errorMessageLines = new ArrayList<>();
if (counterexamples.isEmpty()) {
println("There are no counterexamples to " + relationOperator + ".");
errorMessageLines.add("There are no counterexamples to " + relationOperator + ".");
} else {
if (leftShouldImplyRight) {
println("The implication ⇒ is " + leftImpliesRightCounterexamples.isEmpty() + ".");
errorMessageLines.add(
"The implication ⇒ is " + leftImpliesRightCounterexamples.isEmpty() + ".");
}
if (rightShouldImplyLeft) {
println("The implication ⇐ is " + rightImpliesLeftCounterexamples.isEmpty() + ".");
errorMessageLines.add(
"The implication ⇐ is " + rightImpliesLeftCounterexamples.isEmpty() + ".");
}
}
for (var errorLine : errorMessageLines) {
println(errorLine);
}
errorMessageLines.addAll(counterexamples);
if (failure) {
reportTestFailure(lineNumber, String.join("\n", errorMessageLines).replace('\t', ' '));
}
out.println(failure ? "<table class='f'>" : "<table>");
for (String counterexample : counterexamples) {
out.println("<tr><td>");
Expand All @@ -457,7 +469,8 @@ private static void equivalencesLine(String line, ParsePosition pp) throws Parse
}
}

private static void inLine(ParsePosition pp, String line) throws ParseException {
private static void inLine(ParsePosition pp, String line, int lineNumber)
throws ParseException {
pp.setIndex(2);
final PropertyComparison propertyComparison = getPropertyComparison(pp, line);
final UnicodeMap<String> failures = new UnicodeMap<>();
Expand All @@ -476,13 +489,23 @@ private static void inLine(ParsePosition pp, String line) throws ParseException
if (failureCount != 0) {
testFailureCount++;
printErrorLine("Test Failure", Side.START, testFailureCount);
println(
"## Got unexpected "
String errorMessage =
"Got unexpected "
+ (propertyComparison.shouldBeEqual ? "differences" : "equalities")
+ ": "
+ failureCount);
+ failureCount;
println("## " + errorMessage);

final UnicodeLabel failureProp = new UnicodeProperty.UnicodeMapProperty().set(failures);
errorLister.setValueSource(failureProp);

var monoTable = new StringWriter();
errorLister.setTabber(new Tabber.MonoTabber());
errorLister.setLineSeparator("\n");
errorLister.showSetNames(new PrintWriter(monoTable), failureSet);
errorLister.setTabber(htmlTabber);
reportTestFailure(lineNumber, errorMessage + "\n" + monoTable.toString());

if (doHtml) {
out.println("<table class='f'>");
}
Expand Down Expand Up @@ -710,7 +733,8 @@ private static void showMapLine(String line, ParsePosition pp) {
showLister.setMergeRanges(doRange);
}

private static void testLine(String line, ParsePosition pp) throws ParseException {
private static void testLine(String line, ParsePosition pp, int lineNumber)
throws ParseException {
if (line.startsWith("Test")) {
line = line.substring(4).trim();
}
Expand Down Expand Up @@ -776,21 +800,24 @@ private static void testLine(String line, ParsePosition pp) throws ParseExceptio
"In",
rightSide,
"But Not In",
leftSide);
leftSide,
lineNumber);
checkExpected(
rightAndLeft,
new UnicodeSet(rightSet).retainAll(leftSet),
"In",
rightSide,
"And In",
leftSide);
leftSide,
lineNumber);
checkExpected(
left_right,
new UnicodeSet(leftSet).removeAll(rightSet),
"In",
leftSide,
"But Not In",
rightSide);
rightSide,
lineNumber);
}

public static void checkRelation(ParsePosition pp, char relation) throws ParseException {
Expand All @@ -810,7 +837,8 @@ private static void checkExpected(
String rightStatus,
String rightSide,
String leftStatus,
String leftSide) {
String leftSide,
int lineNumber) {
switch (expected) {
case empty:
if (segment.size() == 0) {
Expand All @@ -829,9 +857,22 @@ private static void checkExpected(
}
testFailureCount++;
printErrorLine("Test Failure", Side.START, testFailureCount);
println("## Expected " + expected + ", got: " + segment.size() + "\t" + segment.toString());
println("## " + rightStatus + "\t" + rightSide);
println("## " + leftStatus + "\t" + leftSide);
final var errorMessageLines =
new String[] {
"Expected " + expected + ", got: " + segment.size() + "\t" + segment.toString(),
rightStatus + "\t" + rightSide,
leftStatus + "\t" + leftSide
};
var monoTable = new StringWriter();
for (String line : errorMessageLines) {
println("## " + line);
}
errorLister.setTabber(new Tabber.MonoTabber());
errorLister.setLineSeparator("\n");
errorLister.showSetNames(new PrintWriter(monoTable), segment);
reportTestFailure(
lineNumber, String.join("\n", errorMessageLines) + "\n" + monoTable.toString());
errorLister.setTabber(htmlTabber);
if (doHtml) {
out.println("<table class='e'>");
}
Expand All @@ -853,7 +894,8 @@ private static void checkExpected(
getProperties(Settings.lastVersion),
IndexUnicodeProperties.make(Settings.lastVersion)));

private static void testMapLine(String line, ParsePosition pp) throws ParseException {
private static void testMapLine(String line, ParsePosition pp, int lineNumber)
throws ParseException {
char relation = 0;
String rightSide = null;
String leftSide = null;
Expand Down Expand Up @@ -915,21 +957,24 @@ private static void testMapLine(String line, ParsePosition pp) throws ParseExcep
"In",
rightSide,
"But Not In",
leftSide);
leftSide,
lineNumber);
checkExpected(
rightAndLeft,
UnicodeMapParser.retainAll(new UnicodeMap<String>().putAll(rightSet), leftSet),
"In",
rightSide,
"And In",
leftSide);
leftSide,
lineNumber);
checkExpected(
left_right,
UnicodeMapParser.removeAll(new UnicodeMap<String>().putAll(leftSet), rightSet),
"In",
leftSide,
"But Not In",
rightSide);
rightSide,
lineNumber);
}

private static void checkExpected(
Expand All @@ -938,7 +983,8 @@ private static void checkExpected(
String rightStatus,
String rightSide,
String leftStatus,
String leftSide) {
String leftSide,
int lineNumber) {
switch (expected) {
case empty:
if (segment.size() == 0) {
Expand Down Expand Up @@ -1015,7 +1061,7 @@ private static void showSet(ParsePosition pp, final String value) {
println();
}

private static int parseError(int parseErrorCount, String line, Exception e) {
private static int parseError(int parseErrorCount, String line, Exception e, int lineNumber) {
parseErrorCount++;
if (e instanceof ParseException) {
final int index = ((ParseException) e).getErrorOffset();
Expand All @@ -1029,7 +1075,9 @@ private static int parseError(int parseErrorCount, String line, Exception e) {
if (message != null) {
println("##" + message);
}
reportParseError(lineNumber, message);
e.printStackTrace(out);

out.println("</pre>");
printErrorLine("Parse Error", Side.END, parseErrorCount);
println();
Expand Down Expand Up @@ -1122,6 +1170,28 @@ private static void println() {
println("");
}

private static void reportParseError(int lineNumber, String message) {
reportError(lineNumber, "Parse error", message);
}

private static void reportTestFailure(int lineNumber, String message) {
reportError(lineNumber, "Invariant test failure", message);
}

private static void reportError(int lineNumber, String title, String message) {
if (EMIT_GITHUB_ERRORS) {
System.err.println(
"::error file=unicodetools/src/main/resources/org/unicode/text/UCD/"
+ DEFAULT_FILE
+ ",line="
+ lineNumber
+ ",title="
+ title
+ "::"
+ message.replace("%", "%25").replace("\n", "%0A"));
}
}

/** Should add to UnicodeSet */
public static String scan(UnicodeSet unicodeSet, String line, ParsePosition pp, boolean in) {
final int start = pp.getIndex();
Expand Down

0 comments on commit 210204f

Please sign in to comment.