Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Show invariant test results in the diff #646

Merged
merged 15 commits into from
Jan 12, 2024
Merged

Show invariant test results in the diff #646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
85c2b67
Failing test
eggrobin Jan 10, 2024
6188e19
an attempt at error messages
eggrobin Jan 10, 2024
afc7d8c
comma
eggrobin Jan 10, 2024
29b341f
table and less escaping
eggrobin Jan 10, 2024
0497007
Try to get the errors only once
eggrobin Jan 10, 2024
4ca3390
We have screwed up since the beginning of time.
eggrobin Jan 10, 2024
80acf01
Revert invariant tests
eggrobin Jan 10, 2024
93c9570
Report various kinds of errors
eggrobin Jan 10, 2024
281f70b
report parse errors
eggrobin Jan 10, 2024
70a4ee2
Break everything
eggrobin Jan 10, 2024
03b74f2
make it a bit more readable hopefully
eggrobin Jan 10, 2024
4ac8e84
Revert "Break everything"
eggrobin Jan 10, 2024
3ba16f5
It is only an error if it is not what we expect.
eggrobin Jan 10, 2024
008fa45
Put the condition in the right place
eggrobin Jan 10, 2024
47ae9c4
Fehlermeldungszeilen
eggrobin Jan 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/cli-build-instructions.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ jobs:
- name: Run command - Build and Test
run: |
cd unicodetools/mine/src
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DEMIT_GITHUB_ERRORS
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Expand Down
134 changes: 102 additions & 32 deletions unicodetools/src/main/java/org/unicode/text/UCD/TestUnicodeInvariants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ public class TestUnicodeInvariants {
private static int showRangeLimit = 20;
static boolean doHtml = true;
public static final String DEFAULT_FILE = "UnicodeInvariantTest.txt";
public static final HTMLTabber htmlTabber = new Tabber.HTMLTabber();
public static final boolean EMIT_GITHUB_ERRORS =
System.getProperty("EMIT_GITHUB_ERRORS") != null;

private static final int
// HELP1 = 0,
Expand Down Expand Up @@ -171,8 +174,6 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
out3.write('\uFEFF'); // BOM
}
try (final BufferedReader in = getInputReader(inputFile)) {
final HTMLTabber tabber = new Tabber.HTMLTabber();

errorLister =
new BagFormatter()
.setMergeRanges(doRange)
Expand All @@ -183,7 +184,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
.setFixName(toHTML);
errorLister.setShowTotal(false);
if (doHtml) {
errorLister.setTabber(tabber);
errorLister.setTabber(htmlTabber);
}

showLister =
Expand All @@ -198,7 +199,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
showLister.setValueSource(LATEST_PROPS.getProperty("script"));
}
if (doHtml) {
showLister.setTabber(tabber);
showLister.setTabber(htmlTabber);
}

// symbolTable = new ChainedSymbolTable();
Expand All @@ -207,7 +208,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
// ToolUnicodePropertySource.make(UCD.lastVersion).getSymbolTable("\u00D7"),
//
// ToolUnicodePropertySource.make(Default.ucdVersion()).getSymbolTable("")});
while (true) {
for (int lineNumber = 1; ; ++lineNumber) {
String line = in.readLine();
if (line == null) {
break;
Expand All @@ -230,24 +231,24 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
} else if (line.startsWith("Let")) {
letLine(pp, line);
} else if (line.startsWith("In")) {
inLine(pp, line);
inLine(pp, line, lineNumber);
} else if (line.startsWith("ShowScript")) {
showScript = true;
} else if (line.startsWith("HideScript")) {
showScript = false;
} else if (line.startsWith("Map")) {
testMapLine(line, pp);
testMapLine(line, pp, lineNumber);
} else if (line.startsWith("ShowMap")) {
showMapLine(line, pp);
} else if (line.startsWith("Show")) {
showLine(line, pp);
} else if (line.startsWith("EquivalencesOf")) {
equivalencesLine(line, pp);
equivalencesLine(line, pp, lineNumber);
} else {
testLine(line, pp);
testLine(line, pp, lineNumber);
}
} catch (final Exception e) {
parseErrorCount = parseError(parseErrorCount, line, e);
parseErrorCount = parseError(parseErrorCount, line, e, lineNumber);
continue;
}
}
Expand Down Expand Up @@ -276,7 +277,8 @@ static class PropertyComparison {
UnicodeProperty property2;
}

private static void equivalencesLine(String line, ParsePosition pp) throws ParseException {
private static void equivalencesLine(String line, ParsePosition pp, int lineNumber)
throws ParseException {
pp.setIndex("EquivalencesOf".length());
final UnicodeSet domain = new UnicodeSet(line, pp, symbolTable);
final var leftProperty = CompoundProperty.of(LATEST_PROPS, line, pp);
Expand Down Expand Up @@ -435,16 +437,26 @@ private static void equivalencesLine(String line, ParsePosition pp) throws Parse
++testFailureCount;
printErrorLine("Test Failure", Side.START, testFailureCount);
}
final List<String> errorMessageLines = new ArrayList<>();
if (counterexamples.isEmpty()) {
println("There are no counterexamples to " + relationOperator + ".");
errorMessageLines.add("There are no counterexamples to " + relationOperator + ".");
} else {
if (leftShouldImplyRight) {
println("The implication ⇒ is " + leftImpliesRightCounterexamples.isEmpty() + ".");
errorMessageLines.add(
"The implication ⇒ is " + leftImpliesRightCounterexamples.isEmpty() + ".");
}
if (rightShouldImplyLeft) {
println("The implication ⇐ is " + rightImpliesLeftCounterexamples.isEmpty() + ".");
errorMessageLines.add(
"The implication ⇐ is " + rightImpliesLeftCounterexamples.isEmpty() + ".");
}
}
for (var errorLine : errorMessageLines) {
println(errorLine);
}
errorMessageLines.addAll(counterexamples);
if (failure) {
reportTestFailure(lineNumber, String.join("\n", errorMessageLines).replace('\t', ' '));
}
out.println(failure ? "<table class='f'>" : "<table>");
for (String counterexample : counterexamples) {
out.println("<tr><td>");
Expand All @@ -457,7 +469,8 @@ private static void equivalencesLine(String line, ParsePosition pp) throws Parse
}
}

private static void inLine(ParsePosition pp, String line) throws ParseException {
private static void inLine(ParsePosition pp, String line, int lineNumber)
throws ParseException {
pp.setIndex(2);
final PropertyComparison propertyComparison = getPropertyComparison(pp, line);
final UnicodeMap<String> failures = new UnicodeMap<>();
Expand All @@ -476,13 +489,23 @@ private static void inLine(ParsePosition pp, String line) throws ParseException
if (failureCount != 0) {
testFailureCount++;
printErrorLine("Test Failure", Side.START, testFailureCount);
println(
"## Got unexpected "
String errorMessage =
"Got unexpected "
+ (propertyComparison.shouldBeEqual ? "differences" : "equalities")
+ ": "
+ failureCount);
+ failureCount;
println("## " + errorMessage);

final UnicodeLabel failureProp = new UnicodeProperty.UnicodeMapProperty().set(failures);
errorLister.setValueSource(failureProp);

var monoTable = new StringWriter();
errorLister.setTabber(new Tabber.MonoTabber());
errorLister.setLineSeparator("\n");
errorLister.showSetNames(new PrintWriter(monoTable), failureSet);
errorLister.setTabber(htmlTabber);
reportTestFailure(lineNumber, errorMessage + "\n" + monoTable.toString());

if (doHtml) {
out.println("<table class='f'>");
}
Expand Down Expand Up @@ -710,7 +733,8 @@ private static void showMapLine(String line, ParsePosition pp) {
showLister.setMergeRanges(doRange);
}

private static void testLine(String line, ParsePosition pp) throws ParseException {
private static void testLine(String line, ParsePosition pp, int lineNumber)
throws ParseException {
if (line.startsWith("Test")) {
line = line.substring(4).trim();
}
Expand Down Expand Up @@ -776,21 +800,24 @@ private static void testLine(String line, ParsePosition pp) throws ParseExceptio
"In",
rightSide,
"But Not In",
leftSide);
leftSide,
lineNumber);
checkExpected(
rightAndLeft,
new UnicodeSet(rightSet).retainAll(leftSet),
"In",
rightSide,
"And In",
leftSide);
leftSide,
lineNumber);
checkExpected(
left_right,
new UnicodeSet(leftSet).removeAll(rightSet),
"In",
leftSide,
"But Not In",
rightSide);
rightSide,
lineNumber);
}

public static void checkRelation(ParsePosition pp, char relation) throws ParseException {
Expand All @@ -810,7 +837,8 @@ private static void checkExpected(
String rightStatus,
String rightSide,
String leftStatus,
String leftSide) {
String leftSide,
int lineNumber) {
switch (expected) {
case empty:
if (segment.size() == 0) {
Expand All @@ -829,9 +857,22 @@ private static void checkExpected(
}
testFailureCount++;
printErrorLine("Test Failure", Side.START, testFailureCount);
println("## Expected " + expected + ", got: " + segment.size() + "\t" + segment.toString());
println("## " + rightStatus + "\t" + rightSide);
println("## " + leftStatus + "\t" + leftSide);
final var errorMessageLines =
new String[] {
"Expected " + expected + ", got: " + segment.size() + "\t" + segment.toString(),
rightStatus + "\t" + rightSide,
leftStatus + "\t" + leftSide
};
var monoTable = new StringWriter();
for (String line : errorMessageLines) {
println("## " + line);
}
errorLister.setTabber(new Tabber.MonoTabber());
errorLister.setLineSeparator("\n");
errorLister.showSetNames(new PrintWriter(monoTable), segment);
reportTestFailure(
lineNumber, String.join("\n", errorMessageLines) + "\n" + monoTable.toString());
errorLister.setTabber(htmlTabber);
if (doHtml) {
out.println("<table class='e'>");
}
Expand All @@ -853,7 +894,8 @@ private static void checkExpected(
getProperties(Settings.lastVersion),
IndexUnicodeProperties.make(Settings.lastVersion)));

private static void testMapLine(String line, ParsePosition pp) throws ParseException {
private static void testMapLine(String line, ParsePosition pp, int lineNumber)
throws ParseException {
char relation = 0;
String rightSide = null;
String leftSide = null;
Expand Down Expand Up @@ -915,21 +957,24 @@ private static void testMapLine(String line, ParsePosition pp) throws ParseExcep
"In",
rightSide,
"But Not In",
leftSide);
leftSide,
lineNumber);
checkExpected(
rightAndLeft,
UnicodeMapParser.retainAll(new UnicodeMap<String>().putAll(rightSet), leftSet),
"In",
rightSide,
"And In",
leftSide);
leftSide,
lineNumber);
checkExpected(
left_right,
UnicodeMapParser.removeAll(new UnicodeMap<String>().putAll(leftSet), rightSet),
"In",
leftSide,
"But Not In",
rightSide);
rightSide,
lineNumber);
}

private static void checkExpected(
Expand All @@ -938,7 +983,8 @@ private static void checkExpected(
String rightStatus,
String rightSide,
String leftStatus,
String leftSide) {
String leftSide,
int lineNumber) {
switch (expected) {
case empty:
if (segment.size() == 0) {
Expand Down Expand Up @@ -1015,7 +1061,7 @@ private static void showSet(ParsePosition pp, final String value) {
println();
}

private static int parseError(int parseErrorCount, String line, Exception e) {
private static int parseError(int parseErrorCount, String line, Exception e, int lineNumber) {
parseErrorCount++;
if (e instanceof ParseException) {
final int index = ((ParseException) e).getErrorOffset();
Expand All @@ -1029,7 +1075,9 @@ private static int parseError(int parseErrorCount, String line, Exception e) {
if (message != null) {
println("##" + message);
}
reportParseError(lineNumber, message);
e.printStackTrace(out);

out.println("</pre>");
printErrorLine("Parse Error", Side.END, parseErrorCount);
println();
Expand Down Expand Up @@ -1122,6 +1170,28 @@ private static void println() {
println("");
}

private static void reportParseError(int lineNumber, String message) {
reportError(lineNumber, "Parse error", message);
}

private static void reportTestFailure(int lineNumber, String message) {
reportError(lineNumber, "Invariant test failure", message);
}

private static void reportError(int lineNumber, String title, String message) {
if (EMIT_GITHUB_ERRORS) {
System.err.println(
"::error file=unicodetools/src/main/resources/org/unicode/text/UCD/"
+ DEFAULT_FILE
+ ",line="
+ lineNumber
+ ",title="
+ title
+ "::"
+ message.replace("%", "%25").replace("\n", "%0A"));
}
}

/** Should add to UnicodeSet */
public static String scan(UnicodeSet unicodeSet, String line, ParsePosition pp, boolean in) {
final int start = pp.getIndex();
Expand Down
Loading