From 5fc5f8830f1fb4978eacc6fcecf689f538badaa7 Mon Sep 17 00:00:00 2001 From: macchiati Date: Mon, 22 Jul 2024 15:04:49 -0700 Subject: [PATCH] CLDR-17544 Enable testing order --- .../cldr/tool/ShowInconsistentAvailable.java | 194 +++++++++++++++--- 1 file changed, 169 insertions(+), 25 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowInconsistentAvailable.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowInconsistentAvailable.java index 7846d58f433..4d15c043c8f 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowInconsistentAvailable.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowInconsistentAvailable.java @@ -9,9 +9,12 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.common.collect.TreeMultimap; +import com.ibm.icu.text.DateIntervalInfo; +import com.ibm.icu.text.DateIntervalInfo.PatternInfo; import com.ibm.icu.text.DateTimePatternGenerator; import com.ibm.icu.text.DateTimePatternGenerator.FormatParser; import com.ibm.icu.text.DateTimePatternGenerator.VariableField; +import com.ibm.icu.util.ULocale; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; @@ -24,6 +27,8 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; +import org.unicode.cldr.tool.Option.Options; +import org.unicode.cldr.tool.Option.Params; import org.unicode.cldr.util.CLDRConfig; import org.unicode.cldr.util.CLDRFile; import org.unicode.cldr.util.CLDRFile.Status; @@ -37,14 +42,18 @@ import org.unicode.cldr.util.XPathParts; public class ShowInconsistentAvailable { + private static final boolean SHOW_ORDERING = true; + private static final boolean SHOW_ROOT = false; + private static final CLDRConfig CONFIG = CLDRConfig.getInstance(); - static boolean INCLUDE_ERA = false; + static boolean INCLUDE_ERA = true; static boolean SHOW_PROGRESS_RAW = false; static boolean SHOW_PROGRESS = false; static String DEBUG_ONLY_CALENDAR = null; // "chinese"; // null == all static SupplementalDataInfo SDI = SupplementalDataInfo.getInstance(); private static final Joiner TAB_JOINER = Joiner.on('\t'); + private static final Joiner LF_JOINER = Joiner.on('\n'); static FormatParser fp = new DateTimePatternGenerator.FormatParser(); static Factory f = CONFIG.getCldrFactory(); static PathHeader.Factory phf = PathHeader.getFactory(); @@ -52,18 +61,58 @@ public class ShowInconsistentAvailable { static int counter = 0; static Set nullErrors = new LinkedHashSet<>(); + private enum MyOptions { + ordering( + new Params() + .setHelp("find the ordering of fields in availableFormats and intervals")), + inconsistencies(new Params().setHelp("find inconsistancies in available formatts")), + root(new Params().setHelp("find root paths")); + + // BOILERPLATE TO COPY + final Option option; + + private MyOptions(Params params) { + option = new Option(this, params); + } + + private static Options myOptions = new Options(); + + static { + for (MyOptions option : MyOptions.values()) { + myOptions.add(option, option.option); + } + } + + private static Set parse(String[] args) { + return myOptions.parse(MyOptions.values()[0], args, true); + } + } + public static void main(String[] args) { - getRootPaths(); - if (true) return; - System.out.println( - "counter, locale, fLocale, calendar, skeleton, alt, coverage, value, sSimple, vSimple, error" - .replace(", ", "\t")); - show("root"); + MyOptions.parse(args); Set cldrLocales = StandardCodes.make().getLocaleCoverageLocales(Organization.cldr); Set specialLocales = StandardCodes.make().getLocaleCoverageLocales(Organization.special); - for (String locale : Sets.difference(cldrLocales, specialLocales)) { - show(locale); + final Set cldrLocalesWithoutSpecial = Sets.difference(cldrLocales, specialLocales); + + if (MyOptions.ordering.option.doesOccur()) { + showOrdering(cldrLocalesWithoutSpecial); + } + if (MyOptions.root.option.doesOccur()) { + getRootPaths(); + } + if (MyOptions.inconsistencies.option.doesOccur()) { + showInconsistencies(cldrLocalesWithoutSpecial); + } + } + + public static void showInconsistencies(final Set cldrLocalesWithoutSpecial) { + System.out.println( + "counter, locale, fLocale, calendar, skeleton, alt, coverage, value, sSimple, vSimple, error" + .replace(", ", "\t")); + showInconsistenciesInLocale("root"); + for (String locale : cldrLocalesWithoutSpecial) { + showInconsistenciesInLocale(locale); } nullErrors = ImmutableSet.copyOf( @@ -73,6 +122,92 @@ public static void main(String[] args) { System.out.println(Joiner.on('\n').join(nullErrors)); } + public static void showOrdering(final Set cldrLocalesWithoutSpecial) { + Multimap calendarToSPatterns = TreeMultimap.create(); + for (String locale : cldrLocalesWithoutSpecial) { + getTCPaths(locale, calendarToSPatterns); + } + Set ts = new TreeSet<>(); + ts.addAll(calendarToSPatterns.values()); + ts = ImmutableSet.copyOf(ts); + Set fields = new TreeSet<>(); + for (SimplePattern xx : ts) { + fields.addAll(xx.internal); + } + fields.remove(VARIABLE_FIELD_U); + + for (VariableField2 field : fields) { + + System.out.print("\n∋Var\tCal"); + for (SimplePattern p : ts) { + if (p.internal.contains(field)) { + System.out.print("\t" + p); + } + } + System.out.println(); + for (String cal : calendarToSPatterns.keySet()) { + System.out.print(SimplePattern.pretty(field) + "\t" + cal); + for (SimplePattern p : ts) { + if (p.internal.contains(field)) { + System.out.print( + "\t" + (calendarToSPatterns.containsEntry(cal, p) ? "Y" : "-")); + } + } + System.out.println(); + } + } + } + + private static void getTCPaths( + String locale, Multimap calendarToSPatterns) { + CLDRFile cldrFile = f.make(locale, false); + DateIntervalInfo fInfo = new DateIntervalInfo(new ULocale(locale)).freeze(); + + for (String path : cldrFile) { + String value = cldrFile.getStringValue(path); + if (value == null || value.isBlank() || value.equals("↑↑↑")) { + continue; + } + XPathParts parts = XPathParts.getFrozenInstance(path); + final String lastElement = parts.getElement(-1); + switch (lastElement) { + case "dateFormatItem": + String calendar = parts.getAttributeValue(3, "type"); + SimplePattern valueSimplePattern = + new SimplePattern(value, PatternType.pattern); + calendarToSPatterns.put(calendar, valueSimplePattern); + break; + case "greatestDifference": + calendar = parts.getAttributeValue(3, "type"); + SimplePattern first = null; + SimplePattern second = null; + PatternInfo pattern = DateIntervalInfo.genPatternInfo(value, false); + try { + first = new SimplePattern(pattern.getFirstPart(), PatternType.pattern); + second = new SimplePattern(pattern.getSecondPart(), PatternType.pattern); + } catch (Exception e) { + } + if (first == null || second == null) { + String skeleton = parts.getAttributeValue(-2, "id"); + String greatest = parts.getAttributeValue(-1, "id"); + System.out.println( + TAB_JOINER.join( + "Error:", + locale, + calendar, + lastElement, + skeleton, + greatest, + value)); + continue; + } + calendarToSPatterns.put(calendar, first); + calendarToSPatterns.put(calendar, second); + break; + } + } + } + private static void getRootPaths() { Multimap skelToCals = TreeMultimap.create(); Map, String> skelCalToSource = new HashMap<>(); @@ -138,7 +273,7 @@ public int hashCode() { } } - static void show(String locale) { + static void showInconsistenciesInLocale(String locale) { CLDRFile cldrFile = f.make(locale, true); Status out = new Status(); @@ -161,8 +296,9 @@ static void show(String locale) { if (alt == null) { alt = ""; } - SimplePattern skeletonSimplePattern = new SimplePattern(skeleton, true); - SimplePattern valueSimplePattern = new SimplePattern(value, false); + SimplePattern skeletonSimplePattern = + new SimplePattern(skeleton, PatternType.skeleton); + SimplePattern valueSimplePattern = new SimplePattern(value, PatternType.pattern); String fLocale = cldrFile.getSourceLocaleID(path, out); String fPath = out.pathWhereFound; @@ -209,8 +345,9 @@ static void show(String locale) { continue; } - SimplePattern skeletonSimplePattern = new SimplePattern(skeleton, true); - SimplePattern valueSimplePattern = new SimplePattern(value, false); + SimplePattern skeletonSimplePattern = + new SimplePattern(skeleton, PatternType.skeleton); + SimplePattern valueSimplePattern = new SimplePattern(value, PatternType.pattern); // we verify that for the same (calendar, skeletonSimplePattern), we only have one // valueSimplePattern @@ -294,44 +431,51 @@ public int hashCode() { } } + public enum PatternType { + skeleton, + pattern + } + + private static final VariableField2 VARIABLE_FIELD_U = new VariableField2("U", true); + static class SimplePattern implements Comparable { static Comparator> comp = Comparators.lexicographical(Comparator.naturalOrder()); Collection internal; - SimplePattern(String id, boolean skeleton) { - internal = skeleton ? new TreeSet<>() : new LinkedHashSet<>(); + SimplePattern(String id, PatternType patternType) { + internal = + patternType == PatternType.skeleton ? new TreeSet<>() : new LinkedHashSet<>(); for (Object item : fp.set(id).getItems()) { if (item instanceof DateTimePatternGenerator.VariableField) { VariableField2 v = new VariableField2(item, true); switch (v.getType()) { case DateTimePatternGenerator.ERA: - if (!INCLUDE_ERA && !skeleton) { + if (!INCLUDE_ERA && patternType == PatternType.pattern) { continue; } break; case DateTimePatternGenerator.DAYPERIOD: continue; case DateTimePatternGenerator.YEAR: // handle r(U) by mapping U to r - v = new VariableField2("U", true); + v = VARIABLE_FIELD_U; break; } internal.add(v); } } - if (!skeleton) { + if (patternType == PatternType.pattern) { internal = List.copyOf(internal); } } @Override public String toString() { - return internal.stream() - .map( - v -> - VariableField.getCanonicalCode(v.getType()) - + (v.isNumeric() ? "ⁿ" : "ˢ")) - .collect(Collectors.joining("")); + return internal.stream().map(v -> pretty(v)).collect(Collectors.joining("")); + } + + public static String pretty(VariableField2 v) { + return VariableField.getCanonicalCode(v.getType()) + (v.isNumeric() ? "ⁿ" : "ˢ"); } @Override