From a3fc051b661590fbd4cb8b870981d53ba756c366 Mon Sep 17 00:00:00 2001 From: macchiati Date: Wed, 15 Nov 2023 11:55:39 -0800 Subject: [PATCH] CLDR-16232 back-out changes that cause rebase conflicts; will do those manually --- .../unicode/cldr/unittest/TestAliases.java | 494 +++--------------- 1 file changed, 64 insertions(+), 430 deletions(-) diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestAliases.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestAliases.java index ec28cc12746..e478a3ddf5f 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestAliases.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestAliases.java @@ -1,47 +1,22 @@ package org.unicode.cldr.unittest; +import com.ibm.icu.dev.test.TestFmwk; import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.unicode.cldr.draft.FileUtilities; import org.unicode.cldr.util.CLDRConfig; import org.unicode.cldr.util.CLDRFile; import org.unicode.cldr.util.CLDRFile.Status; import org.unicode.cldr.util.CLDRPaths; -import org.unicode.cldr.util.CldrUtility; -import org.unicode.cldr.util.Counter; import org.unicode.cldr.util.DtdType; -import org.unicode.cldr.util.Factory; -import org.unicode.cldr.util.Level; -import org.unicode.cldr.util.LocaleIDParser; -import org.unicode.cldr.util.Organization; -import org.unicode.cldr.util.SimpleFactory; -import org.unicode.cldr.util.StandardCodes; -import org.unicode.cldr.util.SupplementalDataInfo; import org.unicode.cldr.util.XMLSource; -import com.google.common.base.Joiner; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Multimap; -import com.google.common.collect.TreeMultimap; -import com.ibm.icu.dev.test.TestFmwk; -import com.ibm.icu.impl.Row.R2; -import com.ibm.icu.util.Output; - // make this a JUnit test? public class TestAliases extends TestFmwk { static final CLDRConfig config = CLDRConfig.getInstance(); @@ -52,9 +27,9 @@ public static void main(String[] args) { public void testAlias() { String[][] testCases = { - { "//ldml/foo[@fii=\"abc\"]", "//ldml" }, - { "//ldml/foo[@fii=\"ab/c\"]", "//ldml" }, - { "//ldml/foo[@fii=\"ab/[c\"]", "//ldml" }, + {"//ldml/foo[@fii=\"abc\"]", "//ldml"}, + {"//ldml/foo[@fii=\"ab/c\"]", "//ldml"}, + {"//ldml/foo[@fii=\"ab/[c\"]", "//ldml"}, }; for (String[] pair : testCases) { if (!XMLSource.Alias.stripLastElement(pair[0]).equals(pair[1])) { @@ -63,30 +38,32 @@ public void testAlias() { } } - /** Check on - * http://unicode.org/cldr/trac/ticket/9477 - * - * - * - - - in {0} qtr. - in {0} qtrs. - - + /** + * Check on http://unicode.org/cldr/trac/ticket/9477 in {0} + * qtr. in {0} + * qtrs. */ public void testCountBase() { String[][] testCases = { - { "en", + { + "en", "//ldml/numbers/currencyFormats/currencyFormatLength[@type=\"short\"]/currencyFormat[@type=\"standard\"]/pattern[@type=\"1000\"][@count=\"one\"]", "en", "//ldml/numbers/currencyFormats[@numberSystem=\"latn\"]/currencyFormatLength[@type=\"short\"]/currencyFormat[@type=\"standard\"]/pattern[@type=\"1000\"][@count=\"one\"]" }, - { "en", "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"one\"]", - "en", "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"one\"]", + { + "en", + "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"one\"]", + "en", + "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"one\"]", }, - { "ak", "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"one\"]", - "root", "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"other\"]", + { + "ak", + "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"one\"]", + "root", + "//ldml/characterLabels/characterLabelPattern[@type=\"strokes\"][@count=\"other\"]", } }; Status status = new Status(); @@ -114,7 +91,8 @@ public void testCountFull() { if (!path.contains("@count") || !path.contains("/field")) { // TODO remove /field continue; } - if (path.equals("//ldml/dates/fields/field[@type=\"wed-narrow\"]/relativeTime[@type=\"future\"]/relativeTimePattern[@count=\"one\"]")) { + if (path.equals( + "//ldml/dates/fields/field[@type=\"wed-narrow\"]/relativeTime[@type=\"future\"]/relativeTimePattern[@count=\"one\"]")) { int debug = 0; } String actualLocale = en.getSourceLocaleID(path, status); @@ -136,41 +114,52 @@ public void testCountFull() { } en.getSourceLocaleID(path, status); // for debugging - sorted.add("locale:\t" + actualLocale - + "\nsource path:\t" + path - + "\nsource fpath:\t" + (path.equals(fullpath) ? "=" : fullpath) - + "\nactual path:\t" + (path.equals(actualPath) ? "=" : actualPath) - + "\nactual value:\t" + value); + sorted.add( + "locale:\t" + + actualLocale + + "\nsource path:\t" + + path + + "\nsource fpath:\t" + + (path.equals(fullpath) ? "=" : fullpath) + + "\nactual path:\t" + + (path.equals(actualPath) ? "=" : actualPath) + + "\nactual value:\t" + + value); } sorted.forEach(x -> System.out.println(x)); } /** * Change to "testEmitChanged()" to emit a file of current inheritance. + * * @throws IOException */ public void checkEmitChanged() throws IOException { Status status = new Status(); Set sorted = new TreeSet<>(); - try (PrintWriter out = FileUtilities.openUTF8Writer(CLDRPaths.AUX_DIRECTORY + "temp", "inheritance0.txt")) { - for (CLDRFile factory : Arrays.asList( - config.getCldrFactory().make("root", true), - en, - config.getCldrFactory().make("en_001", true), // example with double inheritance - config.getCldrFactory().make("ak", true) // example with few strings - )) { + try (PrintWriter out = + FileUtilities.openUTF8Writer( + CLDRPaths.AUX_DIRECTORY + "temp", "inheritance0.txt")) { + for (CLDRFile factory : + Arrays.asList( + config.getCldrFactory().make("root", true), + en, + config.getCldrFactory() + .make("en_001", true), // example with double inheritance + config.getCldrFactory().make("ak", true) // example with few strings + )) { sorted.clear(); out.println("source locale\tactual locale\tsource path\tactual path"); String locale = factory.getLocaleID(); for (String path : factory.fullIterable()) { if (path.contains("calendar[@type=") - && !(path.contains("calendar[@type=\"gregorian") - || path.contains("calendar[@type=\"generic"))) { + && !(path.contains("calendar[@type=\"gregorian") + || path.contains("calendar[@type=\"generic"))) { continue; } if (path.contains("[@numberSystem=") - && !(path.contains("[@numberSystem=\"latn") - || path.contains("[@numberSystem=\"deva"))) { + && !(path.contains("[@numberSystem=\"latn") + || path.contains("[@numberSystem=\"deva"))) { continue; } String actualLocale = factory.getSourceLocaleID(path, status); @@ -180,10 +169,13 @@ public void checkEmitChanged() throws IOException { } sorted.add( - locale - + "\t" + (locale.equals(actualLocale) ? "=" : actualLocale) - + "\t" + path - + "\t" + (path.equals(actualPath) ? "=" : actualPath)); + locale + + "\t" + + (locale.equals(actualLocale) ? "=" : actualLocale) + + "\t" + + path + + "\t" + + (path.equals(actualPath) ? "=" : actualPath)); } System.out.println(locale + "\t" + sorted.size()); sorted.forEach(x -> out.println(x)); @@ -191,8 +183,9 @@ public void checkEmitChanged() throws IOException { } } - /** Test that elements only are in root.html - * For speed in testing, just checks for the presence of " elements only are in root.html For speed in testing, just checks for the + * presence of " elements only are in root.html - * For speed in testing, just checks for the presence of " elements only are in root.html For speed in testing, just checks for the + * presence of " testValues = new LinkedHashSet<>(); - ImmutableList locales; - Set paths = new LinkedHashSet<>(); - - public TestValueSet(String... locales) { - this.locales = ImmutableList.copyOf(locales); - } - - public TestValueSet add(String locale, String path, String value) { - if (!locales.contains(locale)) { - throw new IllegalArgumentException(locale + " must be in " + locales); - } - testValues.add(new LocalePathValue(locale, path, value)); - paths.add(path); - return this; - } - - - public TestValueSet checkReplacements(String section, Factory std) { - TestFactory testFactory = copyIntoTestFactory(std); - setValuesIn(testFactory); - show("\n\t\t" + section + ", BEFORE replacing", testFactory); - TestFactory modifiedFactory = replaceIfInheritedEqual(testFactory, null); - check(section + ", AFTER replacing with null", modifiedFactory); - - TestFactory testFactory2 = copyIntoTestFactory(std); - setValuesIn(testFactory2); - //check("\nBefore replacing with ↑↑↑", testFactory2); - modifiedFactory = replaceIfInheritedEqual(testFactory, CldrUtility.INHERITANCE_MARKER); - check(section + ", AFTER replacing with " + CldrUtility.INHERITANCE_MARKER, modifiedFactory); - return this; - } - - public TestFactory copyIntoTestFactory(Factory std) { - TestFactory testFactory = new TestFactory(); - testFactory.setSupplementalDirectory(std.getSupplementalDirectory()); - - for (String locale : locales) { - CLDRFile cldrFile = std.make(locale, false); - testFactory.addFile(cldrFile.cloneAsThawed()); - } - return testFactory; - } - - public TestFactory replaceIfInheritedEqual(TestFactory testFactory, String replacement) { - Output pathWhereFound = new Output<>(); - Output localeWhereFound = new Output<>(); - HashMultimapactions = HashMultimap.create(); - - // Gather all the values that equal their baileys - - for (LocalePathValue localePathValue : testValues) { - CLDRFile cldrFile = testFactory.make(localePathValue.locale, true); - String value = cldrFile.getStringValueWithBailey(localePathValue.path); - String baileyValue = cldrFile.getBaileyValue(localePathValue.path, pathWhereFound, localeWhereFound); - if (Objects.equals(value, baileyValue)) { - actions.put(localePathValue.locale, localePathValue.path); - } - } - - // Now replace them all - // NOTE: Removing or adding new values might invalidate resolved files, so we create a new factory - - TestFactory testFactory1 = new TestFactory(); - testFactory1.setSupplementalDirectory(testFactory.getSupplementalDirectory()); - for (String locale : locales) { - final CLDRFile cldrFile = testFactory.make(locale, false).cloneAsThawed(); - Set items = actions.get(locale); - if (items != null) { - for (String path : items) { - if (replacement == null) { - cldrFile.remove(path); - } else { - cldrFile.add(path, replacement); - } - } - } - testFactory1.addFile(cldrFile); - } - return testFactory1; - - } - - public void show(String title, TestFactory testFactory) { - logln(title); - CLDRFile cldrFileRoot = testFactory.make("root", true); - for (String path : paths) { - String rawValue = cldrFileRoot.getUnresolved().getStringValue(path); - logln("root" + "\t" + path + "\t" + rawValue); - } - for (LocalePathValue entry : testValues) { - CLDRFile cldrFile = testFactory.make(entry.locale, true); - String rawValue = cldrFile.getUnresolved().getStringValue(entry.path); - logln(entry.locale + "\t" + entry.path + "\t" + rawValue); - } - } - - public void check(String title, TestFactory testFactory) { - logln(title); - for (LocalePathValue entry : testValues) { - CLDRFile cldrFile = testFactory.make(entry.locale, true); - String rawValue = cldrFile.getUnresolved().getStringValue(entry.path); - String resolvedValue = cldrFile.getStringValueWithBailey(entry.path); - assertEquals(entry.toString() + "\t" + rawValue, entry.value, resolvedValue); - } - } - - public TestValueSet setValuesIn(TestFactory testFactory) { - for (LocalePathValue entry : testValues) { - CLDRFile cldrFile = testFactory.make(entry.locale, true); - cldrFile.getUnresolved().add(entry.path, entry.value); - } - return this; - } - } - - public void testLateralInheritance() { -// { -// Multimap suspicious = getSuspicious(CLDRConfig.getInstance().getCLDRFile("fr_CA", true)); -// if (!suspicious.isEmpty()) { -// errln("fr_CA" + "\n\t" + Joiner.on("\n\t").join(suspicious.entries())); -// } -// } - - Map>> allCount = new LinkedHashMap<>(); - Map breakdownMap = new LinkedHashMap<>(); - - for (String ldmlDirectory : DtdType.ldml.directories) { - Map> localeSuspiciousCount = new LinkedHashMap<>(); - - final Factory factory = SimpleFactory.make(CLDRPaths.COMMON_DIRECTORY + ldmlDirectory, ".*"); - for (String locale : factory.getAvailable()) { - if (!"root".equals(LocaleIDParser.getParent(locale))) { // just L1 locales - continue; - } - - CLDRFile cldrFile = factory.make(locale, true); - Multimap suspicious = getSuspicious(cldrFile, breakdownMap); - if (!suspicious.isEmpty()) { - errln("\t" + locale + "\t" + suspicious.entries().size() - + "\n\t" + Joiner.on("\n\t").join(suspicious.entries())); - Counter c = new Counter<>(); - for (Entry> entry : suspicious.asMap().entrySet()) { - localeSuspiciousCount.put(locale, c.add(entry.getKey(), entry.getValue().size())); - } - } - } - if (!localeSuspiciousCount.isEmpty()) { - allCount.put(ldmlDirectory, localeSuspiciousCount); - } - } - for (Entry>> entry : allCount.entrySet()) { - String dir = entry.getKey(); - for (Entry> entry2 : entry.getValue().entrySet()) { - String locale = entry2.getKey(); - Counter c = entry2.getValue(); - for (R2 entry3 : c.getEntrySetSortedByCount(false, null)) { - System.out.println(dir + "\t" + locale + "\t" + entry3.get0() + "\t" + entry3.get1()); - } - } - } - CLDRFile english = config.getEnglish(); - System.out.println("\nlocale\tName\tTarget (Org=cldr)\t" + Breakdown.header()); - for (Entry entry : breakdownMap.entrySet()) { - final String locale = entry.getKey(); - Level target = StandardCodes.make().getLocaleCoverageLevel(Organization.cldr, locale); - String targetString = target.toString(); - switch(target) { - case COMPREHENSIVE: targetString = Level.MODERN.toString(); break; // TODO change once English coverage is fixed. - case UNDETERMINED: targetString = "*"+Level.BASIC.toString(); break; - } - // target == Level.UNDETERMINED ? "-" : - System.out.println(locale - + "\t" + english.getName(locale) - + "\t" + targetString - + "\t" + entry.getValue()); - } - - } - - private static final Joiner TAB_JOINER = Joiner.on('\t'); - - static final class Breakdown { - int notTargetNull; - int notTargetNotNull; - int targetNull; - int notInheritanceMarker; - int iMarkerSamePath; - int iMarkerDiffPathRoot; - int iMarkerDiffPathNotRoot; - @Override - public String toString() { - return TAB_JOINER.join(notTargetNull, notTargetNotNull, targetNull, notInheritanceMarker, iMarkerSamePath, iMarkerDiffPathRoot, iMarkerDiffPathNotRoot); - } - public static String header() { - return TAB_JOINER.join(">Target&Null", ">Target&!Null", "≤Target&Null", "!InheritanceMarker", "iMarker&=Path", "iMarker&≠Path&Root", "iMarker&≠Path&!Root"); - } - } - - static final class SuspiciousData implements Comparable { - final String pathRequested; - final String pathFound; - final String valueFound; - - public SuspiciousData(String pathRequested, String pathFound, String valueFound) { - this.pathRequested = pathRequested; - this.pathFound = pathFound; - this.valueFound = valueFound; - } - @Override - public int compareTo(SuspiciousData o) { - return ComparisonChain.start() - .compare(pathRequested, o.pathRequested) - .compare(pathFound, o.pathFound) - .compare(valueFound, o.valueFound) - .result(); - } - @Override - public String toString() { - return "\t" + valueFound + "\n\t\t" + pathRequested + "\n\t\t" + pathFound; - } - } - - public Multimap getSuspicious(CLDRFile cldrFile, Map breakdownMap) { - Multimap suspicious = TreeMultimap.create(); - if (!cldrFile.isResolved()) { - throw new IllegalArgumentException(); - } - CLDRFile unresolvedCldrFile = cldrFile.getUnresolved(); - String locale = cldrFile.getLocaleID(); - Level target = StandardCodes.make().getLocaleCoverageLevel(Organization.cldr, locale); - switch(target) { - case COMPREHENSIVE: target = Level.MODERN; break; // TODO change once English coverage is fixed. - case UNDETERMINED: target = Level.BASIC; break; - } - - Output foundPath = new Output<>(); - Output foundLocale = new Output<>(); - Breakdown breakdown = breakdownMap.get(locale); - if (breakdown == null) { - breakdownMap.put(locale, breakdown = new Breakdown()); - } - for (String path : cldrFile) { // TODO we only need to look at the actual items, not nulls. But right now we are catching them. - String unresolvedValue = unresolvedCldrFile.getStringValue(path); - - Level level = SupplementalDataInfo.getInstance().getCoverageLevel(path, locale); - if (target.compareTo(level) < 0) { - if (unresolvedValue == null) { - breakdown.notTargetNull++; - } else { - breakdown.notTargetNotNull++; - } - continue; // only worry about modern level paths - } - if (unresolvedValue == null) { - breakdown.targetNull++; - continue; - } - - if (!CldrUtility.INHERITANCE_MARKER.equals(unresolvedValue)) { - breakdown.notInheritanceMarker++; - continue; - } - cldrFile.getBaileyValue(path, foundPath, foundLocale); - if (path.equals(foundPath.value)) { - breakdown.iMarkerSamePath++; - // if we find it in the same path (vertical) we are ok - continue; - } - // at this point, it is horizontal (path) inheritance - // if there is nothing between us and a root value, then - - if ("root".equals(foundLocale.value) || "code-fallback".equals(foundLocale.value)) { - breakdown.iMarkerDiffPathRoot++; - // if we fall all the way back to root that's ok. - continue; - } - // otherwise - breakdown.iMarkerDiffPathNotRoot++; - suspicious.put(foundLocale.value, new SuspiciousData(path, foundPath.value, cldrFile.getStringValueWithBailey(path))); - } - return suspicious; - } }