From da3727f931da95597d7f62c6fa6f029b952367c9 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 14 May 2024 19:10:21 -0500 Subject: [PATCH] CLDR-17634 Verify that all prior-Basic locale names have English and Coverage (#3720) --- .../org/unicode/cldr/tool/ToolConstants.java | 5 ++ .../cldr/util/CalculatedCoverageLevels.java | 15 +++- .../java/org/unicode/cldr/util/Level.java | 7 ++ .../unicode/cldr/util/TestCoverageLevel2.java | 79 +++++++++++++++++++ .../java/org/unicode/cldr/util/TestLevel.java | 9 +++ 5 files changed, 113 insertions(+), 2 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ToolConstants.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ToolConstants.java index dffe1aae042..510edfc3c4b 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ToolConstants.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ToolConstants.java @@ -62,6 +62,11 @@ public static VersionInfo previousVersion(VersionInfo version) { return last; } + /** Version prior to the current DEV version */ + public static VersionInfo previousVersion() { + return previousVersion(DEV_VERSION_VI); + } + public static String previousVersion(String version, int minFields) { VersionInfo result = previousVersion(VersionInfo.getInstance(version)); return result.getVersionString(minFields, 2); diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CalculatedCoverageLevels.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CalculatedCoverageLevels.java index 479208f6644..abd9ecc14d7 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CalculatedCoverageLevels.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CalculatedCoverageLevels.java @@ -1,6 +1,7 @@ package org.unicode.cldr.util; import com.google.common.base.Splitter; +import com.ibm.icu.util.VersionInfo; import java.io.BufferedReader; import java.io.IOException; import java.util.Collections; @@ -8,11 +9,13 @@ import java.util.Map; import java.util.TreeMap; import org.unicode.cldr.draft.FileUtilities; +import org.unicode.cldr.tool.ToolConstants; public class CalculatedCoverageLevels { /** Assumed level for root. CLDR-16420 */ private static final Level DEFAULT_ROOT_LEVEL = Level.MODERN; + /** locale to level */ final Map levels; protected CalculatedCoverageLevels(Map levels) { @@ -63,13 +66,21 @@ public boolean isLocaleAtLeastBasic(String locale) { /** Read the coverage levels from the standard file */ static CalculatedCoverageLevels fromFile() throws IOException { + return fromFile(CLDRPaths.COMMON_DIRECTORY); + } + + /** Read the coverage levels from the specified dir */ + static CalculatedCoverageLevels fromFile(final String dir) throws IOException { try (BufferedReader r = - FileUtilities.openUTF8Reader( - CLDRPaths.COMMON_DIRECTORY + "properties/", "coverageLevels.txt"); ) { + FileUtilities.openUTF8Reader(dir + "properties/", "coverageLevels.txt"); ) { return fromReader(r); } } + static CalculatedCoverageLevels forVersion(VersionInfo v) throws IOException { + return fromFile(ToolConstants.getBaseDirectory(v) + "/common/"); + } + /** * read the coverage levels from a BufferedReader * diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/Level.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/Level.java index 65086344aab..f0d3562b28e 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/Level.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/Level.java @@ -163,4 +163,11 @@ public static Level max(Level... levels) { public boolean isAtLeast(Level other) { return getLevel() >= other.getLevel(); } + + /** + * @return true if this is > other + */ + public boolean isAbove(Level other) { + return getLevel() > other.getLevel(); + } } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCoverageLevel2.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCoverageLevel2.java index c425e94fc58..d7c4a43de8a 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCoverageLevel2.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCoverageLevel2.java @@ -1,10 +1,22 @@ package org.unicode.cldr.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import com.ibm.icu.util.VersionInfo; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.unicode.cldr.icu.LDMLConstants; import org.unicode.cldr.test.CoverageLevel2; +import org.unicode.cldr.tool.ToolConstants; +import org.unicode.cldr.util.StandardCodes.CodeType; public class TestCoverageLevel2 { @@ -28,4 +40,71 @@ public void TestCoveragePerf() { "//ldml/characters/parseLenients[@scope=\"number\"][@level=\"lenient\"]/parseLenient[@sample=\",\"]")); } } + + @Test + public void TestPriorBasicLanguage() throws IOException { + // Fail if the language name is at above this level + final Level failIfAbove = Level.MODERN; + + // we need the CLDR Archive dir for this. + assumeTrue(TestCLDRPaths.canUseArchiveDirectory()); + + // Previous CLDR version + final VersionInfo prev = ToolConstants.previousVersion(); + // read coverageLevels.txt from the *previous* version + final CalculatedCoverageLevels prevCovLevel = CalculatedCoverageLevels.forVersion(prev); + // Our xpath: the language leaf + final XPathParts xpp = + XPathParts.getFrozenInstance("//ldml/localeDisplayNames/languages/language") + .cloneAsThawed(); + // CLDR English File + final CLDRFile english = CLDRConfig.getInstance().getEnglish(); + + // Result: locales not in en.xml + final Set notInEnglish = new TreeSet<>(); + // Result: locales not in coverage + final Set notInCoverage = new TreeSet<>(); + + final Set localesToCheck = + SupplementalDataInfo.getInstance().getLanguageTcOrBasic(); + final Map covs = new HashMap<>(); + + for (final String lang : localesToCheck) { + covs.put(lang, CoverageLevel2.getInstance(sdi, lang)); + } + + for (final String lang : StandardCodes.make().getAvailableCodes(CodeType.language)) { + if (prevCovLevel.isLocaleAtLeastBasic(lang)) { + xpp.setAttribute(-1, LDMLConstants.TYPE, lang); + final String xpath = xpp.toString(); + + if (!english.isHere(xpath.toString())) { + // fail if not in English + notInEnglish.add(lang); + } + + if (covs.values().stream() + .anyMatch((cov) -> cov.getLevel(xpath.toString()).isAbove(failIfAbove))) { + // fail if level > failIfAbove for any of those locales + notInCoverage.add(lang); + } + } + } + + Assertions.assertAll( + () -> + assertTrue( + notInEnglish.isEmpty(), + () -> + "en.xml is missing translations for these languages' names:" + + notInEnglish.toString()), + () -> + assertTrue( + notInCoverage.isEmpty(), + () -> + "coverageLevels.xml has a coverage level >" + + failIfAbove + + " for these language's names:" + + notInCoverage.toString())); + } } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestLevel.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestLevel.java index fc7a2f287b0..425fd6f1bb9 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestLevel.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestLevel.java @@ -1,6 +1,7 @@ package org.unicode.cldr.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -8,6 +9,7 @@ import java.util.TreeSet; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; @@ -103,4 +105,11 @@ public void testModernCurrencies(final String code) { "Coverage for modern currency %s: %s, expected ≤ %s", code, l, expect)); } + + @Test + public void TestMath() { + assertTrue(Level.MODERN.isAbove(Level.MODERATE)); + assertFalse(Level.BASIC.isAtLeast(Level.MODERN)); + assertTrue(Level.MODERN.isAtLeast(Level.BASIC)); + } }