From 2a2dc139964f1a6edc6dd9d624d4e1662863809b Mon Sep 17 00:00:00 2001 From: Tom Bishop Date: Wed, 13 Sep 2023 09:52:38 -0400 Subject: [PATCH] CLDR-16208 Default number systems in Arabic locales, double-check after generation (#3271) -New methods doubleCheckSpecialPaths, doubleCheckLocale check the destination folder contents -Throw an exception in the case of failure -New methods directoryIsSpecial, localeIsSpecial preserve encapsulation of number systems, Arabic -New method getLocaleIdFromFileName encapsulates getting locale ID from xml file name -Comments --- .../cldr/tool/GenerateProductionData.java | 93 ++++++++++++++++--- 1 file changed, 81 insertions(+), 12 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateProductionData.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateProductionData.java index d21574afd92..9cf5504e4c3 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateProductionData.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateProductionData.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -33,6 +34,7 @@ import org.unicode.cldr.util.DtdType; import org.unicode.cldr.util.Factory; import org.unicode.cldr.util.GlossonymConstructor; +import org.unicode.cldr.util.InternalCldrException; import org.unicode.cldr.util.Level; import org.unicode.cldr.util.LocaleIDParser; import org.unicode.cldr.util.LocaleNames; @@ -160,6 +162,8 @@ public static void main(String[] args) { // get directories + Map specialDirectories = new TreeMap<>(); + Arrays.asList(DtdType.values()) // .parallelStream() // .unordered() @@ -178,8 +182,16 @@ public static void main(String[] args) { Stats stats = new Stats(); copyFilesAndReturnIsEmpty( sourceDir, destinationDir, null, isLdmlDtdType, stats); + if (directoryIsSpecial(sourceDir.getAbsolutePath())) { + specialDirectories.put(sourceDir, destinationDir); + } } }); + + for (File source : specialDirectories.keySet()) { + File dest = specialDirectories.get(source); + doubleCheckSpecialPaths(source, dest); + } } private static class Stats { @@ -222,7 +234,6 @@ public void showNonZero(String label) { * @param factory * @param isLdmlDtdType * @param stats - * @param hasChildren * @return true if the file is an ldml file with empty content. */ private static boolean copyFilesAndReturnIsEmpty( @@ -287,9 +298,7 @@ private static boolean copyFilesAndReturnIsEmpty( isLdmlDtdType2, stats2); if (isEmpty) { // only happens for ldml - emptyLocales.add( - file.substring( - 0, file.length() - 4)); // remove .xml for localeId + emptyLocales.add(getLocaleIdFromFileName(file)); } }); stats2.showNonZero("\tTOTAL:\t"); @@ -312,7 +321,7 @@ private static boolean copyFilesAndReturnIsEmpty( if (!file.endsWith(".xml")) { return false; } - String localeId = file.substring(0, file.length() - 4); + String localeId = getLocaleIdFromFileName(file); if (FILE_MATCH != null) { if (!FILE_MATCH.reset(localeId).matches()) { return false; @@ -427,7 +436,7 @@ private static boolean copyFilesAndReturnIsEmpty( gotOne = true; } if (specialPathsAreRequired) { - addSpecialPathsIfMissing(localeId, toRetainSpecial, cldrFileResolved); + addSpecialPathsIfMissing(toRetainSpecial); } // we even add empty files, but can delete them back on the directory level. @@ -540,6 +549,10 @@ private static boolean copyFilesAndReturnIsEmpty( } } + private static String getLocaleIdFromFileName(String file) { + return file.substring(0, file.length() - 4); // drop ".xml" + } + /** * Are any "special paths" required to be explicitly included for this locale, in this * directory? @@ -548,13 +561,21 @@ private static boolean copyFilesAndReturnIsEmpty( * common/main * * @param localeId the locale ID such as "ar_KM" - * @param directory the string describing the output directory; currently only "common/main" has + * @param directory the string describing the source directory; currently only "common/main" has * special paths * @return true if required, else false */ private static boolean areSpecialPathsRequired(String localeId, String directory) { - return (localeId.equals("ar") || localeId.startsWith("ar_")) - && directory.contains("common/main"); + + return localeIsSpecial(localeId) && directoryIsSpecial(directory); + } + + private static boolean directoryIsSpecial(String directory) { + return directory.contains("common/main"); + } + + private static boolean localeIsSpecial(String localeId) { + return localeId.equals("ar") || localeId.startsWith("ar_"); } private static final String[] SPECIAL_PATHS = @@ -574,8 +595,7 @@ private static boolean pathIsSpecial(String xpath) { return SPECIAL_PATH_SET.contains(xpath); } - private static void addSpecialPathsIfMissing( - String localeId, Set toRetainSpecial, CLDRFile cldrFileResolved) { + private static void addSpecialPathsIfMissing(Set toRetainSpecial) { for (String xpath : SPECIAL_PATH_SET) { if (!toRetainSpecial.contains(xpath)) { toRetainSpecial.add(xpath); @@ -583,6 +603,55 @@ private static void addSpecialPathsIfMissing( } } + /** + * Confirm that a file (in the destination) is present for each "special" locale (in the + * source(), and that each such destination file contains all the required "special" paths + * + * @param sourceDir a directory like ".../common/main" + * @param destDir a directory + */ + private static void doubleCheckSpecialPaths(File sourceDir, File destDir) { + Set sorted = new TreeSet<>(); + sorted.addAll(Arrays.asList(sourceDir.list())); + Factory factory = Factory.make(destDir.toString(), ".*"); + sorted.forEach( + file -> { + doubleCheckLocale(sourceDir, destDir, file, factory); + }); + } + + private static void doubleCheckLocale( + File sourceDir, File destDir, String file, Factory factory) { + if (!file.endsWith(".xml")) { + return; + } + String localeId = getLocaleIdFromFileName(file); + if (!localeIsSpecial(localeId)) { + return; + } + File destFile = new File(destDir, file); + if (!destFile.exists()) { + throw new InternalCldrException("doubleCheckLocale FILE NOT FOUND: " + destFile); + } + // Note: factory.make will fail here unless ../common/dtd/ldml.dtd exists in relation to the + // destination folder + CLDRFile cldrFileUnresolved = factory.make(localeId, false); + for (String xpath : SPECIAL_PATH_SET) { + String value = cldrFileUnresolved.getStringValue(xpath); + if (value == null) { + throw new InternalCldrException( + "Locale " + localeId + " missing required special path " + xpath); + } + if (CldrUtility.INHERITANCE_MARKER.equals(value)) { + throw new InternalCldrException( + "Locale " + + localeId + + " has INHERITANCE_MARKER for required special path " + + xpath); + } + } + } + private static void showIfNonZero(String localeId, String title, Set toRemove) { if (toRemove.size() != 0) { System.out.println(localeId + ": " + title + ": " + toRemove.size()); @@ -757,7 +826,7 @@ private static Set getChildless( /** * Recursively checks that all children are empty (including that there are no children) * - * @param name + * @param locale * @param emptyLocales * @param parent2child * @return