CLDR-16208 Default number systems in Arabic locales, double-check aft…

…er generation (#3271)

-New methods doubleCheckSpecialPaths, doubleCheckLocale check the destination folder contents

-Throw an exception in the case of failure

-New methods directoryIsSpecial, localeIsSpecial preserve encapsulation of number systems, Arabic

-New method getLocaleIdFromFileName encapsulates getting locale ID from xml file name

-Comments

tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateProductionData.java

@@ -20,6 +20,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -33,6 +34,7 @@
 import org.unicode.cldr.util.DtdType;
 import org.unicode.cldr.util.Factory;
 import org.unicode.cldr.util.GlossonymConstructor;
+import org.unicode.cldr.util.InternalCldrException;
 import org.unicode.cldr.util.Level;
 import org.unicode.cldr.util.LocaleIDParser;
 import org.unicode.cldr.util.LocaleNames;
@@ -160,6 +162,8 @@ public static void main(String[] args) {
 
         // get directories
 
+        Map<File, File> specialDirectories = new TreeMap<>();
+
         Arrays.asList(DtdType.values())
                 // .parallelStream()
                 // .unordered()
@@ -178,8 +182,16 @@ public static void main(String[] args) {
                                 Stats stats = new Stats();
                                 copyFilesAndReturnIsEmpty(
                                         sourceDir, destinationDir, null, isLdmlDtdType, stats);
+                                if (directoryIsSpecial(sourceDir.getAbsolutePath())) {
+                                    specialDirectories.put(sourceDir, destinationDir);
+                                }
                             }
                         });
+
+        for (File source : specialDirectories.keySet()) {
+            File dest = specialDirectories.get(source);
+            doubleCheckSpecialPaths(source, dest);
+        }
     }
 
     private static class Stats {
@@ -222,7 +234,6 @@ public void showNonZero(String label) {
      * @param factory
      * @param isLdmlDtdType
      * @param stats
-     * @param hasChildren
      * @return true if the file is an ldml file with empty content.
      */
     private static boolean copyFilesAndReturnIsEmpty(
@@ -287,9 +298,7 @@ private static boolean copyFilesAndReturnIsEmpty(
                                         isLdmlDtdType2,
                                         stats2);
                         if (isEmpty) { // only happens for ldml
-                            emptyLocales.add(
-                                    file.substring(
-                                            0, file.length() - 4)); // remove .xml for localeId
+                            emptyLocales.add(getLocaleIdFromFileName(file));
                         }
                     });
             stats2.showNonZero("\tTOTAL:\t");
@@ -312,7 +321,7 @@ private static boolean copyFilesAndReturnIsEmpty(
             if (!file.endsWith(".xml")) {
                 return false;
             }
-            String localeId = file.substring(0, file.length() - 4);
+            String localeId = getLocaleIdFromFileName(file);
             if (FILE_MATCH != null) {
                 if (!FILE_MATCH.reset(localeId).matches()) {
                     return false;
@@ -427,7 +436,7 @@ private static boolean copyFilesAndReturnIsEmpty(
                 gotOne = true;
             }
             if (specialPathsAreRequired) {
-                addSpecialPathsIfMissing(localeId, toRetainSpecial, cldrFileResolved);
+                addSpecialPathsIfMissing(toRetainSpecial);
             }
 
             // we even add empty files, but can delete them back on the directory level.
@@ -540,6 +549,10 @@ private static boolean copyFilesAndReturnIsEmpty(
         }
     }
 
+    private static String getLocaleIdFromFileName(String file) {
+        return file.substring(0, file.length() - 4); // drop ".xml"
+    }
+
     /**
      * Are any "special paths" required to be explicitly included for this locale, in this
      * directory?
@@ -548,13 +561,21 @@ private static boolean copyFilesAndReturnIsEmpty(
      * common/main
      *
      * @param localeId the locale ID such as "ar_KM"
-     * @param directory the string describing the output directory; currently only "common/main" has
+     * @param directory the string describing the source directory; currently only "common/main" has
      *     special paths
      * @return true if required, else false
      */
     private static boolean areSpecialPathsRequired(String localeId, String directory) {
-        return (localeId.equals("ar") || localeId.startsWith("ar_"))
-                && directory.contains("common/main");
+
+        return localeIsSpecial(localeId) && directoryIsSpecial(directory);
+    }
+
+    private static boolean directoryIsSpecial(String directory) {
+        return directory.contains("common/main");
+    }
+
+    private static boolean localeIsSpecial(String localeId) {
+        return localeId.equals("ar") || localeId.startsWith("ar_");
     }
 
     private static final String[] SPECIAL_PATHS =
@@ -574,15 +595,63 @@ private static boolean pathIsSpecial(String xpath) {
         return SPECIAL_PATH_SET.contains(xpath);
     }
 
-    private static void addSpecialPathsIfMissing(
-            String localeId, Set<String> toRetainSpecial, CLDRFile cldrFileResolved) {
+    private static void addSpecialPathsIfMissing(Set<String> toRetainSpecial) {
         for (String xpath : SPECIAL_PATH_SET) {
             if (!toRetainSpecial.contains(xpath)) {
                 toRetainSpecial.add(xpath);
             }
         }
     }
 
+    /**
+     * Confirm that a file (in the destination) is present for each "special" locale (in the
+     * source(), and that each such destination file contains all the required "special" paths
+     *
+     * @param sourceDir a directory like ".../common/main"
+     * @param destDir a directory
+     */
+    private static void doubleCheckSpecialPaths(File sourceDir, File destDir) {
+        Set<String> sorted = new TreeSet<>();
+        sorted.addAll(Arrays.asList(sourceDir.list()));
+        Factory factory = Factory.make(destDir.toString(), ".*");
+        sorted.forEach(
+                file -> {
+                    doubleCheckLocale(sourceDir, destDir, file, factory);
+                });
+    }
+
+    private static void doubleCheckLocale(
+            File sourceDir, File destDir, String file, Factory factory) {
+        if (!file.endsWith(".xml")) {
+            return;
+        }
+        String localeId = getLocaleIdFromFileName(file);
+        if (!localeIsSpecial(localeId)) {
+            return;
+        }
+        File destFile = new File(destDir, file);
+        if (!destFile.exists()) {
+            throw new InternalCldrException("doubleCheckLocale FILE NOT FOUND: " + destFile);
+        }
+        // Note: factory.make will fail here unless ../common/dtd/ldml.dtd exists in relation to the
+        // destination folder
+        CLDRFile cldrFileUnresolved = factory.make(localeId, false);
+        for (String xpath : SPECIAL_PATH_SET) {
+            String value = cldrFileUnresolved.getStringValue(xpath);
+            if (value == null) {
+                throw new InternalCldrException(
+                        "Locale " + localeId + " missing required special path " + xpath);
+            }
+            if (CldrUtility.INHERITANCE_MARKER.equals(value)) {
+                throw new InternalCldrException(
+                        "Locale "
+                                + localeId
+                                + " has INHERITANCE_MARKER for required special path "
+                                + xpath);
+            }
+        }
+    }
+
     private static void showIfNonZero(String localeId, String title, Set<String> toRemove) {
         if (toRemove.size() != 0) {
             System.out.println(localeId + ": " + title + ": " + toRemove.size());
@@ -757,7 +826,7 @@ private static Set<String> getChildless(
     /**
      * Recursively checks that all children are empty (including that there are no children)
      *
-     * @param name
+     * @param locale
      * @param emptyLocales
      * @param parent2child
      * @return