CLDR-11155 Pages too big: add test for large pages (#3563)

unicode-org · Mar 12, 2024 · cd0e83b · cd0e83b
1 parent 7859f93
commit cd0e83b
Showing 1 changed file with 68 additions and 0 deletions.
diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java
@@ -26,11 +26,13 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
+import java.util.stream.Collectors;
 import org.unicode.cldr.test.CoverageLevel2;
 import org.unicode.cldr.test.ExampleGenerator;
 import org.unicode.cldr.util.CLDRConfig;
 import org.unicode.cldr.util.CLDRFile;
 import org.unicode.cldr.util.CLDRFile.Status;
+import org.unicode.cldr.util.CLDRLocale;
 import org.unicode.cldr.util.CLDRPaths;
 import org.unicode.cldr.util.CLDRURLS;
 import org.unicode.cldr.util.CldrUtility;
@@ -1677,4 +1679,70 @@ public void TestUnitOrder() {
             }
         }
     }
+
+    public void testPageSize() {
+        final long minError = 946; // above this, emit error
+        final long minLog = 700; // otherwise above this, emit warning
+        Factory factory = CLDRConfig.getInstance().getCommonAndSeedAndMainAndAnnotationsFactory();
+        // "en", "cs", "ar", "pl"
+        List<String> locales =
+                StandardCodes.make()
+                        .getLocaleCoverageLocales(Organization.cldr, ImmutableSet.of(Level.MODERN))
+                        .stream()
+                        .filter(x -> CLDRLocale.getInstance(x).getCountry().isEmpty())
+                        .collect(Collectors.toUnmodifiableList());
+        List<Counter<PageId>> counters = new ArrayList<>();
+        for (String locale : locales) {
+            CLDRFile cldrFile = factory.make(locale, false);
+            PathHeader.Factory phf = PathHeader.getFactory();
+            Counter<PageId> c = new Counter<>();
+            counters.add(c);
+            for (String path : cldrFile) {
+                PathHeader ph = phf.fromPath(path);
+                c.add(ph.getPageId(), 1);
+            }
+            for (PageId entry : c.getKeysetSortedByKey()) {
+                long count = c.getCount(entry);
+                if (count > minError) {
+                    errln(
+                            locale
+                                    + "\t"
+                                    + entry.getSectionId()
+                                    + "\t"
+                                    + entry
+                                    + "\thas too many entries:\t"
+                                    + count);
+                } else if (count > minLog) {
+                    warnln(
+                            locale
+                                    + "\t"
+                                    + entry.getSectionId()
+                                    + "\t"
+                                    + "\thas too many entries:\t"
+                                    + count);
+                }
+            }
+        }
+        if (isVerbose()) {
+            System.out.println();
+            Set<PageId> sorted = new TreeSet<>();
+            for (Counter<PageId> counter : counters) {
+                sorted.addAll(counter.keySet());
+            }
+            int i = 0;
+            System.out.print("Order" + "\t" + "Section" + "\t" + "Page");
+            for (String c : locales) {
+                System.out.print("\t" + c);
+            }
+            System.out.println();
+
+            for (PageId entry : sorted) {
+                System.out.print(++i + "\t" + entry.getSectionId() + "\t" + entry);
+                for (Counter<PageId> c : counters) {
+                    System.out.print("\t" + c.get(entry));
+                }
+                System.out.println();
+            }
+        }
+    }
 }