From dc5c4c0a9d38858f0f41ac5304c83dd99c42c291 Mon Sep 17 00:00:00 2001 From: macchiati Date: Sat, 29 Jun 2024 20:14:39 -1000 Subject: [PATCH] CLDR-17756 Minor cleanup, allow multiple locales --- .../cldr/unittest/TestExampleGenerator.java | 164 +++++++++--------- 1 file changed, 83 insertions(+), 81 deletions(-) diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java index c9fbe34d8f5..f98e6699154 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestExampleGenerator.java @@ -1705,91 +1705,93 @@ public int hashCode() { .build(); public void TestForMissing() { - CLDRFile cldrFile = info.getEnglish(); - ExampleGenerator exampleGenerator = - new ExampleGenerator(info.getEnglish(), info.getEnglish()); - PathStarrer ps = new PathStarrer(); - ps.setSubstitutionPattern("*"); - Counter countWithExamples = new Counter<>(); - Map samplesForWith = new HashMap<>(); - Counter countWithoutExamples = new Counter<>(); - Multimap samplesForWithout = TreeMultimap.create(); - DtdData dtdData = DtdData.getInstance(DtdType.ldml); - PathHeader.Factory phf = PathHeader.getFactory(); - final String separator = "•"; - - for (String xpath : cldrFile.fullIterable()) { - if (xpath.endsWith("/alias")) { - continue; - } - final XPathParts parts = XPathParts.getFrozenInstance(xpath); - if (dtdData.isDeprecated(parts)) { - continue; - } - Level level = SDI.getCoverageLevel(xpath, "en"); - if (level.compareTo(Level.COMPREHENSIVE) == 0) { - continue; - } - String value = cldrFile.getStringValue(xpath); - String starred = ps.set(xpath); - String attrs = ps.getAttributesString(separator); - if (OK_IF_MISSING.containsEntry(starred, attrs)) { - logln("OK if missing: " + starred + ";\t" + attrs); - continue; - } - String example = null; - PathHeader ph = phf.fromPath(xpath); - String heading = ph.getSection() + " > " + ph.getPageId(); - SectionId section = ph.getSectionId(); - MissingKey key = new MissingKey(ph.getSectionId(), ph.getPageId(), starred); - try { - example = exampleGenerator.getExampleHtml(xpath, value); - } catch (Exception e) { + Factory factory = info.getCldrFactory(); // don't worry about examples for annotations + for (String localeId : List.of("en", "de")) { + CLDRFile cldrFile = factory.make(localeId, true); + ExampleGenerator exampleGenerator = new ExampleGenerator(cldrFile, info.getEnglish()); + PathStarrer ps = new PathStarrer(); + ps.setSubstitutionPattern("*"); + Counter countWithExamples = new Counter<>(); + Map samplesForWith = new HashMap<>(); + Counter countWithoutExamples = new Counter<>(); + Multimap samplesForWithout = TreeMultimap.create(); + DtdData dtdData = DtdData.getInstance(DtdType.ldml); + PathHeader.Factory phf = PathHeader.getFactory(); + final String separator = "•"; + + // collect all of the strings in the file to test + for (String xpath : cldrFile.fullIterable()) { + if (xpath.endsWith("/alias")) { + continue; + } + final XPathParts parts = XPathParts.getFrozenInstance(xpath); + if (dtdData.isDeprecated(parts)) { + continue; + } + Level level = SDI.getCoverageLevel(xpath, "en"); + if (level.compareTo(Level.COMPREHENSIVE) == 0) { + continue; + } + String value = cldrFile.getStringValue(xpath); + String starred = ps.set(xpath); + String attrs = ps.getAttributesString(separator); + if (OK_IF_MISSING.containsEntry(starred, attrs)) { + logln("OK if missing: " + starred + ";\t" + attrs); + continue; + } + String example = null; + PathHeader ph = phf.fromPath(xpath); + String heading = ph.getSection() + " > " + ph.getPageId(); + SectionId section = ph.getSectionId(); + MissingKey key = new MissingKey(ph.getSectionId(), ph.getPageId(), starred); + try { + example = exampleGenerator.getExampleHtml(xpath, value); + } catch (Exception e) { + } + if (example == null) { + samplesForWithout.put(key, sampleAttrAndValue(ps, separator, value)); + countWithoutExamples.add(key, 1); + } else { + if (!samplesForWith.containsKey(key)) { + samplesForWith.put(key, sampleAttrAndValue(ps, separator, value)); + } + countWithExamples.add(key, 1); + } } - if (example == null) { - samplesForWithout.put(key, sampleAttrAndValue(ps, separator, value)); - countWithoutExamples.add(key, 1); - } else { - if (!samplesForWith.containsKey(key)) { - samplesForWith.put(key, sampleAttrAndValue(ps, separator, value)); + Set keys = new TreeSet<>(); + keys.addAll(countWithoutExamples.keySet()); + keys.addAll(countWithExamples.keySet()); + List missingItems = new ArrayList<>(); + for (MissingKey key : keys) { + final long countWithout = countWithoutExamples.get(key); + if (countWithout == 0) { // ok, no missing + continue; } - countWithExamples.add(key, 1); + final Collection sampleForWithoutItem = samplesForWithout.get(key); + final String sampleForWithItem = samplesForWith.get(key); + final long countWith = countWithExamples.get(key); + final double doneRatio = countWith / (double) (countWith + countWithout); + missingItems.add( + TAB_JOINER.join( + doneRatio, + countWithout, + (sampleForWithItem == null + ? sampleForWithoutItem.iterator().next() + : Joiner.on("; ") + .join(Iterables.limit(sampleForWithoutItem, 5))), + countWith, + (sampleForWithItem == null ? "n/a" : sampleForWithItem), + key.sectionId, + key.pageId, + key.starred)); } - } - Set keys = new TreeSet<>(); - keys.addAll(countWithoutExamples.keySet()); - keys.addAll(countWithExamples.keySet()); - List missingItems = new ArrayList<>(); - for (MissingKey key : keys) { - final long countWithout = countWithoutExamples.get(key); - if (countWithout == 0) { // ok, no missing - continue; + if (!missingItems.isEmpty()) { + errln( + TAB_JOINER.join(localeId, "missing examples:", missingItems.size()) + + "\n" + + "\nDone?\tWithout\tSample Attrs\tWith\tSample Attrs\tSection\tPage\tStarred Pattern\n" + + Joiner.on("\n").join(missingItems)); } - final Collection sampleForWithoutItem = samplesForWithout.get(key); - final String sampleForWithItem = samplesForWith.get(key); - final long countWith = countWithExamples.get(key); - final double doneRatio = countWith / (double) (countWith + countWithout); - missingItems.add( - TAB_JOINER.join( - doneRatio, - countWithout, - (sampleForWithItem == null - ? sampleForWithoutItem.iterator().next() - : Joiner.on("; ") - .join(Iterables.limit(sampleForWithoutItem, 5))), - countWith, - (sampleForWithItem == null ? "n/a" : sampleForWithItem), - key.sectionId, - key.pageId, - key.starred)); - } - if (!missingItems.isEmpty()) { - errln( - "Missing Examples:\t" - + missingItems.size() - + "\n" - + "\nDone?\tWithout\tSample Attrs\tWith\tSample Attrs\tSection\tPage\tStarred Pattern\n" - + Joiner.on("\n").join(missingItems)); } }