diff --git a/common/main/root.xml b/common/main/root.xml
index 9bacdb25e07..3e4cf2afbe4 100644
--- a/common/main/root.xml
+++ b/common/main/root.xml
@@ -453,7 +453,6 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
U MMM
U MMM d
r(U)
- r-MM-dd
r(U)
r-MM
r-MM-dd
diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/SearchCLDR.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/SearchCLDR.java
index a10791a185e..61ea8b79c57 100644
--- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/SearchCLDR.java
+++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/SearchCLDR.java
@@ -3,6 +3,10 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+import com.google.common.collect.TreeMultimap;
+import com.ibm.icu.text.DateTimePatternGenerator;
+import com.ibm.icu.text.DateTimePatternGenerator.FormatParser;
+import com.ibm.icu.text.DateTimePatternGenerator.VariableField;
import com.ibm.icu.util.ICUUncheckedIOException;
import com.ibm.icu.util.Output;
import com.ibm.icu.util.VersionInfo;
@@ -27,6 +31,7 @@
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRFile.Status;
+import org.unicode.cldr.util.CLDRLocale;
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.Counter;
import org.unicode.cldr.util.Factory;
@@ -38,6 +43,7 @@
import org.unicode.cldr.util.PatternCache;
import org.unicode.cldr.util.SimpleFactory;
import org.unicode.cldr.util.StandardCodes;
+import org.unicode.cldr.util.XPathParts;
public class SearchCLDR {
// private static final int
@@ -77,6 +83,8 @@ public class SearchCLDR {
// + "-s\t show English value"
// ;
+ private static final CLDRConfig CONFIG = CLDRConfig.getInstance();
+
enum PathStyle {
none,
path,
@@ -110,6 +118,11 @@ enum PathStyle {
.add("resolved", null, null, "use resolved locales")
.add("q-showParent", null, null, "show parent value")
.add("english", null, null, "show english value")
+ .add(
+ "RootUncovered" + "",
+ null,
+ "false",
+ "filter to items that are in root but not overriden")
.add("Verbose", null, null, "verbose output")
.add(
"PathStyle",
@@ -135,6 +148,7 @@ enum PathStyle {
private static boolean showSurveyToolUrl;
private static Subtype subtype;
private static CheckCLDR checkCldr;
+ private static boolean rootUncovered;
static PathHeader.Factory pathHeaderFactory = PathHeader.getFactory();
@@ -160,16 +174,22 @@ public static void main(String[] args) {
Boolean valueExclude = exclude.value;
countOnly = myOptions.get("count").doesOccur();
- boolean resolved = myOptions.get("resolved").doesOccur();
+ final boolean resolved = myOptions.get("resolved").doesOccur();
showPath = myOptions.get("z-showPath").doesOccur();
String orgString = myOptions.get("organization").getValue();
Organization organization = orgString == null ? null : Organization.fromString(orgString);
- final CLDRFile english = CLDRConfig.getInstance().getEnglish();
+ final CLDRFile english = CONFIG.getEnglish();
showPathHeader = PathStyle.valueOf(myOptions.get("PathStyle").getValue());
+ rootUncovered = myOptions.get("RootUncovered").doesOccur();
+ if (rootUncovered && resolved) {
+ throw new IllegalArgumentException(
+ "Doesn't make sense to have both rootUncovered && resolved");
+ }
+
showSurveyToolUrl = myOptions.get("SurveyTool").doesOccur();
boolean showParent = myOptions.get("q-showParent").doesOccur();
@@ -236,8 +256,15 @@ public static void main(String[] args) {
Map options = new HashMap<>();
int totalCount = 0;
+ final CLDRFile ROOT = CONFIG.getRoot();
for (String locale : locales) {
+ if (rootUncovered) { // only look at locales with parent=root)
+ CLDRLocale clocale = CLDRLocale.getInstance(locale);
+ if (!clocale.isParentRoot()) {
+ continue;
+ }
+ }
int localeCount = 0;
// Level organizationLevel = organization == null ? null
// : StandardCodes.make().getLocaleCoverageLevel(organization, locale);
@@ -274,14 +301,32 @@ public static void main(String[] args) {
level = CoverageLevel2.getInstance(locale);
Status status = new Status();
Set sorted = new TreeSet<>();
- for (String path : file.fullIterable()) {
- if (locale.equals("eo") && path.contains("type=\"MK\"")) {
+ final Iterable pathSource =
+ rootUncovered ? (Iterable) ROOT : file.fullIterable();
+ RelatedPaths relatedPathsWithNonNullValues = new RelatedPaths();
+
+ for (String path : pathSource) {
+ if (path.contains("yMd") && path.contains("chinese")) {
int debug = 0;
}
+ if (pathMatcher != null && !pathMatcher.find(path)) {
+ continue;
+ }
String stringValue = file.getStringValue(path);
- if (stringValue == null) {
+ if (rootUncovered) {
+ if (stringValue != null) {
+ // Record any cases where there are non-null, non-inherited values
+ if (!stringValue.equals("↑↑↑")) {
+ relatedPathsWithNonNullValues.addRelated(path);
+ }
+ continue;
+ }
+ // we will add the path if the value for this path is null
+ // (uncovered in this file)
+ } else if (stringValue == null) {
continue;
}
+
String diffStringValue;
if (diffFile != null) {
diffStringValue = diffFile.getWinningValueWithBailey(path);
@@ -300,18 +345,26 @@ public static void main(String[] args) {
}
sorted.add(pathHeaderFactory.fromPath(path));
}
+
for (PathHeader pathHeader : sorted) {
String path = pathHeader.getOriginalPath();
+ String relatedNonNullKey = null;
+ if (rootUncovered) {
+ // We've collected paths from root that are uncovered in this file.
+ // That means they have a null value in this file, but non-null in root.
+ // We want to skip away all of those UNLESS there is a related path in this file
+ // that has a non-null value.
+ relatedNonNullKey = relatedPathsWithNonNullValues.hasKeyFor(path);
+ if (relatedNonNullKey == null) {
+ continue;
+ }
+ }
String fullPath = file.getFullXPath(path);
String value = file.getStringValue(path);
if (locale.equals("eo") && path.contains("type=\"MK\"")) {
int debug = 0;
}
- if (pathMatcher != null && !pathMatcher.find(fullPath)) {
- continue;
- }
-
{
pathLevel = level.getLevel(path);
levelCounter.add(pathLevel, 1);
@@ -409,6 +462,19 @@ public static void main(String[] args) {
null,
null);
} else {
+ String extra =
+ !rootUncovered
+ ? ""
+ : "\t"
+ + relatedNonNullKey
+ + "\t"
+ + XPathParts.getFrozenInstance(path)
+ .getAttributeValue(-1, "id")
+ + " → “"
+ + ROOT.getStringValue(path)
+ + "”\t"
+ + relatedPathsWithNonNullValues.show(
+ file, relatedNonNullKey);
showLine(
showPathHeader,
showParent,
@@ -421,7 +487,7 @@ public static void main(String[] args) {
!showParent ? null : english.getBaileyValue(path, null, null),
english == null ? null : english.getStringValue(path),
resolvedSource,
- Objects.toString(pathLevel));
+ Objects.toString(pathLevel) + extra);
}
totalCount++;
localeCount++;
@@ -446,6 +512,64 @@ public static void main(String[] args) {
+ " minutes");
}
+ /**
+ * Related with related values that are not null. NOTE: For now this is quite specific to
+ * availableFormats
+ */
+ static class RelatedPaths {
+ TreeMultimap skeletaToRelatedPathWithValue = TreeMultimap.create();
+ static final FormatParser parser = new DateTimePatternGenerator.FormatParser();
+
+ String getKey(String path) {
+ // ldml/dates/calendars/calendar[@type="chinese"]/dateTimeFormats/availableFormats/dateFormatItem[@id="d"]
+ XPathParts parts = XPathParts.getFrozenInstance(path);
+ if (parts.size() != 7 || !"availableFormats".equals(parts.getElement(5))) {
+ return null;
+ }
+ return parts.getAttributeValue(3, "type")
+ + "|"
+ + simplePattern(parts.getAttributeValue(-1, "id"));
+ }
+
+ public String hasKeyFor(String path) {
+ String key = getKey(path);
+ return skeletaToRelatedPathWithValue.containsKey(key) ? key : null;
+ }
+
+ public String show(CLDRFile file, String key) {
+ final List sorted = new ArrayList<>();
+ skeletaToRelatedPathWithValue.get(key).stream()
+ .forEach(
+ x -> {
+ String y = file.getStringValue(x);
+ XPathParts parts = XPathParts.getFrozenInstance(x);
+ sorted.add(parts.getAttributeValue(-1, "id") + " → “" + y + "”");
+ });
+ return Joiner.on(", ").join(sorted);
+ }
+
+ private String simplePattern(String id) {
+ TreeSet chars = new TreeSet<>();
+ for (Object item : parser.set(id).getItems()) {
+ if (item instanceof DateTimePatternGenerator.VariableField) {
+ VariableField v = (DateTimePatternGenerator.VariableField) item;
+ chars.add(
+ VariableField.getCanonicalCode(v.getType())
+ + (v.isNumeric() ? "ⁿ" : "ˢ"));
+ }
+ }
+ return Joiner.on("").join(chars);
+ }
+
+ void addRelated(String path) {
+ skeletaToRelatedPathWithValue.put(getKey(path), path);
+ }
+
+ Set getRelated(String path) {
+ return skeletaToRelatedPathWithValue.get(getKey(path));
+ }
+ }
+
private static File[] getCorrespondingDirectories(String base, Factory cldrFactory) {
File[] sourceDirs = cldrFactory.getSourceDirectories();
File[] newDirs = new File[sourceDirs.length];
diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java
index 83a5a089f1f..ddf60ed9ce0 100644
--- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java
+++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java
@@ -122,26 +122,31 @@ public void TestPathHeadersAndValues() {
CLDRFile englishFile = testInfo.getCldrFactory().make("en", true);
PathHeader.Factory phf = PathHeader.getFactory(englishFile);
Status status = new Status();
+ final String exemptLocale = "sv";
+ final String exemptPathIfLocale =
+ "//ldml/dates/calendars/calendar[@type=\"dangi\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"yMd\"]";
+
for (String locale : getLocalesToTest()) {
+ boolean isExemptLocale = locale.equals(exemptLocale);
+
if (!StandardCodes.isLocaleAtLeastBasic(locale)) {
continue;
}
CLDRFile file = testInfo.getCLDRFile(locale, true);
logln("Testing path headers and values for locale => " + locale);
final Collection extraPaths = file.getExtraPaths();
+
for (Iterator it = file.iterator(); it.hasNext(); ) {
String path = it.next();
- if (extraPaths.contains(path)) {
+ if (isExemptLocale && path.equals(exemptPathIfLocale)) {
+ logKnownIssue("CLDR-17544", "Can't reproduce locally");
continue;
}
- checkFullpathValue(path, file, locale, status, false /* not extra path */);
- if (!pathsSeen.contains(path)) {
- pathsSeen.add(path);
- checkPrettyPaths(path, phf);
+ if (extraPaths.contains(path)) {
+ checkFullpathValue(path, file, locale, status, true /* extra path */);
+ } else {
+ checkFullpathValue(path, file, locale, status, false /* not extra path */);
}
- }
- for (String path : extraPaths) {
- checkFullpathValue(path, file, locale, status, true /* extra path */);
if (!pathsSeen.contains(path)) {
pathsSeen.add(path);
checkPrettyPaths(path, phf);