Skip to content

Commit

Permalink
CLDR-17544 In en, remove yMd in chinese calendar availableFormats
Browse files Browse the repository at this point in the history
  • Loading branch information
@macchiati
macchiati committed May 7, 2024
1 parent abea2dd commit 11671dd
Show file tree
Hide file tree
Showing 2 changed files with 134 additions and 11 deletions.
1 change: 0 additions & 1 deletion common/main/root.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,6 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<dateFormatItem id="UMMM">U MMM</dateFormatItem>
<dateFormatItem id="UMMMd">U MMM d</dateFormatItem>
<dateFormatItem id="y">r(U)</dateFormatItem>
<dateFormatItem id="yMd">r-MM-dd</dateFormatItem>
<dateFormatItem id="yyyy">r(U)</dateFormatItem>
<dateFormatItem id="yyyyM">r-MM</dateFormatItem>
<dateFormatItem id="yyyyMd">r-MM-dd</dateFormatItem>
Expand Down
144 changes: 134 additions & 10 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/SearchCLDR.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeMultimap;
import com.ibm.icu.text.DateTimePatternGenerator;
import com.ibm.icu.text.DateTimePatternGenerator.FormatParser;
import com.ibm.icu.text.DateTimePatternGenerator.VariableField;
import com.ibm.icu.util.ICUUncheckedIOException;
import com.ibm.icu.util.Output;
import com.ibm.icu.util.VersionInfo;
Expand All @@ -27,6 +31,7 @@
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRFile.Status;
import org.unicode.cldr.util.CLDRLocale;
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.Counter;
import org.unicode.cldr.util.Factory;
Expand All @@ -38,6 +43,7 @@
import org.unicode.cldr.util.PatternCache;
import org.unicode.cldr.util.SimpleFactory;
import org.unicode.cldr.util.StandardCodes;
import org.unicode.cldr.util.XPathParts;

public class SearchCLDR {
// private static final int
Expand Down Expand Up @@ -77,6 +83,8 @@ public class SearchCLDR {
// + "-s\t show English value"
// ;

private static final CLDRConfig CONFIG = CLDRConfig.getInstance();

enum PathStyle {
none,
path,
Expand Down Expand Up @@ -110,6 +118,11 @@ enum PathStyle {
.add("resolved", null, null, "use resolved locales")
.add("q-showParent", null, null, "show parent value")
.add("english", null, null, "show english value")
.add(
"RootUncovered" + "",
null,
"false",
"filter to items that are in root but not overriden")
.add("Verbose", null, null, "verbose output")
.add(
"PathStyle",
Expand All @@ -135,6 +148,7 @@ enum PathStyle {
private static boolean showSurveyToolUrl;
private static Subtype subtype;
private static CheckCLDR checkCldr;
private static boolean rootUncovered;

static PathHeader.Factory pathHeaderFactory = PathHeader.getFactory();

Expand All @@ -160,16 +174,22 @@ public static void main(String[] args) {
Boolean valueExclude = exclude.value;

countOnly = myOptions.get("count").doesOccur();
boolean resolved = myOptions.get("resolved").doesOccur();
final boolean resolved = myOptions.get("resolved").doesOccur();

showPath = myOptions.get("z-showPath").doesOccur();
String orgString = myOptions.get("organization").getValue();
Organization organization = orgString == null ? null : Organization.fromString(orgString);

final CLDRFile english = CLDRConfig.getInstance().getEnglish();
final CLDRFile english = CONFIG.getEnglish();

showPathHeader = PathStyle.valueOf(myOptions.get("PathStyle").getValue());

rootUncovered = myOptions.get("RootUncovered").doesOccur();
if (rootUncovered && resolved) {
throw new IllegalArgumentException(
"Doesn't make sense to have both rootUncovered && resolved");
}

showSurveyToolUrl = myOptions.get("SurveyTool").doesOccur();

boolean showParent = myOptions.get("q-showParent").doesOccur();
Expand Down Expand Up @@ -236,8 +256,15 @@ public static void main(String[] args) {
Map<String, String> options = new HashMap<>();

int totalCount = 0;
final CLDRFile ROOT = CONFIG.getRoot();

for (String locale : locales) {
if (rootUncovered) { // only look at locales with parent=root)
CLDRLocale clocale = CLDRLocale.getInstance(locale);
if (!clocale.isParentRoot()) {
continue;
}
}
int localeCount = 0;
// Level organizationLevel = organization == null ? null
// : StandardCodes.make().getLocaleCoverageLevel(organization, locale);
Expand Down Expand Up @@ -274,14 +301,32 @@ public static void main(String[] args) {
level = CoverageLevel2.getInstance(locale);
Status status = new Status();
Set<PathHeader> sorted = new TreeSet<>();
for (String path : file.fullIterable()) {
if (locale.equals("eo") && path.contains("type=\"MK\"")) {
final Iterable<String> pathSource =
rootUncovered ? (Iterable<String>) ROOT : file.fullIterable();
RelatedPaths relatedPathsWithNonNullValues = new RelatedPaths();

for (String path : pathSource) {
if (path.contains("yMd") && path.contains("chinese")) {
int debug = 0;
}
if (pathMatcher != null && !pathMatcher.find(path)) {
continue;
}
String stringValue = file.getStringValue(path);
if (stringValue == null) {
if (rootUncovered) {
if (stringValue != null) {
// Record any cases where there are non-null, non-inherited values
if (!stringValue.equals("↑↑↑")) {
relatedPathsWithNonNullValues.addRelated(path);
}
continue;
}
// we will add the path if the value for this path is null
// (uncovered in this file)
} else if (stringValue == null) {
continue;
}

String diffStringValue;
if (diffFile != null) {
diffStringValue = diffFile.getWinningValueWithBailey(path);
Expand All @@ -300,18 +345,26 @@ public static void main(String[] args) {
}
sorted.add(pathHeaderFactory.fromPath(path));
}

for (PathHeader pathHeader : sorted) {
String path = pathHeader.getOriginalPath();
String relatedNonNullKey = null;
if (rootUncovered) {
// We've collected paths from root that are uncovered in this file.
// That means they have a null value in this file, but non-null in root.
// We want to skip away all of those UNLESS there is a related path in this file
// that has a non-null value.
relatedNonNullKey = relatedPathsWithNonNullValues.hasKeyFor(path);
if (relatedNonNullKey == null) {
continue;
}
}
String fullPath = file.getFullXPath(path);
String value = file.getStringValue(path);
if (locale.equals("eo") && path.contains("type=\"MK\"")) {
int debug = 0;
}

if (pathMatcher != null && !pathMatcher.find(fullPath)) {
continue;
}

{
pathLevel = level.getLevel(path);
levelCounter.add(pathLevel, 1);
Expand Down Expand Up @@ -409,6 +462,19 @@ public static void main(String[] args) {
null,
null);
} else {
String extra =
!rootUncovered
? ""
: "\t"
+ relatedNonNullKey
+ "\t"
+ XPathParts.getFrozenInstance(path)
.getAttributeValue(-1, "id")
+ " → “"
+ ROOT.getStringValue(path)
+ "”\t"
+ relatedPathsWithNonNullValues.show(
file, relatedNonNullKey);
showLine(
showPathHeader,
showParent,
Expand All @@ -421,7 +487,7 @@ public static void main(String[] args) {
!showParent ? null : english.getBaileyValue(path, null, null),
english == null ? null : english.getStringValue(path),
resolvedSource,
Objects.toString(pathLevel));
Objects.toString(pathLevel) + extra);
}
totalCount++;
localeCount++;
Expand All @@ -446,6 +512,64 @@ public static void main(String[] args) {
+ " minutes");
}

/**
* Related with related values that are not null. NOTE: For now this is quite specific to
* availableFormats
*/
static class RelatedPaths {
TreeMultimap<String, String> skeletaToRelatedPathWithValue = TreeMultimap.create();
static final FormatParser parser = new DateTimePatternGenerator.FormatParser();

String getKey(String path) {
// ldml/dates/calendars/calendar[@type="chinese"]/dateTimeFormats/availableFormats/dateFormatItem[@id="d"]
XPathParts parts = XPathParts.getFrozenInstance(path);
if (parts.size() != 7 || !"availableFormats".equals(parts.getElement(5))) {
return null;
}
return parts.getAttributeValue(3, "type")
+ "|"
+ simplePattern(parts.getAttributeValue(-1, "id"));
}

public String hasKeyFor(String path) {
String key = getKey(path);
return skeletaToRelatedPathWithValue.containsKey(key) ? key : null;
}

public String show(CLDRFile file, String key) {
final List<String> sorted = new ArrayList<>();
skeletaToRelatedPathWithValue.get(key).stream()
.forEach(
x -> {
String y = file.getStringValue(x);
XPathParts parts = XPathParts.getFrozenInstance(x);
sorted.add(parts.getAttributeValue(-1, "id") + " → “" + y + "”");
});
return Joiner.on(", ").join(sorted);
}

private String simplePattern(String id) {
TreeSet<String> chars = new TreeSet<>();
for (Object item : parser.set(id).getItems()) {
if (item instanceof DateTimePatternGenerator.VariableField) {
VariableField v = (DateTimePatternGenerator.VariableField) item;
chars.add(
VariableField.getCanonicalCode(v.getType())
+ (v.isNumeric() ? "ⁿ" : "ˢ"));
}
}
return Joiner.on("").join(chars);
}

void addRelated(String path) {
skeletaToRelatedPathWithValue.put(getKey(path), path);
}

Set<String> getRelated(String path) {
return skeletaToRelatedPathWithValue.get(getKey(path));
}
}

private static File[] getCorrespondingDirectories(String base, Factory cldrFactory) {
File[] sourceDirs = cldrFactory.getSourceDirectories();
File[] newDirs = new File[sourceDirs.length];
Expand Down

0 comments on commit 11671dd

Please sign in to comment.