Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CLDR-17544 In en, remove yMd in chinese calendar availableFormats #3696

Merged
merged 2 commits into from
May 8, 2024
Merged

CLDR-17544 In en, remove yMd in chinese calendar availableFormats #3696

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11671dd
CLDR-17544 In en, remove yMd in chinese calendar availableFormats
macchiati May 7, 2024
91442ce
CLDR-17544 Added log known issue for failure that (a) doesn't look se…
macchiati May 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion common/main/root.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,6 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<dateFormatItem id="UMMM">U MMM</dateFormatItem>
<dateFormatItem id="UMMMd">U MMM d</dateFormatItem>
<dateFormatItem id="y">r(U)</dateFormatItem>
<dateFormatItem id="yMd">r-MM-dd</dateFormatItem>
<dateFormatItem id="yyyy">r(U)</dateFormatItem>
<dateFormatItem id="yyyyM">r-MM</dateFormatItem>
<dateFormatItem id="yyyyMd">r-MM-dd</dateFormatItem>
Expand Down
144 changes: 134 additions & 10 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/SearchCLDR.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeMultimap;
import com.ibm.icu.text.DateTimePatternGenerator;
import com.ibm.icu.text.DateTimePatternGenerator.FormatParser;
import com.ibm.icu.text.DateTimePatternGenerator.VariableField;
import com.ibm.icu.util.ICUUncheckedIOException;
import com.ibm.icu.util.Output;
import com.ibm.icu.util.VersionInfo;
Expand All @@ -27,6 +31,7 @@
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRFile.Status;
import org.unicode.cldr.util.CLDRLocale;
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.Counter;
import org.unicode.cldr.util.Factory;
Expand All @@ -38,6 +43,7 @@
import org.unicode.cldr.util.PatternCache;
import org.unicode.cldr.util.SimpleFactory;
import org.unicode.cldr.util.StandardCodes;
import org.unicode.cldr.util.XPathParts;

public class SearchCLDR {
// private static final int
Expand Down Expand Up @@ -77,6 +83,8 @@ public class SearchCLDR {
// + "-s\t show English value"
// ;

private static final CLDRConfig CONFIG = CLDRConfig.getInstance();

enum PathStyle {
none,
path,
Expand Down Expand Up @@ -110,6 +118,11 @@ enum PathStyle {
.add("resolved", null, null, "use resolved locales")
.add("q-showParent", null, null, "show parent value")
.add("english", null, null, "show english value")
.add(
"RootUncovered" + "",
null,
"false",
"filter to items that are in root but not overriden")
.add("Verbose", null, null, "verbose output")
.add(
"PathStyle",
Expand All @@ -135,6 +148,7 @@ enum PathStyle {
private static boolean showSurveyToolUrl;
private static Subtype subtype;
private static CheckCLDR checkCldr;
private static boolean rootUncovered;

static PathHeader.Factory pathHeaderFactory = PathHeader.getFactory();

Expand All @@ -160,16 +174,22 @@ public static void main(String[] args) {
Boolean valueExclude = exclude.value;

countOnly = myOptions.get("count").doesOccur();
boolean resolved = myOptions.get("resolved").doesOccur();
final boolean resolved = myOptions.get("resolved").doesOccur();

showPath = myOptions.get("z-showPath").doesOccur();
String orgString = myOptions.get("organization").getValue();
Organization organization = orgString == null ? null : Organization.fromString(orgString);

final CLDRFile english = CLDRConfig.getInstance().getEnglish();
final CLDRFile english = CONFIG.getEnglish();

showPathHeader = PathStyle.valueOf(myOptions.get("PathStyle").getValue());

rootUncovered = myOptions.get("RootUncovered").doesOccur();
if (rootUncovered && resolved) {
throw new IllegalArgumentException(
"Doesn't make sense to have both rootUncovered && resolved");
}

showSurveyToolUrl = myOptions.get("SurveyTool").doesOccur();

boolean showParent = myOptions.get("q-showParent").doesOccur();
Expand Down Expand Up @@ -236,8 +256,15 @@ public static void main(String[] args) {
Map<String, String> options = new HashMap<>();

int totalCount = 0;
final CLDRFile ROOT = CONFIG.getRoot();

for (String locale : locales) {
if (rootUncovered) { // only look at locales with parent=root)
CLDRLocale clocale = CLDRLocale.getInstance(locale);
if (!clocale.isParentRoot()) {
continue;
}
}
int localeCount = 0;
// Level organizationLevel = organization == null ? null
// : StandardCodes.make().getLocaleCoverageLevel(organization, locale);
Expand Down Expand Up @@ -274,14 +301,32 @@ public static void main(String[] args) {
level = CoverageLevel2.getInstance(locale);
Status status = new Status();
Set<PathHeader> sorted = new TreeSet<>();
for (String path : file.fullIterable()) {
if (locale.equals("eo") && path.contains("type=\"MK\"")) {
final Iterable<String> pathSource =
rootUncovered ? (Iterable<String>) ROOT : file.fullIterable();
RelatedPaths relatedPathsWithNonNullValues = new RelatedPaths();

for (String path : pathSource) {
if (path.contains("yMd") && path.contains("chinese")) {
int debug = 0;
}
Comment on lines +309 to 311
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (path.contains("yMd") && path.contains("chinese")) {
int debug = 0;
}

you should be able to do these with a conditional breakpoint, anyway

if (pathMatcher != null && !pathMatcher.find(path)) {
continue;
}
String stringValue = file.getStringValue(path);
if (stringValue == null) {
if (rootUncovered) {
if (stringValue != null) {
// Record any cases where there are non-null, non-inherited values
if (!stringValue.equals("↑↑↑")) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (!stringValue.equals("↑↑↑")) {
if (!stringValue.equals(CldrUtility.INHERITANCE_MARKER)) {

relatedPathsWithNonNullValues.addRelated(path);
}
continue;
}
// we will add the path if the value for this path is null
// (uncovered in this file)
} else if (stringValue == null) {
continue;
}

String diffStringValue;
if (diffFile != null) {
diffStringValue = diffFile.getWinningValueWithBailey(path);
Expand All @@ -300,18 +345,26 @@ public static void main(String[] args) {
}
sorted.add(pathHeaderFactory.fromPath(path));
}

for (PathHeader pathHeader : sorted) {
String path = pathHeader.getOriginalPath();
String relatedNonNullKey = null;
if (rootUncovered) {
// We've collected paths from root that are uncovered in this file.
// That means they have a null value in this file, but non-null in root.
// We want to skip away all of those UNLESS there is a related path in this file
// that has a non-null value.
relatedNonNullKey = relatedPathsWithNonNullValues.hasKeyFor(path);
if (relatedNonNullKey == null) {
continue;
}
}
String fullPath = file.getFullXPath(path);
String value = file.getStringValue(path);
if (locale.equals("eo") && path.contains("type=\"MK\"")) {
int debug = 0;
}

if (pathMatcher != null && !pathMatcher.find(fullPath)) {
continue;
}

{
pathLevel = level.getLevel(path);
levelCounter.add(pathLevel, 1);
Expand Down Expand Up @@ -409,6 +462,19 @@ public static void main(String[] args) {
null,
null);
} else {
String extra =
!rootUncovered
? ""
: "\t"
+ relatedNonNullKey
+ "\t"
+ XPathParts.getFrozenInstance(path)
.getAttributeValue(-1, "id")
+ " → “"
+ ROOT.getStringValue(path)
+ "”\t"
+ relatedPathsWithNonNullValues.show(
file, relatedNonNullKey);
showLine(
showPathHeader,
showParent,
Expand All @@ -421,7 +487,7 @@ public static void main(String[] args) {
!showParent ? null : english.getBaileyValue(path, null, null),
english == null ? null : english.getStringValue(path),
resolvedSource,
Objects.toString(pathLevel));
Objects.toString(pathLevel) + extra);
}
totalCount++;
localeCount++;
Expand All @@ -446,6 +512,64 @@ public static void main(String[] args) {
+ " minutes");
}

/**
* Related with related values that are not null. NOTE: For now this is quite specific to
* availableFormats
*/
static class RelatedPaths {
TreeMultimap<String, String> skeletaToRelatedPathWithValue = TreeMultimap.create();
static final FormatParser parser = new DateTimePatternGenerator.FormatParser();

String getKey(String path) {
// ldml/dates/calendars/calendar[@type="chinese"]/dateTimeFormats/availableFormats/dateFormatItem[@id="d"]
XPathParts parts = XPathParts.getFrozenInstance(path);
if (parts.size() != 7 || !"availableFormats".equals(parts.getElement(5))) {
return null;
}
return parts.getAttributeValue(3, "type")
+ "|"
+ simplePattern(parts.getAttributeValue(-1, "id"));
}

public String hasKeyFor(String path) {
String key = getKey(path);
return skeletaToRelatedPathWithValue.containsKey(key) ? key : null;
}

public String show(CLDRFile file, String key) {
final List<String> sorted = new ArrayList<>();
skeletaToRelatedPathWithValue.get(key).stream()
.forEach(
x -> {
String y = file.getStringValue(x);
XPathParts parts = XPathParts.getFrozenInstance(x);
sorted.add(parts.getAttributeValue(-1, "id") + " → “" + y + "”");
});
return Joiner.on(", ").join(sorted);
}

private String simplePattern(String id) {
TreeSet<String> chars = new TreeSet<>();
for (Object item : parser.set(id).getItems()) {
if (item instanceof DateTimePatternGenerator.VariableField) {
VariableField v = (DateTimePatternGenerator.VariableField) item;
chars.add(
VariableField.getCanonicalCode(v.getType())
+ (v.isNumeric() ? "ⁿ" : "ˢ"));
}
}
return Joiner.on("").join(chars);
}

void addRelated(String path) {
skeletaToRelatedPathWithValue.put(getKey(path), path);
}

Set<String> getRelated(String path) {
return skeletaToRelatedPathWithValue.get(getKey(path));
}
}

private static File[] getCorrespondingDirectories(String base, Factory cldrFactory) {
File[] sourceDirs = cldrFactory.getSourceDirectories();
File[] newDirs = new File[sourceDirs.length];
Expand Down
21 changes: 13 additions & 8 deletions tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,26 +122,31 @@ public void TestPathHeadersAndValues() {
CLDRFile englishFile = testInfo.getCldrFactory().make("en", true);
PathHeader.Factory phf = PathHeader.getFactory(englishFile);
Status status = new Status();
final String exemptLocale = "sv";
final String exemptPathIfLocale =
"//ldml/dates/calendars/calendar[@type=\"dangi\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"yMd\"]";

for (String locale : getLocalesToTest()) {
boolean isExemptLocale = locale.equals(exemptLocale);

if (!StandardCodes.isLocaleAtLeastBasic(locale)) {
continue;
}
CLDRFile file = testInfo.getCLDRFile(locale, true);
logln("Testing path headers and values for locale => " + locale);
final Collection<String> extraPaths = file.getExtraPaths();

for (Iterator<String> it = file.iterator(); it.hasNext(); ) {
String path = it.next();
if (extraPaths.contains(path)) {
if (isExemptLocale && path.equals(exemptPathIfLocale)) {
logKnownIssue("CLDR-17544", "Can't reproduce locally");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

^ the message isn't clear as to what's happening.

continue;
}
checkFullpathValue(path, file, locale, status, false /* not extra path */);
if (!pathsSeen.contains(path)) {
pathsSeen.add(path);
checkPrettyPaths(path, phf);
if (extraPaths.contains(path)) {
checkFullpathValue(path, file, locale, status, true /* extra path */);
} else {
checkFullpathValue(path, file, locale, status, false /* not extra path */);
}
}
for (String path : extraPaths) {
checkFullpathValue(path, file, locale, status, true /* extra path */);
if (!pathsSeen.contains(path)) {
pathsSeen.add(path);
checkPrettyPaths(path, phf);
Expand Down
Loading