Skip to content

Commit

Permalink
CLDR-16393 fix breakage with old DTDs
Browse files Browse the repository at this point in the history 
- skip ldmlICU versions that are missing
- ElementAttributeInfo needs to skip XML files that are missing
  • Loading branch information
@srl295
srl295 committed Aug 31, 2023
1 parent 0b683a6 commit 8e95df1
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 33 deletions.
7 changes: 7 additions & 0 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.SupplementalDataInfo;

Expand Down Expand Up @@ -66,6 +68,7 @@ public enum CldrVersion {
v41_0,
v42_0,
v43_0,
/** @see CLDRFile#GEN_VERSION */
baseline;

private final String baseDirectory;
Expand Down Expand Up @@ -97,6 +100,10 @@ public static CldrVersion from(VersionInfo versionInfo) {
}

public static CldrVersion from(String versionString) {
// treat 'current' as baseline
if (versionString.equals(CLDRFile.GEN_VERSION) || versionString.equals(CLDRFile.GEN_VERSION + ".0")) {
return CldrVersion.baseline;
}
return valueOf(
versionString.charAt(0) < 'A'
? "v" + versionString.replace('.', '_')
Expand Down
63 changes: 34 additions & 29 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,35 +55,7 @@ public static final ElementAttributeInfo getInstance(String commonDirectory, Dtd
}
}
if (result == null) {
result = new HashMap<>();
// pick short files that are in repository
// Add to this when a DTD is added
result.put(
DtdType.ldml,
new ElementAttributeInfo(
canonicalCommonDirectory + "/main/root.xml", DtdType.ldml));
result.put(
DtdType.supplementalData,
new ElementAttributeInfo(
canonicalCommonDirectory + "/supplemental/plurals.xml",
DtdType.supplementalData));
result.put(
DtdType.ldmlBCP47,
new ElementAttributeInfo(
canonicalCommonDirectory + "/bcp47/calendar.xml",
DtdType.ldmlBCP47));
result.put(
DtdType.keyboard,
new ElementAttributeInfo(
canonicalCommonDirectory
+ "/../keyboards/3.0/fr-t-k0-azerty.xml",
DtdType.keyboard));
result.put(
DtdType.keyboardTest,
new ElementAttributeInfo(
canonicalCommonDirectory
+ "/../keyboards/test/fr-t-k0-azerty-test.xml",
DtdType.keyboardTest));
result = makeElementAttributeInfoMap(canonicalCommonDirectory);
cache.put(commonDirectory, result);
cache.put(canonicalCommonDirectory, result);
}
Expand All @@ -101,6 +73,39 @@ public static final ElementAttributeInfo getInstance(String commonDirectory, Dtd
return eai;
}

private static void addElementAttributeInfo(Map<DtdType, ElementAttributeInfo> result, DtdType type, String path) throws IOException {
if (!new File(path).canRead()) {
System.err.println("ElementAttributeInfo: Warning: Sample file did not exist: " + path + " for DtdType " + type.name());
return; // file doesn't exist.
}
result.put(type, new ElementAttributeInfo(path, type));
}

private static Map<DtdType, ElementAttributeInfo> makeElementAttributeInfoMap(String canonicalCommonDirectory) throws IOException {
Map<DtdType, ElementAttributeInfo> result;
result = new HashMap<>();
// pick short files that are in repository
// Add to this when a DTD is added
addElementAttributeInfo(result,
DtdType.ldml,
canonicalCommonDirectory + "/main/root.xml");
addElementAttributeInfo(result,
DtdType.supplementalData,
canonicalCommonDirectory + "/supplemental/plurals.xml");
addElementAttributeInfo(result,
DtdType.ldmlBCP47,
canonicalCommonDirectory + "/bcp47/calendar.xml");
addElementAttributeInfo(result,
DtdType.keyboard,
canonicalCommonDirectory
+ "/../keyboards/3.0/fr-t-k0-azerty.xml");
addElementAttributeInfo(result,
DtdType.keyboardTest,
canonicalCommonDirectory
+ "/../keyboards/test/fr-t-k0-azerty-test.xml");
return result;
}

// static {
// try {
// addFromDTD(CldrUtility.COMMON_DIRECTORY + "main/en.xml", DtdType.ldml);
Expand Down
32 changes: 28 additions & 4 deletions tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
Expand Down Expand Up @@ -61,6 +62,7 @@
import org.unicode.cldr.util.DtdData.Attribute;
import org.unicode.cldr.util.DtdData.Element;
import org.unicode.cldr.util.DtdData.ElementType;
import org.unicode.cldr.util.DtdType.DtdStatus;
import org.unicode.cldr.util.DtdType;
import org.unicode.cldr.util.ElementAttributeInfo;
import org.unicode.cldr.util.Factory;
Expand Down Expand Up @@ -1005,6 +1007,8 @@ static boolean isLikelyEquivalent(String locale1, String locale2) {

static final Map<String, String> likelyData = SUPPLEMENTAL_DATA_INFO.getLikelySubtags();

private static final EnumSet<CldrVersion> badLdmlICUVersions = EnumSet.of(CldrVersion.v1_1_1, CldrVersion.v1_2, CldrVersion.v1_4_1, CldrVersion.v1_5_1);

public void TestLikelySubtagsComplete() {
LanguageTagParser ltp = new LanguageTagParser();
for (String locale : testInfo.getCldrFactory().getAvailable()) {
Expand Down Expand Up @@ -1228,6 +1232,10 @@ public void TestBasicDTDCompatibility() {
// test all DTDs
for (DtdType dtd : DtdType.values()) {
if (dtd.getStatus() != DtdType.DtdStatus.active) continue;
if (dtd.firstVersion != null && CldrVersion.LAST_RELEASE_VERSION.isOlderThan(CldrVersion.from(dtd.firstVersion))) {
continue; // DTD didn't exist in last release
}
if (dtd == DtdType.ldmlICU) continue;
try {
ElementAttributeInfo oldDtd = ElementAttributeInfo.getInstance(oldCommon, dtd);
ElementAttributeInfo newDtd = ElementAttributeInfo.getInstance(dtd);
Expand Down Expand Up @@ -1401,16 +1409,26 @@ public void TestDtdCompatibility() {
if (version == CldrVersion.unknown || version == CldrVersion.baseline) {
continue;
}
if (type.getStatus() != DtdStatus.active) {
continue; // not active
}
if (type.firstVersion != null && version.isOlderThan(CldrVersion.from(type.firstVersion))) {
continue; // didn't exist at that point
}
DtdData dtdDataOld;
try {
dtdDataOld = DtdData.getInstance(type, version.toString());
} catch (IllegalArgumentException e) {
boolean tooOld = false;
switch (type) {
case ldmlBCP47:
case ldmlICU:
tooOld = badLdmlICUVersions.contains(version);
break;
case ldmlBCP47:
case keyboard:
tooOld = version.isOlderThan(CldrVersion.from(type.firstVersion));
if (type.firstVersion != null) {
tooOld = version.isOlderThan(CldrVersion.from(type.firstVersion));
}
break;
default:
break;
Expand All @@ -1419,8 +1437,8 @@ public void TestDtdCompatibility() {
continue;
} else {
errln(
version
+ ": "
"v" + version
+ ": "
+ e.getClass().getSimpleName()
+ ", "
+ e.getMessage());
Expand Down Expand Up @@ -1450,6 +1468,12 @@ public void TestDtdCompatibility() {
continue;
}
Element newChild = newElement.getChildNamed(oldChild.getName());
// skip certain items
if (version.isOlderThan(CldrVersion.v1_6_1) && newElement.getName().equals("zone") && oldChild.getName().equals("usesMetazone")) {
if (logKnownIssue("CLDR-17054", "Breakage with items older than 1.6.1: " + newElement.getName() + " / " + oldChild.getName())) {
continue;
}
}

if (knownChildExceptions.contains(
Pair.of(newElement.getName(), oldChild.getName()))) {
Expand Down

0 comments on commit 8e95df1

Please sign in to comment.