From bdd4931f2a15077746ee42445ed99bb33e7c42ac Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 23 Aug 2023 18:43:25 -0500 Subject: [PATCH 01/10] CLDR-16393 assume cldr-archive is present for tests - UNLESS -DNO_CLDR_ARCHIVE=true is set --- .../unicode/cldr/unittest/TestTransforms.java | 6 +-- .../unicode/cldr/unittest/TestValidity.java | 7 ++-- .../org/unicode/cldr/util/TestCLDRPaths.java | 38 +++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestTransforms.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestTransforms.java index d183c6bb6b8..41b607844a5 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestTransforms.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestTransforms.java @@ -46,6 +46,7 @@ import org.unicode.cldr.util.Pair; import org.unicode.cldr.util.PathUtilities; import org.unicode.cldr.util.StandardCodes; +import org.unicode.cldr.util.TestCLDRPaths; import org.unicode.cldr.util.UnicodeRelation; import org.unicode.cldr.util.XMLFileReader; import org.unicode.cldr.util.XPathParts; @@ -405,9 +406,8 @@ public void TestTransformIDs() { checkTransformID(entry.getKey(), entry.getValue()); } - // Only run the rest in exhaustive mode since it requires CLDR_ARCHIVE_DIRECTORY. - if (getInclusion() <= 5) { - return; + if (!TestCLDRPaths.canUseArchiveDirectory()) { + return; // skipped } Set removedTransforms = new HashSet<>(); diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestValidity.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestValidity.java index 3885b33b4b1..9a822dfbb58 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestValidity.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestValidity.java @@ -33,6 +33,7 @@ import org.unicode.cldr.util.StandardCodes; import org.unicode.cldr.util.StandardCodes.LstrField; import org.unicode.cldr.util.StandardCodes.LstrType; +import org.unicode.cldr.util.TestCLDRPaths; import org.unicode.cldr.util.TransliteratorUtilities; import org.unicode.cldr.util.Units; import org.unicode.cldr.util.Validity; @@ -143,10 +144,10 @@ public void TestBasicValidity() { static final Set ALLOWED_REGULAR_TO_SPECIAL = ImmutableSet.of("Zanb", "Zinh", "Zyyy"); public void TestCompatibility() { - // Only run the rest in exhaustive mode, since it requires CLDR_ARCHIVE_DIRECTORY - if (getInclusion() <= 5) { - return; + if (!TestCLDRPaths.canUseArchiveDirectory()) { + return; // skipped } + Set messages = new HashSet<>(); File archive = new File(CLDRPaths.ARCHIVE_DIRECTORY); for (File cldrArchive : archive.listFiles()) { diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java new file mode 100644 index 00000000000..09b4ac35bdd --- /dev/null +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java @@ -0,0 +1,38 @@ +package org.unicode.cldr.util; + +import java.io.File; +import org.junit.jupiter.api.Test; + +public class TestCLDRPaths { + + @Test + void TestCanUseArchiveDirectory() { + if (!canUseArchiveDirectory()) { + // We print this warning as part of the unit tests. + System.err.println( + "WARNING: skipping using cldr-archive. Ideally, -DNO_CLDR_ARCHIVE=false and see "); + } + } + + /** + * @return true if it's OK to read CLDRPaths.ARCHIVE_DIRECTORY, false to skip. + */ + public static final boolean canUseArchiveDirectory() { + if (CLDRConfig.getInstance().getProperty("NO_CLDR_ARCHIVE", false)) { + return false; // skip, NO_CLDR_ARCHIVE is set. + } + + final File archiveDir = new File(CLDRPaths.ARCHIVE_DIRECTORY); + if (!archiveDir.isDirectory()) { + throw new IllegalArgumentException( + String.format( + "Could not read archive directory %s. " + + "Please: " + + "1) setup the archive, , " + + "2) set the -DARCHIVE= property to the correct archive location, or " + + "3) inhibit reading of cldr-archive with -DNO_CLDR_ARCHIVE=true", + archiveDir.getAbsolutePath())); + } + return true; // OK to use + } +} From 1ed1b38d7f9bc86244636a1f7f525e644b205799 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 23 Aug 2023 18:49:51 -0500 Subject: [PATCH 02/10] CLDR-16393 reinstate some tests that depend on cldr-archive --- .../test/java/org/unicode/cldr/unittest/TestBasic.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java index fc0887bd595..5eed72b995e 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java @@ -75,6 +75,7 @@ import org.unicode.cldr.util.SupplementalDataInfo; import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo; import org.unicode.cldr.util.SupplementalDataInfo.PluralType; +import org.unicode.cldr.util.TestCLDRPaths; import org.unicode.cldr.util.XMLFileReader; import org.unicode.cldr.util.XPathParts; import org.xml.sax.ErrorHandler; @@ -1196,9 +1197,7 @@ public void TestDtdCompleteness() { public void TestBasicDTDCompatibility() { - if (logKnownIssue( - "cldrbug:16393", - "Comment out until we detect whether to enable cldr-archive for unit tests")) { + if (!TestCLDRPaths.canUseArchiveDirectory()) { return; } @@ -1390,9 +1389,7 @@ public void TestDtdCompatibility() { Collections.EMPTY_SET, elementsWithoutSpecial); - if (logKnownIssue( - "cldrbug:16393", - "Comment out test until cldr-archive is signaled for unit tests on CI")) { + if (!TestCLDRPaths.canUseArchiveDirectory()) { return; } From cde86d93d9f446f180a65cd6511dc9530ad54cbe Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 30 Aug 2023 17:50:36 -0500 Subject: [PATCH 03/10] CLDR-16393 add a test for loading SDI - also, reinstate the 'platform' DtdType as 'removed' --- .../org/unicode/cldr/tool/CheckHtmlFiles.java | 1 + .../org/unicode/cldr/tool/GenerateDtd.java | 1 + .../unicode/cldr/tool/GenerateItemCounts.java | 1 + .../cldr/tool/GenerateReformattedXml.java | 1 + .../cldr/tool/ShowStarredCoverage.java | 1 + .../java/org/unicode/cldr/util/DtdType.java | 25 +++++++++++++++++++ .../cldr/util/SupplementalDataInfo.java | 5 ++++ .../org/unicode/cldr/unittest/TestBasic.java | 4 +++ .../cldr/unittest/TestComparisonBuilder.java | 3 +++ .../cldr/unittest/TestDTDAttributes.java | 1 + .../unicode/cldr/unittest/TestDtdData.java | 2 ++ .../unicode/cldr/unittest/TestPathHeader.java | 1 + .../org/unicode/cldr/unittest/TestPaths.java | 1 + .../org/unicode/cldr/util/TestCLDRPaths.java | 12 +++++++++ 14 files changed, 59 insertions(+) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java index b85fda5c23c..c232c0bd14c 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java @@ -214,6 +214,7 @@ private static void checkForDtd(Data target) { M4 typeToElements = ChainedMap.of(new TreeMap(), new TreeMap(), new TreeMap(), Boolean.class); for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; if (type == DtdType.ldmlICU) continue; DtdData dtdData = DtdData.getInstance(type); Set elements = dtdData.getElements(); diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java index 9caafc40311..ff81a54fbeb 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java @@ -21,6 +21,7 @@ public static void main(String[] args) throws IOException { final DTD2Markdown dtd2md = new DTD2Markdown(); // System.setProperty("show_all", "true"); for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; if (type == DtdType.ldmlICU) { continue; } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateItemCounts.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateItemCounts.java index 95d2d035489..abd9e6c0832 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateItemCounts.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateItemCounts.java @@ -197,6 +197,7 @@ private static void checkBadAttributes(Relation path2value2, Str SupplementalDataInfo supp = SUPPLEMENTAL_DATA_INFO; for (DtdType dtdType : DtdType.values()) { + if (dtdType.getStatus() != DtdType.DtdStatus.active) continue; if (dtdType == DtdType.ldmlICU) { continue; } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateReformattedXml.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateReformattedXml.java index 09f135a65b2..bfe7682d5cf 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateReformattedXml.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateReformattedXml.java @@ -6,6 +6,7 @@ public class GenerateReformattedXml { public static void main(String[] args) { for (DtdType dtdType : DtdType.values()) { + if (dtdType.getStatus() != DtdType.DtdStatus.active) continue; if (args.length > 0 && !dtdType.toString().matches(args[0])) { continue; } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowStarredCoverage.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowStarredCoverage.java index e1d70a2c4e2..40e25d09cc2 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowStarredCoverage.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowStarredCoverage.java @@ -134,6 +134,7 @@ public static void main(String[] args) { System.out.println("№\tLevel\tStarredPath\tPH Status\t?\tAttributes"); } for (DtdType dtdType : DtdType.values()) { + if (dtdType.getStatus() != DtdType.DtdStatus.active) continue; if (dtdTypes != null && !dtdTypes.contains(dtdType)) { continue; } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdType.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdType.java index fc128dc55a8..5bc4812820a 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdType.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdType.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableSet; import java.lang.annotation.Annotation; +import java.util.Collections; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -29,6 +30,7 @@ public enum DtdType { "transforms", "validity"), ldmlBCP47("common/dtd/ldmlBCP47.dtd", "1.7.2", null, "bcp47"), + // keyboard 3.0 keyboard3("keyboards/dtd/ldmlKeyboard3.dtd", "44.0", null, "../keyboards/3.0"), keyboardTest3("keyboards/dtd/ldmlKeyboardTest3.dtd", "44.0", null, "../keyboards/test"); @@ -48,10 +50,32 @@ public enum DtdType { public final String firstVersion; public final Set directories; + private final DtdStatus status; + + /** Get the ststus, whether this is an active DTD or not */ + public DtdStatus getStatus() { + return status; + } + + public enum DtdStatus { + /** DTD active (default) */ + active, + /** DTD no longer used */ + removed, + }; + private DtdType(String dtdPath) { this(dtdPath, null, null); } + private DtdType(DtdStatus status, String sinceVersion) { + this.directories = Collections.emptySet(); + this.dtdPath = null; + this.rootType = this; + this.firstVersion = sinceVersion; + this.status = status; + } + private DtdType(String dtdPath, DtdType realType) { this(dtdPath, null, realType); } @@ -61,6 +85,7 @@ private DtdType(String dtdPath, String firstVersion, DtdType realType, String... this.rootType = realType == null ? this : realType; this.firstVersion = firstVersion; this.directories = ImmutableSet.copyOf(directories); + this.status = DtdStatus.active; } public static DtdType fromPath(String elementOrPath) { diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java index 1b24733627d..e21d200c434 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/SupplementalDataInfo.java @@ -68,6 +68,7 @@ import org.unicode.cldr.util.Builder.CBuilder; import org.unicode.cldr.util.CldrUtility.VariableReplacer; import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; +import org.unicode.cldr.util.DtdType.DtdStatus; import org.unicode.cldr.util.GrammarInfo.GrammaticalFeature; import org.unicode.cldr.util.GrammarInfo.GrammaticalScope; import org.unicode.cldr.util.GrammarInfo.GrammaticalTarget; @@ -4572,6 +4573,10 @@ public MetaZoneRange getMetaZoneRange(String zone, long date) { } public boolean isDeprecated(DtdType type, String element, String attribute, String value) { + if (type.getStatus() == DtdStatus.removed) { + // if the DTD was removed, skip + return true; + } return DtdData.getInstance(type).isDeprecated(element, attribute, value); } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java index 5eed72b995e..ddc2364514d 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java @@ -225,6 +225,7 @@ public void showFoundElements(Relation, String> foundAtt Relation, String> theoryAttributes = Relation.of(new TreeMap, Set>(), TreeSet.class); for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; DtdData dtdData = DtdData.getInstance(type); for (Element element : dtdData.getElementFromName().values()) { String name = element.getName(); @@ -1166,6 +1167,7 @@ private boolean isTopLevel(String localeID) { /** Tests that every dtd item is connected from root */ public void TestDtdCompleteness() { for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; DtdData dtdData = DtdData.getInstance(type); Set descendents = new LinkedHashSet<>(); dtdData.getDescendents(dtdData.ROOT, descendents); @@ -1225,6 +1227,7 @@ public void TestBasicDTDCompatibility() { // test all DTDs for (DtdType dtd : DtdType.values()) { + if (dtd.getStatus() != DtdType.DtdStatus.active) continue; try { ElementAttributeInfo oldDtd = ElementAttributeInfo.getInstance(oldCommon, dtd); ElementAttributeInfo newDtd = ElementAttributeInfo.getInstance(dtd); @@ -1333,6 +1336,7 @@ private Set containsInOrder(Set superset, Set subset) { public void TestDtdCompatibility() { for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; DtdData dtdData = DtdData.getInstance(type); Map currentElementFromName = dtdData.getElementFromName(); diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java index 588c9c90bcf..a7e2fe707aa 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java @@ -49,6 +49,7 @@ private void dtdAttributes() { // unless FindDtdOrder also checks attributes. Builder builder = new Builder(Ordering.NATURAL); for (DtdType dtd : DtdType.values()) { + if (dtd.getStatus() != DtdType.DtdStatus.active) continue; Relation eaInfo = ElementAttributeInfo.getInstance(dtd).getElement2Attributes(); for (String element : eaInfo.keySet()) { @@ -68,6 +69,7 @@ private void dtdAttributes() { logln("Attribute Ordering:\t" + comp.getOrdering().toString()); for (DtdType dtd : DtdType.values()) { + if (dtd.getStatus() != DtdType.DtdStatus.active) continue; // check that the ordering is right Relation eaInfo = ElementAttributeInfo.getInstance(dtd).getElement2Attributes(); @@ -82,6 +84,7 @@ public void TestDtdElements() { Set specials = new HashSet(Arrays.asList(new String[] {"EMPTY", "PCDATA", "ANY"})); for (DtdType dtd : DtdType.values()) { + if (dtd.getStatus() != DtdType.DtdStatus.active) continue; if (dtd.rootType != dtd) { continue; } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java index e4bc7741215..b4b06129dee 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java @@ -96,6 +96,7 @@ private void checkOrdering(DtdType type) { public void TestDistinguishing() { for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; if (type == DtdType.ldmlICU) { continue; } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java index 46db897ae8b..805ed2ff88a 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java @@ -187,6 +187,7 @@ public void TestDirectories() throws IOException { public void TestValueAttributesWithChildren() { Multimap m = TreeMultimap.create(); for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; if (type == DtdType.ldmlICU) { continue; } @@ -355,6 +356,7 @@ private String showPath(List parents) { public void TestNewDtdData() { for (DtdType type : DtdType.values()) { + if (type.getStatus() != DtdType.DtdStatus.active) continue; if (type == DtdType.ldmlICU) { continue; } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java index bbfbc007e97..f2629488bad 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java @@ -1269,6 +1269,7 @@ public void TestCompletenessNonLdmlDtd() { Multimap pathValuePairs = LinkedListMultimap.create(); // get all the directories containing non-Ldml dtd files for (DtdType dtdType : DtdType.values()) { + if (dtdType.getStatus() != DtdType.DtdStatus.active) continue; if (dtdType == DtdType.ldml || dtdType == DtdType.ldmlICU || dtdType == DtdType.keyboard3 diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java index 88cbcce72ac..52269861920 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java @@ -377,6 +377,7 @@ public boolean check(XPathParts parts, String fullName) { public void show(int inclusion) { for (DtdType dtdType : DtdType.values()) { + if (dtdType.getStatus() != DtdType.DtdStatus.active) continue; if (dtdType == DtdType.ldmlICU) { continue; } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java index 09b4ac35bdd..76536a5198b 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java @@ -1,5 +1,8 @@ package org.unicode.cldr.util; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + import java.io.File; import org.junit.jupiter.api.Test; @@ -35,4 +38,13 @@ public static final boolean canUseArchiveDirectory() { } return true; // OK to use } + + @Test + void TestReadPrevSDI() { + assumeTrue(canUseArchiveDirectory()); + SupplementalDataInfo SDI_LAST = + SupplementalDataInfo.getInstance( + CLDRPaths.LAST_RELEASE_DIRECTORY + "common/supplemental/"); + assertNotNull(SDI_LAST); + } } From a5d79ebb97a29b319c034325f1e334407ca951c4 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 30 Aug 2023 20:32:43 -0500 Subject: [PATCH 04/10] CLDR-16393 fix breakage with old DTDs - skip ldmlICU versions that are missing - ElementAttributeInfo needs to skip XML files that are missing --- .../org/unicode/cldr/tool/CldrVersion.java | 7 +++ .../cldr/util/ElementAttributeInfo.java | 63 ++++++++++--------- .../org/unicode/cldr/unittest/TestBasic.java | 32 ++++++++-- 3 files changed, 69 insertions(+), 33 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java index a977cecf742..d3e6c48e8eb 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java @@ -12,6 +12,8 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; + +import org.unicode.cldr.util.CLDRFile; import org.unicode.cldr.util.CLDRPaths; import org.unicode.cldr.util.SupplementalDataInfo; @@ -67,6 +69,7 @@ public enum CldrVersion { v42_0, v43_0, v44_0, + /** @see CLDRFile#GEN_VERSION */ baseline; private final String baseDirectory; @@ -98,6 +101,10 @@ public static CldrVersion from(VersionInfo versionInfo) { } public static CldrVersion from(String versionString) { + // treat 'current' as baseline + if (versionString.equals(CLDRFile.GEN_VERSION) || versionString.equals(CLDRFile.GEN_VERSION + ".0")) { + return CldrVersion.baseline; + } return valueOf( versionString.charAt(0) < 'A' ? "v" + versionString.replace('.', '_') diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java index 85c02e1affb..790ab0fd75e 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java @@ -55,35 +55,7 @@ public static final ElementAttributeInfo getInstance(String commonDirectory, Dtd } } if (result == null) { - result = new HashMap<>(); - // pick short files that are in repository - // Add to this when a DTD is added - result.put( - DtdType.ldml, - new ElementAttributeInfo( - canonicalCommonDirectory + "/main/root.xml", DtdType.ldml)); - result.put( - DtdType.supplementalData, - new ElementAttributeInfo( - canonicalCommonDirectory + "/supplemental/plurals.xml", - DtdType.supplementalData)); - result.put( - DtdType.ldmlBCP47, - new ElementAttributeInfo( - canonicalCommonDirectory + "/bcp47/calendar.xml", - DtdType.ldmlBCP47)); - result.put( - DtdType.keyboard3, - new ElementAttributeInfo( - canonicalCommonDirectory - + "/../keyboards/3.0/fr-t-k0-azerty.xml", - DtdType.keyboard3)); - result.put( - DtdType.keyboardTest3, - new ElementAttributeInfo( - canonicalCommonDirectory - + "/../keyboards/test/fr-t-k0-azerty-test.xml", - DtdType.keyboardTest3)); + result = makeElementAttributeInfoMap(canonicalCommonDirectory); cache.put(commonDirectory, result); cache.put(canonicalCommonDirectory, result); } @@ -101,6 +73,39 @@ public static final ElementAttributeInfo getInstance(String commonDirectory, Dtd return eai; } + private static void addElementAttributeInfo(Map result, DtdType type, String path) throws IOException { + if (!new File(path).canRead()) { + System.err.println("ElementAttributeInfo: Warning: Sample file did not exist: " + path + " for DtdType " + type.name()); + return; // file doesn't exist. + } + result.put(type, new ElementAttributeInfo(path, type)); + } + + private static Map makeElementAttributeInfoMap(String canonicalCommonDirectory) throws IOException { + Map result; + result = new HashMap<>(); + // pick short files that are in repository + // Add to this when a DTD is added + addElementAttributeInfo(result, + DtdType.ldml, + canonicalCommonDirectory + "/main/root.xml"); + addElementAttributeInfo(result, + DtdType.supplementalData, + canonicalCommonDirectory + "/supplemental/plurals.xml"); + addElementAttributeInfo(result, + DtdType.ldmlBCP47, + canonicalCommonDirectory + "/bcp47/calendar.xml"); + addElementAttributeInfo(result, + DtdType.keyboard3, + canonicalCommonDirectory + + "/../keyboards/3.0/fr-t-k0-azerty.xml"); + addElementAttributeInfo(result, + DtdType.keyboardTest3, + canonicalCommonDirectory + + "/../keyboards/test/fr-t-k0-azerty-test.xml"); + return result; + } + // static { // try { // addFromDTD(CldrUtility.COMMON_DIRECTORY + "main/en.xml", DtdType.ldml); diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java index ddc2364514d..f54c5bc115d 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java @@ -31,6 +31,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.EnumSet; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; @@ -61,6 +62,7 @@ import org.unicode.cldr.util.DtdData.Attribute; import org.unicode.cldr.util.DtdData.Element; import org.unicode.cldr.util.DtdData.ElementType; +import org.unicode.cldr.util.DtdType.DtdStatus; import org.unicode.cldr.util.DtdType; import org.unicode.cldr.util.ElementAttributeInfo; import org.unicode.cldr.util.Factory; @@ -1005,6 +1007,8 @@ static boolean isLikelyEquivalent(String locale1, String locale2) { static final Map likelyData = SUPPLEMENTAL_DATA_INFO.getLikelySubtags(); + private static final EnumSet badLdmlICUVersions = EnumSet.of(CldrVersion.v1_1_1, CldrVersion.v1_2, CldrVersion.v1_4_1, CldrVersion.v1_5_1); + public void TestLikelySubtagsComplete() { LanguageTagParser ltp = new LanguageTagParser(); for (String locale : testInfo.getCldrFactory().getAvailable()) { @@ -1228,6 +1232,10 @@ public void TestBasicDTDCompatibility() { // test all DTDs for (DtdType dtd : DtdType.values()) { if (dtd.getStatus() != DtdType.DtdStatus.active) continue; + if (dtd.firstVersion != null && CldrVersion.LAST_RELEASE_VERSION.isOlderThan(CldrVersion.from(dtd.firstVersion))) { + continue; // DTD didn't exist in last release + } + if (dtd == DtdType.ldmlICU) continue; try { ElementAttributeInfo oldDtd = ElementAttributeInfo.getInstance(oldCommon, dtd); ElementAttributeInfo newDtd = ElementAttributeInfo.getInstance(dtd); @@ -1401,16 +1409,26 @@ public void TestDtdCompatibility() { if (version == CldrVersion.unknown || version == CldrVersion.baseline) { continue; } + if (type.getStatus() != DtdStatus.active) { + continue; // not active + } + if (type.firstVersion != null && version.isOlderThan(CldrVersion.from(type.firstVersion))) { + continue; // didn't exist at that point + } DtdData dtdDataOld; try { dtdDataOld = DtdData.getInstance(type, version.toString()); } catch (IllegalArgumentException e) { boolean tooOld = false; switch (type) { - case ldmlBCP47: case ldmlICU: + tooOld = badLdmlICUVersions.contains(version); + break; + case ldmlBCP47: case keyboard3: - tooOld = version.isOlderThan(CldrVersion.from(type.firstVersion)); + if (type.firstVersion != null) { + tooOld = version.isOlderThan(CldrVersion.from(type.firstVersion)); + } break; default: break; @@ -1419,8 +1437,8 @@ public void TestDtdCompatibility() { continue; } else { errln( - version - + ": " + "v" + version + + ": " + e.getClass().getSimpleName() + ", " + e.getMessage()); @@ -1450,6 +1468,12 @@ public void TestDtdCompatibility() { continue; } Element newChild = newElement.getChildNamed(oldChild.getName()); + // skip certain items + if (version.isOlderThan(CldrVersion.v1_6_1) && newElement.getName().equals("zone") && oldChild.getName().equals("usesMetazone")) { + if (logKnownIssue("CLDR-17054", "Breakage with items older than 1.6.1: " + newElement.getName() + " / " + oldChild.getName())) { + continue; + } + } if (knownChildExceptions.contains( Pair.of(newElement.getName(), oldChild.getName()))) { From 50d887705f757fbe7e077f607c03d15e2c8f0432 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 30 Aug 2023 20:36:51 -0500 Subject: [PATCH 05/10] CLDR-16393 spotless --- .../org/unicode/cldr/tool/CldrVersion.java | 4 +-- .../cldr/util/ElementAttributeInfo.java | 13 +++++++-- .../org/unicode/cldr/unittest/TestBasic.java | 29 ++++++++++++++----- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java index d3e6c48e8eb..3ef06b939e8 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java @@ -12,7 +12,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; - import org.unicode.cldr.util.CLDRFile; import org.unicode.cldr.util.CLDRPaths; import org.unicode.cldr.util.SupplementalDataInfo; @@ -102,7 +101,8 @@ public static CldrVersion from(VersionInfo versionInfo) { public static CldrVersion from(String versionString) { // treat 'current' as baseline - if (versionString.equals(CLDRFile.GEN_VERSION) || versionString.equals(CLDRFile.GEN_VERSION + ".0")) { + if (versionString.equals(CLDRFile.GEN_VERSION) + || versionString.equals(CLDRFile.GEN_VERSION + ".0")) { return CldrVersion.baseline; } return valueOf( diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java index 790ab0fd75e..aa44f95dfc1 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java @@ -73,15 +73,22 @@ public static final ElementAttributeInfo getInstance(String commonDirectory, Dtd return eai; } - private static void addElementAttributeInfo(Map result, DtdType type, String path) throws IOException { + private static void addElementAttributeInfo( + Map result, DtdType type, String path) + throws IOException { if (!new File(path).canRead()) { - System.err.println("ElementAttributeInfo: Warning: Sample file did not exist: " + path + " for DtdType " + type.name()); + System.err.println( + "ElementAttributeInfo: Warning: Sample file did not exist: " + + path + + " for DtdType " + + type.name()); return; // file doesn't exist. } result.put(type, new ElementAttributeInfo(path, type)); } - private static Map makeElementAttributeInfoMap(String canonicalCommonDirectory) throws IOException { + private static Map makeElementAttributeInfoMap( + String canonicalCommonDirectory) throws IOException { Map result; result = new HashMap<>(); // pick short files that are in repository diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java index f54c5bc115d..1eba28dec56 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java @@ -62,8 +62,8 @@ import org.unicode.cldr.util.DtdData.Attribute; import org.unicode.cldr.util.DtdData.Element; import org.unicode.cldr.util.DtdData.ElementType; -import org.unicode.cldr.util.DtdType.DtdStatus; import org.unicode.cldr.util.DtdType; +import org.unicode.cldr.util.DtdType.DtdStatus; import org.unicode.cldr.util.ElementAttributeInfo; import org.unicode.cldr.util.Factory; import org.unicode.cldr.util.InputStreamFactory; @@ -1007,7 +1007,9 @@ static boolean isLikelyEquivalent(String locale1, String locale2) { static final Map likelyData = SUPPLEMENTAL_DATA_INFO.getLikelySubtags(); - private static final EnumSet badLdmlICUVersions = EnumSet.of(CldrVersion.v1_1_1, CldrVersion.v1_2, CldrVersion.v1_4_1, CldrVersion.v1_5_1); + private static final EnumSet badLdmlICUVersions = + EnumSet.of( + CldrVersion.v1_1_1, CldrVersion.v1_2, CldrVersion.v1_4_1, CldrVersion.v1_5_1); public void TestLikelySubtagsComplete() { LanguageTagParser ltp = new LanguageTagParser(); @@ -1232,7 +1234,9 @@ public void TestBasicDTDCompatibility() { // test all DTDs for (DtdType dtd : DtdType.values()) { if (dtd.getStatus() != DtdType.DtdStatus.active) continue; - if (dtd.firstVersion != null && CldrVersion.LAST_RELEASE_VERSION.isOlderThan(CldrVersion.from(dtd.firstVersion))) { + if (dtd.firstVersion != null + && CldrVersion.LAST_RELEASE_VERSION.isOlderThan( + CldrVersion.from(dtd.firstVersion))) { continue; // DTD didn't exist in last release } if (dtd == DtdType.ldmlICU) continue; @@ -1412,7 +1416,8 @@ public void TestDtdCompatibility() { if (type.getStatus() != DtdStatus.active) { continue; // not active } - if (type.firstVersion != null && version.isOlderThan(CldrVersion.from(type.firstVersion))) { + if (type.firstVersion != null + && version.isOlderThan(CldrVersion.from(type.firstVersion))) { continue; // didn't exist at that point } DtdData dtdDataOld; @@ -1437,8 +1442,9 @@ public void TestDtdCompatibility() { continue; } else { errln( - "v" + version - + ": " + "v" + + version + + ": " + e.getClass().getSimpleName() + ", " + e.getMessage()); @@ -1469,8 +1475,15 @@ public void TestDtdCompatibility() { } Element newChild = newElement.getChildNamed(oldChild.getName()); // skip certain items - if (version.isOlderThan(CldrVersion.v1_6_1) && newElement.getName().equals("zone") && oldChild.getName().equals("usesMetazone")) { - if (logKnownIssue("CLDR-17054", "Breakage with items older than 1.6.1: " + newElement.getName() + " / " + oldChild.getName())) { + if (version.isOlderThan(CldrVersion.v1_6_1) + && newElement.getName().equals("zone") + && oldChild.getName().equals("usesMetazone")) { + if (logKnownIssue( + "CLDR-17054", + "Breakage with items older than 1.6.1: " + + newElement.getName() + + " / " + + oldChild.getName())) { continue; } } From 4bd571175b816b2e76154fbaf699cc6cef4a789b Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 29 Nov 2023 14:56:47 -0600 Subject: [PATCH 06/10] CLDR-16393 merge --- .../org/unicode/cldr/tool/CldrVersion.java | 4 ++- .../cldr/util/ElementAttributeInfo.java | 32 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java index 3ef06b939e8..e33907614a1 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java @@ -68,7 +68,9 @@ public enum CldrVersion { v42_0, v43_0, v44_0, - /** @see CLDRFile#GEN_VERSION */ + /** + * @see CLDRFile#GEN_VERSION + */ baseline; private final String baseDirectory; diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java index aa44f95dfc1..239fad781a1 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/ElementAttributeInfo.java @@ -93,23 +93,21 @@ private static Map makeElementAttributeInfoMap( result = new HashMap<>(); // pick short files that are in repository // Add to this when a DTD is added - addElementAttributeInfo(result, - DtdType.ldml, - canonicalCommonDirectory + "/main/root.xml"); - addElementAttributeInfo(result, - DtdType.supplementalData, - canonicalCommonDirectory + "/supplemental/plurals.xml"); - addElementAttributeInfo(result, - DtdType.ldmlBCP47, - canonicalCommonDirectory + "/bcp47/calendar.xml"); - addElementAttributeInfo(result, - DtdType.keyboard3, - canonicalCommonDirectory - + "/../keyboards/3.0/fr-t-k0-azerty.xml"); - addElementAttributeInfo(result, - DtdType.keyboardTest3, - canonicalCommonDirectory - + "/../keyboards/test/fr-t-k0-azerty-test.xml"); + addElementAttributeInfo(result, DtdType.ldml, canonicalCommonDirectory + "/main/root.xml"); + addElementAttributeInfo( + result, + DtdType.supplementalData, + canonicalCommonDirectory + "/supplemental/plurals.xml"); + addElementAttributeInfo( + result, DtdType.ldmlBCP47, canonicalCommonDirectory + "/bcp47/calendar.xml"); + addElementAttributeInfo( + result, + DtdType.keyboard3, + canonicalCommonDirectory + "/../keyboards/3.0/fr-t-k0-azerty.xml"); + addElementAttributeInfo( + result, + DtdType.keyboardTest3, + canonicalCommonDirectory + "/../keyboards/test/fr-t-k0-azerty-test.xml"); return result; } From bb8544032ef0476a2cf07aa31d5f43250f66df41 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 29 Nov 2023 18:34:37 -0600 Subject: [PATCH 07/10] CLDR-16393 code review comments - use braces, spotless --- .../org/unicode/cldr/tool/CheckHtmlFiles.java | 7 +++++-- .../java/org/unicode/cldr/tool/GenerateDtd.java | 5 +++-- .../org/unicode/cldr/unittest/TestBasic.java | 16 ++++++++++++---- .../cldr/unittest/TestComparisonBuilder.java | 12 +++++++++--- .../unicode/cldr/unittest/TestDTDAttributes.java | 4 +++- .../org/unicode/cldr/unittest/TestDtdData.java | 8 ++++++-- .../unicode/cldr/unittest/TestPathHeader.java | 4 +++- .../org/unicode/cldr/unittest/TestPaths.java | 4 +++- 8 files changed, 44 insertions(+), 16 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java index c232c0bd14c..8acd0daae6d 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/CheckHtmlFiles.java @@ -214,8 +214,11 @@ private static void checkForDtd(Data target) { M4 typeToElements = ChainedMap.of(new TreeMap(), new TreeMap(), new TreeMap(), Boolean.class); for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; - if (type == DtdType.ldmlICU) continue; + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } else if (type == DtdType.ldmlICU) { + continue; + } DtdData dtdData = DtdData.getInstance(type); Set elements = dtdData.getElements(); for (Element element : elements) { diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java index ff81a54fbeb..a7b3da08261 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDtd.java @@ -21,8 +21,9 @@ public static void main(String[] args) throws IOException { final DTD2Markdown dtd2md = new DTD2Markdown(); // System.setProperty("show_all", "true"); for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; - if (type == DtdType.ldmlICU) { + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } else if (type == DtdType.ldmlICU) { continue; } DtdData data = DtdData.getInstance(type); diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java index 1eba28dec56..d4925bf8115 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestBasic.java @@ -227,7 +227,9 @@ public void showFoundElements(Relation, String> foundAtt Relation, String> theoryAttributes = Relation.of(new TreeMap, Set>(), TreeSet.class); for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } DtdData dtdData = DtdData.getInstance(type); for (Element element : dtdData.getElementFromName().values()) { String name = element.getName(); @@ -1173,7 +1175,9 @@ private boolean isTopLevel(String localeID) { /** Tests that every dtd item is connected from root */ public void TestDtdCompleteness() { for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } DtdData dtdData = DtdData.getInstance(type); Set descendents = new LinkedHashSet<>(); dtdData.getDescendents(dtdData.ROOT, descendents); @@ -1233,7 +1237,9 @@ public void TestBasicDTDCompatibility() { // test all DTDs for (DtdType dtd : DtdType.values()) { - if (dtd.getStatus() != DtdType.DtdStatus.active) continue; + if (dtd.getStatus() != DtdType.DtdStatus.active) { + continue; + } if (dtd.firstVersion != null && CldrVersion.LAST_RELEASE_VERSION.isOlderThan( CldrVersion.from(dtd.firstVersion))) { @@ -1348,7 +1354,9 @@ private Set containsInOrder(Set superset, Set subset) { public void TestDtdCompatibility() { for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } DtdData dtdData = DtdData.getInstance(type); Map currentElementFromName = dtdData.getElementFromName(); diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java index a7e2fe707aa..dfebc9cac5b 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestComparisonBuilder.java @@ -49,7 +49,9 @@ private void dtdAttributes() { // unless FindDtdOrder also checks attributes. Builder builder = new Builder(Ordering.NATURAL); for (DtdType dtd : DtdType.values()) { - if (dtd.getStatus() != DtdType.DtdStatus.active) continue; + if (dtd.getStatus() != DtdType.DtdStatus.active) { + continue; + } Relation eaInfo = ElementAttributeInfo.getInstance(dtd).getElement2Attributes(); for (String element : eaInfo.keySet()) { @@ -69,7 +71,9 @@ private void dtdAttributes() { logln("Attribute Ordering:\t" + comp.getOrdering().toString()); for (DtdType dtd : DtdType.values()) { - if (dtd.getStatus() != DtdType.DtdStatus.active) continue; + if (dtd.getStatus() != DtdType.DtdStatus.active) { + continue; + } // check that the ordering is right Relation eaInfo = ElementAttributeInfo.getInstance(dtd).getElement2Attributes(); @@ -84,7 +88,9 @@ public void TestDtdElements() { Set specials = new HashSet(Arrays.asList(new String[] {"EMPTY", "PCDATA", "ANY"})); for (DtdType dtd : DtdType.values()) { - if (dtd.getStatus() != DtdType.DtdStatus.active) continue; + if (dtd.getStatus() != DtdType.DtdStatus.active) { + continue; + } if (dtd.rootType != dtd) { continue; } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java index b4b06129dee..588be4eea1b 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDTDAttributes.java @@ -96,7 +96,9 @@ private void checkOrdering(DtdType type) { public void TestDistinguishing() { for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } if (type == DtdType.ldmlICU) { continue; } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java index 805ed2ff88a..2d3215d6f4d 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDtdData.java @@ -187,7 +187,9 @@ public void TestDirectories() throws IOException { public void TestValueAttributesWithChildren() { Multimap m = TreeMultimap.create(); for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } if (type == DtdType.ldmlICU) { continue; } @@ -356,7 +358,9 @@ private String showPath(List parents) { public void TestNewDtdData() { for (DtdType type : DtdType.values()) { - if (type.getStatus() != DtdType.DtdStatus.active) continue; + if (type.getStatus() != DtdType.DtdStatus.active) { + continue; + } if (type == DtdType.ldmlICU) { continue; } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java index f2629488bad..ee0225cb3ed 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPathHeader.java @@ -1269,7 +1269,9 @@ public void TestCompletenessNonLdmlDtd() { Multimap pathValuePairs = LinkedListMultimap.create(); // get all the directories containing non-Ldml dtd files for (DtdType dtdType : DtdType.values()) { - if (dtdType.getStatus() != DtdType.DtdStatus.active) continue; + if (dtdType.getStatus() != DtdType.DtdStatus.active) { + continue; + } if (dtdType == DtdType.ldml || dtdType == DtdType.ldmlICU || dtdType == DtdType.keyboard3 diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java index 52269861920..b4d205adde5 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestPaths.java @@ -377,7 +377,9 @@ public boolean check(XPathParts parts, String fullName) { public void show(int inclusion) { for (DtdType dtdType : DtdType.values()) { - if (dtdType.getStatus() != DtdType.DtdStatus.active) continue; + if (dtdType.getStatus() != DtdType.DtdStatus.active) { + continue; + } if (dtdType == DtdType.ldmlICU) { continue; } From 50bf437ddd77444c271cb52c890a8c87392c1c41 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 29 Nov 2023 18:47:06 -0600 Subject: [PATCH 08/10] CLDR-16393 change the property to -DHAS_CLDR_ARCHIVE - also refactor in test code --- .../java/org/unicode/cldr/util/CLDRURLS.java | 2 ++ .../org/unicode/cldr/util/TestCLDRPaths.java | 20 ++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java index a4f76dcdf5a..7514cdeae0f 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java @@ -161,6 +161,8 @@ public enum Special { public static final String UPDATING_CURRENCY_CODES = "https://cldr.unicode.org/development/updating-codes/update-currency-codes"; + public static final String CLDR_ARCHIVE = "https://cldr.unicode.org/development/creating-the-archive"; + /** * Get the relative base URL for the SurveyTool. This may be "/cldr-apps", for example. * diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java index 76536a5198b..3e2f980cf3a 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java @@ -7,13 +7,17 @@ import org.junit.jupiter.api.Test; public class TestCLDRPaths { + public static String HAS_CLDR_ARCHIVE = "HAS_CLDR_ARCHIVE"; @Test void TestCanUseArchiveDirectory() { if (!canUseArchiveDirectory()) { - // We print this warning as part of the unit tests. + // We print this warning once as part of the unit tests. System.err.println( - "WARNING: skipping using cldr-archive. Ideally, -DNO_CLDR_ARCHIVE=false and see "); + String.format( + "WARNING: -D%s=false, so skipping tests which use cldr-archive\n" + + "See <%s>", + HAS_CLDR_ARCHIVE, CLDRURLS.CLDR_ARCHIVE)); } } @@ -21,8 +25,8 @@ void TestCanUseArchiveDirectory() { * @return true if it's OK to read CLDRPaths.ARCHIVE_DIRECTORY, false to skip. */ public static final boolean canUseArchiveDirectory() { - if (CLDRConfig.getInstance().getProperty("NO_CLDR_ARCHIVE", false)) { - return false; // skip, NO_CLDR_ARCHIVE is set. + if (!CLDRConfig.getInstance().getProperty("HAS_CLDR_ARCHIVE", true)) { + return false; // skip due to property } final File archiveDir = new File(CLDRPaths.ARCHIVE_DIRECTORY); @@ -31,16 +35,18 @@ public static final boolean canUseArchiveDirectory() { String.format( "Could not read archive directory %s. " + "Please: " - + "1) setup the archive, , " + + "1) setup the archive, <%s>, " + "2) set the -DARCHIVE= property to the correct archive location, or " - + "3) inhibit reading of cldr-archive with -DNO_CLDR_ARCHIVE=true", - archiveDir.getAbsolutePath())); + + "3) inhibit reading of cldr-archive with -D%s=false", + archiveDir.getAbsolutePath(), CLDRURLS.CLDR_ARCHIVE, HAS_CLDR_ARCHIVE)); } return true; // OK to use } @Test void TestReadPrevSDI() { + // This is also an example of a test that's skipped (by the next line) if + // cldr-archive isn't available. assumeTrue(canUseArchiveDirectory()); SupplementalDataInfo SDI_LAST = SupplementalDataInfo.getInstance( From 84a677c40c8c6f4860e9d2c90a2b8c3f043b7f3e Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 29 Nov 2023 18:53:40 -0600 Subject: [PATCH 09/10] CLDR-16393 update failure message --- .../src/test/java/org/unicode/cldr/util/TestCLDRPaths.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java index 3e2f980cf3a..ca5bee780f5 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java @@ -33,11 +33,8 @@ public static final boolean canUseArchiveDirectory() { if (!archiveDir.isDirectory()) { throw new IllegalArgumentException( String.format( - "Could not read archive directory %s. " - + "Please: " - + "1) setup the archive, <%s>, " - + "2) set the -DARCHIVE= property to the correct archive location, or " - + "3) inhibit reading of cldr-archive with -D%s=false", + "Could not read archive directory -DARCHIVE=%s … You must either: 1) follow instructions at <%s>, " + + "or 2) skip cldr-archive tests by setting -D%s=false", archiveDir.getAbsolutePath(), CLDRURLS.CLDR_ARCHIVE, HAS_CLDR_ARCHIVE)); } return true; // OK to use From d29b102b0f4141b3c40ef6a4721ba0bf8d7fdabe Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 29 Nov 2023 19:14:14 -0600 Subject: [PATCH 10/10] CLDR-16393 spotless --- .../src/main/java/org/unicode/cldr/util/CLDRURLS.java | 3 ++- .../src/test/java/org/unicode/cldr/util/TestCLDRPaths.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java index 7514cdeae0f..1be46a7a1b1 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRURLS.java @@ -161,7 +161,8 @@ public enum Special { public static final String UPDATING_CURRENCY_CODES = "https://cldr.unicode.org/development/updating-codes/update-currency-codes"; - public static final String CLDR_ARCHIVE = "https://cldr.unicode.org/development/creating-the-archive"; + public static final String CLDR_ARCHIVE = + "https://cldr.unicode.org/development/creating-the-archive"; /** * Get the relative base URL for the SurveyTool. This may be "/cldr-apps", for example. diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java index ca5bee780f5..979422acb9a 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCLDRPaths.java @@ -15,8 +15,8 @@ void TestCanUseArchiveDirectory() { // We print this warning once as part of the unit tests. System.err.println( String.format( - "WARNING: -D%s=false, so skipping tests which use cldr-archive\n" + - "See <%s>", + "WARNING: -D%s=false, so skipping tests which use cldr-archive\n" + + "See <%s>", HAS_CLDR_ARCHIVE, CLDRURLS.CLDR_ARCHIVE)); } }