CLDR-17792 Don't strip ZWSP for value compare, allow currency symbol …

…collision for it
unicode-org · Jul 13, 2024 · 3f3e93d · 3f3e93d
1 parent 2dd4e08
commit 3f3e93d
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 2 deletions.
diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/test/CheckDisplayCollisions.java b/tools/cldr-code/src/main/java/org/unicode/cldr/test/CheckDisplayCollisions.java
@@ -491,6 +491,15 @@ public CheckCLDR handleCheck(
             if (path.contains("/decimal") || path.contains("/group")) {
                 return this;
             }
+            // Currency symbol value \u200B (ZWSP) is allowed to match other paths; when it is used,
+            // the actual currency symbol must be in a decimal element for the specific currency.
+            if (path.contains("/symbol") && value.equals("\\u200B")) {
+                String decimalPath = path.replace("/symbol", "/decimal");
+                String decimalValue = getResolvedCldrFileToCheck().getWinningValue(decimalPath);
+                if (decimalValue != null && decimalValue.length() > 0) {
+                    return this;
+                }
+            }
             XPathParts parts = XPathParts.getFrozenInstance(path);
             String currency = parts.getAttributeValue(-2, "type");
             Iterator<String> iterator = paths.iterator();

diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/SimpleXMLSource.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/SimpleXMLSource.java
@@ -170,10 +170,12 @@ public void getPathsWithValue(String valueToMatch, String pathPrefix, Set<String
     static final Normalizer2 NFKC = Normalizer2.getNFKCInstance();
 
     // The following includes letters, marks, numbers, currencies, and *selected*
-    // symbols/punctuation
+    // symbols/punctuation.
+    // Also \u200B ZWSP which has a special function, should not strip it when
+    // normalizing values for comparison.
     static final UnicodeSet NON_ALPHANUM =
             new UnicodeSet(
-                            "[^[:L:][:M:][:N:][:Sc:][\\u202F\uFFFF _ ¡ « ( ) \\- \\[ \\] \\{ \\} § / \\\\ % ٪ ‰ ؉ ‱-″ ` \\^ ¯ ¨ ° + ¬ | ¦ ~ − ⊕ ⍰ ☉ © ®]]")
+                            "[^[:L:][:M:][:N:][:Sc:][\\u200B\\u202F\uFFFF _ ¡ « ( ) \\- \\[ \\] \\{ \\} § / \\\\ % ٪ ‰ ؉ ‱-″ ` \\^ ¯ ¨ ° + ¬ | ¦ ~ − ⊕ ⍰ ☉ © ®]]")
                     .freeze();
 
     public static String normalize(String valueToMatch) {

diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestCheckDisplayCollisions.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestCheckDisplayCollisions.java
@@ -275,6 +275,19 @@ public void checkDayPeriodCollisions() {
         checkDisplayCollisions("fil", filPathValuePairs, filFactory);
     }
 
+    public void checkCurrencySymbolCollisions() {
+        // CLDR-17792
+        Map<String, String> pt_PTPathValuePairs =
+                ImmutableMap.of(
+                        "ldml/numbers/currencies/currency[@type=\"EUR\"]/symbol", "€",
+                        "ldml/numbers/currencies/currency[@type=\"GBP\"]/symbol", "£",
+                        "ldml/numbers/currencies/currency[@type=\"PTE\"]/symbol", "\u200B",
+                        "ldml/numbers/currencies/currency[@type=\"PTE\"]/decimal", "$",
+                        "ldml/numbers/currencies/currency[@type=\"USD\"]/symbol", "US$");
+        TestFactory pt_PTFactory = makeFakeCldrFile("pt_PT", pt_PTPathValuePairs);
+        checkDisplayCollisions("pt_PT", pt_PTPathValuePairs, pt_PTFactory);
+    }
+
     public void checkDisplayCollisions(
             String locale, Map<String, String> pathValuePairs, TestFactory factory) {
         CheckDisplayCollisions cdc = new CheckDisplayCollisions(factory);

diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDisplayAndInputProcessor.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDisplayAndInputProcessor.java
@@ -600,6 +600,13 @@ public static PathSpaceData[] getArray() {
                 // PathSpaceData("//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod[@type=\"am\"]",
                 //    " pizza \u00A0\u202F cardboard ", "pizza\u202Fcardboard",
                 //    PathSpaceType.allowNNbsp),
+
+                // Verify that ZWSP u200B as symbol for PTE (in e.g. pt_PT) is not altered
+                new PathSpaceData(
+                        "//ldml/numbers/currencies/currency[@type=\"PTE\"]/symbol",
+                        "\u200B",
+                        "\u200B",
+                        PathSpaceType.allowSpOrNbsp),
             };
             return a;
         }