Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CLDR-17792 Don't strip ZWSP for value compare, allow currency symbol collision for it #3867

Merged
merged 1 commit into from
Jul 14, 2024
Merged

CLDR-17792 Don't strip ZWSP for value compare, allow currency symbol collision for it #3867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions tools/cldr-code/src/main/java/org/unicode/cldr/test/CheckDisplayCollisions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,15 @@ public CheckCLDR handleCheck(
if (path.contains("/decimal") || path.contains("/group")) {
return this;
}
// Currency symbol value \u200B (ZWSP) is allowed to match other paths; when it is used,
// the actual currency symbol must be in a decimal element for the specific currency.
if (path.contains("/symbol") && value.equals("\\u200B")) {
String decimalPath = path.replace("/symbol", "/decimal");
String decimalValue = getResolvedCldrFileToCheck().getWinningValue(decimalPath);
if (decimalValue != null && decimalValue.length() > 0) {
return this;
}
}
XPathParts parts = XPathParts.getFrozenInstance(path);
String currency = parts.getAttributeValue(-2, "type");
Iterator<String> iterator = paths.iterator();
Expand Down
6 changes: 4 additions & 2 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/SimpleXMLSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,12 @@ public void getPathsWithValue(String valueToMatch, String pathPrefix, Set<String
static final Normalizer2 NFKC = Normalizer2.getNFKCInstance();

// The following includes letters, marks, numbers, currencies, and *selected*
// symbols/punctuation
// symbols/punctuation.
// Also \u200B ZWSP which has a special function, should not strip it when
// normalizing values for comparison.
static final UnicodeSet NON_ALPHANUM =
new UnicodeSet(
"[^[:L:][:M:][:N:][:Sc:][\\u202F\uFFFF _ ¡ « ( ) \\- \\[ \\] \\{ \\} § / \\\\ % ٪ ‰ ؉ ‱-″ ` \\^ ¯ ¨ ° + ¬ | ¦ ~ − ⊕ ⍰ ☉ © ®]]")
"[^[:L:][:M:][:N:][:Sc:][\\u200B\\u202F\uFFFF _ ¡ « ( ) \\- \\[ \\] \\{ \\} § / \\\\ % ٪ ‰ ؉ ‱-″ ` \\^ ¯ ¨ ° + ¬ | ¦ ~ − ⊕ ⍰ ☉ © ®]]")
.freeze();

public static String normalize(String valueToMatch) {
Expand Down
13 changes: 13 additions & 0 deletions tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestCheckDisplayCollisions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,19 @@ public void checkDayPeriodCollisions() {
checkDisplayCollisions("fil", filPathValuePairs, filFactory);
}

public void checkCurrencySymbolCollisions() {
// CLDR-17792
Map<String, String> pt_PTPathValuePairs =
ImmutableMap.of(
"ldml/numbers/currencies/currency[@type=\"EUR\"]/symbol", "€",
"ldml/numbers/currencies/currency[@type=\"GBP\"]/symbol", "£",
"ldml/numbers/currencies/currency[@type=\"PTE\"]/symbol", "\u200B",
"ldml/numbers/currencies/currency[@type=\"PTE\"]/decimal", "$",
"ldml/numbers/currencies/currency[@type=\"USD\"]/symbol", "US$");
TestFactory pt_PTFactory = makeFakeCldrFile("pt_PT", pt_PTPathValuePairs);
checkDisplayCollisions("pt_PT", pt_PTPathValuePairs, pt_PTFactory);
}

public void checkDisplayCollisions(
String locale, Map<String, String> pathValuePairs, TestFactory factory) {
CheckDisplayCollisions cdc = new CheckDisplayCollisions(factory);
Expand Down
7 changes: 7 additions & 0 deletions tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestDisplayAndInputProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -600,6 +600,13 @@ public static PathSpaceData[] getArray() {
// PathSpaceData("//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod[@type=\"am\"]",
// " pizza \u00A0\u202F cardboard ", "pizza\u202Fcardboard",
// PathSpaceType.allowNNbsp),

// Verify that ZWSP u200B as symbol for PTE (in e.g. pt_PT) is not altered
new PathSpaceData(
"//ldml/numbers/currencies/currency[@type=\"PTE\"]/symbol",
"\u200B",
"\u200B",
PathSpaceType.allowSpOrNbsp),
};
return a;
}
Expand Down
Loading