Skip to content

Commit

Permalink
CLDR-17233 Also coalesce thin + regular space to just thin; then CLDR…
Browse files Browse the repository at this point in the history 
…Modify -fP fixes this in 5 files
  • Loading branch information
@pedberg-icu
pedberg-icu committed Nov 27, 2023
1 parent 48820cc commit 01d9632
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 68 deletions.
10 changes: 5 additions & 5 deletions common/main/hi_Latn.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2211,14 +2211,14 @@ annotations.
<intervalFormatItem id="GyMMMd">
<greatestDifference id="d">G y MMM d–d</greatestDifference>
<greatestDifference id="G">G y MMM d – G y MMM d</greatestDifference>
<greatestDifference id="M">G y MMM d  – MMM d</greatestDifference>
<greatestDifference id="y">G y MMM d –  y d MMM</greatestDifference>
<greatestDifference id="M">G y MMM d – MMM d</greatestDifference>
<greatestDifference id="y">G y MMM d – y d MMM</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="GyMMMEd">
<greatestDifference id="d">G y MMM d, E – MMM d, E</greatestDifference>
<greatestDifference id="G">G y MMM d, E –  G y MMM d, E</greatestDifference>
<greatestDifference id="M">G y MMM d, E –  MMM d, E</greatestDifference>
<greatestDifference id="y">G y MMM d, E –  y MMM d, E</greatestDifference>
<greatestDifference id="G">G y MMM d, E – G y MMM d, E</greatestDifference>
<greatestDifference id="M">G y MMM d, E – MMM d, E</greatestDifference>
<greatestDifference id="y">G y MMM d, E – y MMM d, E</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="h">
<greatestDifference id="a">↑↑↑</greatestDifference>
Expand Down
12 changes: 6 additions & 6 deletions common/main/ku.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1186,9 +1186,9 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<greatestDifference id="y">G d'ê' MMM'a' y'an' – d'ê' MMM'a' y'an'</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMEd">
<greatestDifference id="d">G d'ê' MMM'a' y'an' E  – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="M">G d'ê' MMM'a' y'an' E  – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="y">G d'ê' MMM'a' y'an' E  – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="d">G d'ê' MMM'a' y'an' E – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="M">G d'ê' MMM'a' y'an' E – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="y">G d'ê' MMM'a' y'an' E – d'ê' MMM'a' y'an' E</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMM">
<greatestDifference id="M">G MMMM – MMMM y</greatestDifference>
Expand Down Expand Up @@ -1722,9 +1722,9 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<greatestDifference id="y">d'ê' MMM'a' y'an' – d'ê' MMM'a' y'an'</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMEd">
<greatestDifference id="d">d'ê' MMM'a' y'an' E  – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="M">d'ê' MMM'a' y'an' E  – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="y">d'ê' MMM'a' y'an' E  – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="d">d'ê' MMM'a' y'an' E – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="M">d'ê' MMM'a' y'an' E – d'ê' MMM'a' y'an' E</greatestDifference>
<greatestDifference id="y">d'ê' MMM'a' y'an' E – d'ê' MMM'a' y'an' E</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMM">
<greatestDifference id="M">MMMM – MMMM y</greatestDifference>
Expand Down
4 changes: 2 additions & 2 deletions common/main/pap.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
</intervalFormatItem>
<intervalFormatItem id="GyMMMEd">
<greatestDifference id="d" draft="unconfirmed">E, d MMM – d MMM y G</greatestDifference>
<greatestDifference id="G" draft="unconfirmed">E, d MMM y G  – E, d MMM y G</greatestDifference>
<greatestDifference id="G" draft="unconfirmed">E, d MMM y G – E, d MMM y G</greatestDifference>
<greatestDifference id="M" draft="unconfirmed">E, d MMM – E, d MMM y G</greatestDifference>
<greatestDifference id="y" draft="unconfirmed">E, d MMM y – E, d MMM y G</greatestDifference>
</intervalFormatItem>
Expand Down Expand Up @@ -905,7 +905,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
</intervalFormatItem>
<intervalFormatItem id="MMMd">
<greatestDifference id="d" draft="unconfirmed">d–d MMM</greatestDifference>
<greatestDifference id="M" draft="unconfirmed">d MMM  – d MMM</greatestDifference>
<greatestDifference id="M" draft="unconfirmed">d MMM – d MMM</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MMMEd">
<greatestDifference id="d" draft="unconfirmed">E, d MMM – E, d MMM</greatestDifference>
Expand Down
2 changes: 1 addition & 1 deletion common/main/syr.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1200,7 +1200,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
<appendItem request="Timezone">↑↑↑</appendItem>
</appendItems>
<intervalFormats>
<intervalFormatFallback>{0}  –  {1}</intervalFormatFallback>
<intervalFormatFallback>{0} – {1}</intervalFormatFallback>
<intervalFormatItem id="Bh">
<greatestDifference id="B">↑↑↑</greatestDifference>
<greatestDifference id="h">↑↑↑</greatestDifference>
Expand Down
90 changes: 45 additions & 45 deletions common/main/vi.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2523,15 +2523,15 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
</intervalFormatItem>
<intervalFormatItem id="GyMd">
<greatestDifference id="d">d/M/y – d/M/y GGGGG</greatestDifference>
<greatestDifference id="G">d/M/y GGGGG  –  d/M/y GGGGG</greatestDifference>
<greatestDifference id="G">d/M/y GGGGG – d/M/y GGGGG</greatestDifference>
<greatestDifference id="M">d/M/y – d/M/y GGGGG</greatestDifference>
<greatestDifference id="y">d/M/y – d/M/y GGGGG</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="GyMEd">
<greatestDifference id="d">E, d/M/y  –  E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="G">E, d/M/y GGGGG  –  E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="M">E, d/M/y  –  E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="y">E, d/M/y  –  E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="d">E, d/M/y – E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="G">E, d/M/y GGGGG – E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="M">E, d/M/y – E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="y">E, d/M/y – E, d/M/y GGGGG</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="GyMMM">
<greatestDifference id="G">MMM y G – MMM y G</greatestDifference>
Expand Down Expand Up @@ -2590,19 +2590,19 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<greatestDifference id="M">d/M – d/M</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MEd">
<greatestDifference id="d">E, d/M  –  E, d/M</greatestDifference>
<greatestDifference id="M">E, d/M  –  E, d/M</greatestDifference>
<greatestDifference id="d">E, d/M – E, d/M</greatestDifference>
<greatestDifference id="M">E, d/M – E, d/M</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MMM">
<greatestDifference id="M">MMM  –  MMM</greatestDifference>
<greatestDifference id="M">MMM – MMM</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MMMd">
<greatestDifference id="d">d – d MMM</greatestDifference>
<greatestDifference id="M">d MMM  –  d MMM</greatestDifference>
<greatestDifference id="M">d MMM – d MMM</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MMMEd">
<greatestDifference id="d">E, d MMM  –  E, d MMM</greatestDifference>
<greatestDifference id="M">E, d MMM  –  E, d MMM</greatestDifference>
<greatestDifference id="d">E, d MMM – E, d MMM</greatestDifference>
<greatestDifference id="M">E, d MMM – E, d MMM</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="y">
<greatestDifference id="y">y–y G</greatestDifference>
Expand All @@ -2617,26 +2617,26 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<greatestDifference id="y">d/M/y – d/M/y GGGGG</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMEd">
<greatestDifference id="d">E, d/M/y  –  E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="M">E, d/M/y  –  E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="y">E, d/M/y  –  E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="d">E, d/M/y – E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="M">E, d/M/y – E, d/M/y GGGGG</greatestDifference>
<greatestDifference id="y">E, d/M/y – E, d/M/y GGGGG</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMM">
<greatestDifference id="M">MMM  –  MMM y G</greatestDifference>
<greatestDifference id="y">MMM y  –  MMM y G</greatestDifference>
<greatestDifference id="M">MMM – MMM y G</greatestDifference>
<greatestDifference id="y">MMM y – MMM y G</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMd">
<greatestDifference id="d">d – d MMM, y G</greatestDifference>
<greatestDifference id="M">d MMM  –  d MMM, y G</greatestDifference>
<greatestDifference id="y">d MMM, y  –  d MMM, y G</greatestDifference>
<greatestDifference id="M">d MMM – d MMM, y G</greatestDifference>
<greatestDifference id="y">d MMM, y – d MMM, y G</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMEd">
<greatestDifference id="d">E, d MMM  –  E, d MMM, y G</greatestDifference>
<greatestDifference id="M">E, d MMM  –  E, d MMM, y G</greatestDifference>
<greatestDifference id="y">E, d MMM, y  –  E, d MMM, y G</greatestDifference>
<greatestDifference id="d">E, d MMM – E, d MMM, y G</greatestDifference>
<greatestDifference id="M">E, d MMM – E, d MMM, y G</greatestDifference>
<greatestDifference id="y">E, d MMM, y – E, d MMM, y G</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMM">
<greatestDifference id="M">MMMM  –  MMMM y G</greatestDifference>
<greatestDifference id="M">MMMM – MMMM y G</greatestDifference>
<greatestDifference id="y">MMMM y – MMMM y G</greatestDifference>
</intervalFormatItem>
</intervalFormats>
Expand Down Expand Up @@ -3081,7 +3081,7 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<intervalFormats>
<intervalFormatFallback>↑↑↑</intervalFormatFallback>
<intervalFormatItem id="Bh">
<greatestDifference id="B">h 'giờ' B  –  h 'giờ' B</greatestDifference>
<greatestDifference id="B">h 'giờ' B – h 'giờ' B</greatestDifference>
<greatestDifference id="h">h – h 'giờ' B</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="Bhm">
Expand All @@ -3097,24 +3097,24 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<greatestDifference id="y">y – y G</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="GyM">
<greatestDifference id="G">M/y G  –  M/y G</greatestDifference>
<greatestDifference id="G">M/y G – M/y G</greatestDifference>
<greatestDifference id="M">M/y– M/y G</greatestDifference>
<greatestDifference id="y">M/y – M/y G</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="GyMd">
<greatestDifference id="d">d/M/y – d/M/y G</greatestDifference>
<greatestDifference id="G">d/M/y G  –  d/M/y G</greatestDifference>
<greatestDifference id="G">d/M/y G – d/M/y G</greatestDifference>
<greatestDifference id="M">d/M/y – d/M/y G</greatestDifference>
<greatestDifference id="y">d/M/y – d/M/y G</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="GyMEd">
<greatestDifference id="d">E, d/M/y  –  E, d/M/y G</greatestDifference>
<greatestDifference id="G">E, d/M/y G  –  E, d/M/y G</greatestDifference>
<greatestDifference id="M">E, d/M/y  –  E, d/M/y G</greatestDifference>
<greatestDifference id="y">E, d/M/y  –  E, d/M/y G</greatestDifference>
<greatestDifference id="d">E, d/M/y – E, d/M/y G</greatestDifference>
<greatestDifference id="G">E, d/M/y G – E, d/M/y G</greatestDifference>
<greatestDifference id="M">E, d/M/y – E, d/M/y G</greatestDifference>
<greatestDifference id="y">E, d/M/y – E, d/M/y G</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="GyMMM">
<greatestDifference id="G">MMM y G  –  MMM y G</greatestDifference>
<greatestDifference id="G">MMM y G – MMM y G</greatestDifference>
<greatestDifference id="M">MMM – MMM y G</greatestDifference>
<greatestDifference id="y">MMM y – MMM y G</greatestDifference>
</intervalFormatItem>
Expand Down Expand Up @@ -3170,15 +3170,15 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<greatestDifference id="M">d/M – d/M</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MEd">
<greatestDifference id="d">E, d/M  –  E, d/M</greatestDifference>
<greatestDifference id="M">E, d/M  –  E, d/M</greatestDifference>
<greatestDifference id="d">E, d/M – E, d/M</greatestDifference>
<greatestDifference id="M">E, d/M – E, d/M</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MMM">
<greatestDifference id="M">MMM – MMM</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MMMd">
<greatestDifference id="d">d – d MMM</greatestDifference>
<greatestDifference id="M">d MMM  –  d MMM</greatestDifference>
<greatestDifference id="M">d MMM – d MMM</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="MMMEd">
<greatestDifference id="d">E, d MMM – E, d MMM</greatestDifference>
Expand All @@ -3197,27 +3197,27 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
<greatestDifference id="y">d/M/y – d/M/y</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMEd">
<greatestDifference id="d">E, d/M/y  –  E, d/M/y</greatestDifference>
<greatestDifference id="M">E, d/M/y  –  E, d/M/y</greatestDifference>
<greatestDifference id="y">E, d/M/y  –  E, d/M/y</greatestDifference>
<greatestDifference id="d">E, d/M/y – E, d/M/y</greatestDifference>
<greatestDifference id="M">E, d/M/y – E, d/M/y</greatestDifference>
<greatestDifference id="y">E, d/M/y – E, d/M/y</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMM">
<greatestDifference id="M">MMM  –  MMM y</greatestDifference>
<greatestDifference id="y">MMM y  –  MMM y</greatestDifference>
<greatestDifference id="M">MMM – MMM y</greatestDifference>
<greatestDifference id="y">MMM y – MMM y</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMd">
<greatestDifference id="d">d – d MMM, y</greatestDifference>
<greatestDifference id="M">d MMM  –  d MMM, y</greatestDifference>
<greatestDifference id="y">d MMM, y  –  d MMM, y</greatestDifference>
<greatestDifference id="M">d MMM – d MMM, y</greatestDifference>
<greatestDifference id="y">d MMM, y – d MMM, y</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMEd">
<greatestDifference id="d">E, d MMM  –  E, d MMM, y</greatestDifference>
<greatestDifference id="M">E, d MMM  –  E, d MMM, y</greatestDifference>
<greatestDifference id="y">E, d MMM, y  –  E, d MMM, y</greatestDifference>
<greatestDifference id="d">E, d MMM – E, d MMM, y</greatestDifference>
<greatestDifference id="M">E, d MMM – E, d MMM, y</greatestDifference>
<greatestDifference id="y">E, d MMM, y – E, d MMM, y</greatestDifference>
</intervalFormatItem>
<intervalFormatItem id="yMMMM">
<greatestDifference id="M">MMMM  –  MMMM 'năm' y</greatestDifference>
<greatestDifference id="y">MMMM 'năm' y  –  MMMM 'năm' y</greatestDifference>
<greatestDifference id="M">MMMM – MMMM 'năm' y</greatestDifference>
<greatestDifference id="y">MMMM 'năm' y – MMMM 'năm' y</greatestDifference>
</intervalFormatItem>
</intervalFormats>
</dateTimeFormats>
Expand Down
22 changes: 13 additions & 9 deletions tools/cldr-code/src/main/java/org/unicode/cldr/test/DisplayAndInputProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,13 +155,15 @@ public class DisplayAndInputProcessor {
// private static final Pattern SPACE_PLUS_NBSP_TO_NORMALIZE =
// PatternCache.get("\\u0020+[\\u00A0\\u202F]+");

// NNBSP among other spaces
private static final Pattern NNBSP_AND_OTHER_SPACES =
PatternCache.get(
"[\\u0020\\u00A0]+\\u202F[\\u0020\\u00A0\\u202F]*|\\u202F[\\u0020\\u00A0\\u202F]+");
// NBSP among other spaces (after handling NNBSP among other spaces)
private static final Pattern NBSP_AND_SPACES =
PatternCache.get("\\u0020+\\u00A0[\\u0020\\u00A0]*|\\u00A0[\\u0020\\u00A0]+");
// NNBSP 202F among other horizontal spaces (includes 0020, 00A0, 2009, 202F, etc.)
private static final Pattern NNBSP_AMONG_OTHER_SPACES =
PatternCache.get("[\\h&&[^\\u202F]]+\\u202F\\h*|\\u202F\\h+");
// NBSP 00A0 among other horizontal spaces
private static final Pattern NBSP_AMONG_OTHER_SPACES =
PatternCache.get("[\\h&&[^\\u00A0]]+\\u00A0\\h*|\\u00A0\\h+");
// THIN SPACE 2009 among other horizontal spaces
private static final Pattern THIN_SPACE_AMONG_OTHER_SPACES =
PatternCache.get("[\\h&&[^\\u2009]]+\\u2009\\h*|\\u2009\\h+");

private static final Pattern INITIAL_NBSP = PatternCache.get("^[\\u00A0\\u202F]+");
private static final Pattern FINAL_NBSP = PatternCache.get("[\\u00A0\\u202F]+$");
Expand Down Expand Up @@ -1299,9 +1301,11 @@ private String normalizeWhitespace(String path, String value) {

// Finally, replace remaining space combinations with most restrictive type CLDR-17233
// If we have NNBSP U+202F in combination with other spaces, keep just it
value = NNBSP_AND_OTHER_SPACES.matcher(value).replaceAll("\u202F");
value = NNBSP_AMONG_OTHER_SPACES.matcher(value).replaceAll("\u202F");
// Else if we have NBSP U+00A0 in combination with other spaces, keep just it
value = NBSP_AND_SPACES.matcher(value).replaceAll("\u00A0");
value = NBSP_AMONG_OTHER_SPACES.matcher(value).replaceAll("\u00A0");
// Else if we have THIN SPACE U+2009 in combination with other spaces, keep just it
value = THIN_SPACE_AMONG_OTHER_SPACES.matcher(value).replaceAll("\u2009");

return value;
}
Expand Down

0 comments on commit 01d9632

Please sign in to comment.