CLDR-17233 Have DAIP coalesce multiple spaces to strictest; fix in fr…

….xml, reorder zones in root
unicode-org · Nov 27, 2023 · 3bfd2cd · 3bfd2cd
1 parent b07c9d8
commit 3bfd2cd
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 13 deletions.
diff --git a/common/main/fr.xml b/common/main/fr.xml
@@ -11702,7 +11702,7 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
 			<unit type="energy-foodcalorie">
 				<displayName>↑↑↑</displayName>
 				<unitPattern count="one">{0} kcal</unitPattern>
-				<unitPattern count="other">{0}  kcal</unitPattern>
+				<unitPattern count="other">{0} kcal</unitPattern>
 			</unit>
 			<unit type="energy-kilojoule">
 				<displayName>↑↑↑</displayName>

diff --git a/common/main/root.xml b/common/main/root.xml
@@ -2907,6 +2907,9 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
 			<zone type="America/St_Johns">
 				<exemplarCity>St. John’s</exemplarCity>
 			</zone>
+			<zone type="America/Curacao">
+				<exemplarCity>Curaçao</exemplarCity>
+			</zone>
 			<zone type="Africa/Asmera">
 				<exemplarCity>Asmara</exemplarCity>
 			</zone>
@@ -2955,9 +2958,18 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
 			<zone type="Asia/Katmandu">
 				<exemplarCity>Kathmandu</exemplarCity>
 			</zone>
+			<zone type="America/Asuncion">
+				<exemplarCity>Asunción</exemplarCity>
+			</zone>
+			<zone type="Indian/Reunion">
+				<exemplarCity>Réunion</exemplarCity>
+			</zone>
 			<zone type="Atlantic/St_Helena">
 				<exemplarCity>St. Helena</exemplarCity>
 			</zone>
+			<zone type="Africa/Sao_Tome">
+				<exemplarCity>São Tomé</exemplarCity>
+			</zone>
 			<zone type="America/Lower_Princes">
 				<exemplarCity>Lower Prince’s Quarter</exemplarCity>
 			</zone>
@@ -3006,18 +3018,6 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
 			<zone type="Asia/Saigon">
 				<exemplarCity>Ho Chi Minh</exemplarCity>
 			</zone>
-			<zone type="America/Curacao">
-				<exemplarCity>Curaçao</exemplarCity>
-			</zone>
-			<zone type="America/Asuncion">
-				<exemplarCity>Asunción</exemplarCity>
-			</zone>
-			<zone type="Indian/Reunion">
-				<exemplarCity>Réunion</exemplarCity>
-			</zone>
-			<zone type="Africa/Sao_Tome">
-				<exemplarCity>São Tomé</exemplarCity>
-			</zone>
 		</timeZoneNames>
 	</dates>
 	<numbers>

diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/test/DisplayAndInputProcessor.java b/tools/cldr-code/src/main/java/org/unicode/cldr/test/DisplayAndInputProcessor.java
@@ -155,6 +155,13 @@ public class DisplayAndInputProcessor {
     // private static final Pattern SPACE_PLUS_NBSP_TO_NORMALIZE =
     // PatternCache.get("\\u0020+[\\u00A0\\u202F]+");
 
+    // NNBSP among other spaces
+    private static final Pattern NNBSP_AND_OTHER_SPACES =
+            PatternCache.get("[\\u0020\\u00A0]+\\u202F[\\u0020\\u00A0\\u202F]*|\\u202F[\\u0020\\u00A0\\u202F]+");
+    // NBSP among other spaces (after handling NNBSP among other spaces)
+    private static final Pattern NBSP_AND_SPACES =
+            PatternCache.get("\\u0020+\\u00A0[\\u0020\\u00A0]*|\\u00A0[\\u0020\\u00A0]+");
+
     private static final Pattern INITIAL_NBSP = PatternCache.get("^[\\u00A0\\u202F]+");
     private static final Pattern FINAL_NBSP = PatternCache.get("[\\u00A0\\u202F]+$");
 
@@ -1289,6 +1296,12 @@ private String normalizeWhitespace(String path, String value) {
             value = PLACEHOLDER_SPACE_BEFORE.matcher(value).replaceAll("\u00A0{");
         }
 
+        // Finally, replace remaining space combinations with most restrictive type CLDR-17233
+        // If we have NNBSP U+202F in combination with other spaces, keep just it
+        value = NNBSP_AND_OTHER_SPACES.matcher(value).replaceAll("\u202F");
+        // Else if we have NBSP U+00A0 in combination with other spaces, keep just it
+        value = NBSP_AND_SPACES.matcher(value).replaceAll("\u00A0");
+
         return value;
     }