Skip to content

Commit

Permalink
CLDR-15954 Add test of unit preferences test
Browse files Browse the repository at this point in the history
  • Loading branch information
@macchiati
macchiati committed Mar 16, 2024
1 parent b828ffb commit cfe4a4e
Show file tree
Hide file tree
Showing 4 changed files with 470 additions and 35 deletions.
2 changes: 1 addition & 1 deletion common/supplemental/units.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ For terms of use, see http://www.unicode.org/copyright.html
<convertUnit source='revolution' baseUnit='revolution' systems="metric_adjacent ussystem uksystem"/>

<!-- substance-amount -->
<convertUnit source='item' baseUnit='item' systems="metric_adjacent ussystem uksystem"/>
<convertUnit source='item' baseUnit='item' systems="metric ussystem uksystem"/>
<convertUnit source='mole' baseUnit='item' factor='item_per_mole' systems="si metric prefixable"/>

<!-- portion -->
Expand Down
27 changes: 27 additions & 0 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.google.common.collect.TreeMultimap;
import com.ibm.icu.impl.Row.R2;
import com.ibm.icu.lang.UCharacter;
import com.ibm.icu.number.UnlocalizedNumberFormatter;
import com.ibm.icu.text.PluralRules;
import com.ibm.icu.util.Freezable;
import com.ibm.icu.util.Output;
Expand Down Expand Up @@ -2236,4 +2237,30 @@ public String resolve(String unit) {
String resolved = unitId.resolve().toString();
return getStandardUnit(resolved.isBlank() ? unit : resolved);
}

public String format(
final String languageTag,
Rational outputAmount,
final String unit,
UnlocalizedNumberFormatter nf3) {
final CLDRConfig config = CLDRConfig.getInstance();
Factory factory = config.getCldrFactory();
int pos = languageTag.indexOf("-u");
String localeBase =
(pos < 0 ? languageTag : languageTag.substring(0, pos)).replace('-', '_');
CLDRFile localeFile = factory.make(localeBase, true);
PluralRules pluralRules =
config.getSupplementalDataInfo()
.getPluralRules(
localeBase, com.ibm.icu.text.PluralRules.PluralType.CARDINAL);
String pluralCategory = pluralRules.select(outputAmount.doubleValue());
String path =
UnitPathType.unit.getTranslationPath(
localeFile, "long", unit, pluralCategory, "nominative", "neuter");
String pattern = localeFile.getStringValue(path);
final ULocale uLocale = ULocale.forLanguageTag(languageTag);
String cldrFormattedNumber =
nf3.locale(uLocale).format(outputAmount.doubleValue()).toString();
return com.ibm.icu.text.MessageFormat.format(pattern, cldrFormattedNumber);
}
}
149 changes: 135 additions & 14 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPreferences.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,25 @@

import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Multimap;
import com.ibm.icu.impl.locale.XCldrStub.ImmutableMap;
import com.ibm.icu.util.Freezable;
import com.ibm.icu.util.Output;
import com.ibm.icu.util.ULocale;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import org.unicode.cldr.tool.LikelySubtags;
import org.unicode.cldr.util.UnitConverter.ConversionInfo;

public class UnitPreferences implements Freezable<UnitPreferences> {
Map<String, Map<String, Multimap<Set<String>, UnitPreference>>> quantityToUsageToRegionsToInfo =
Expand Down Expand Up @@ -93,8 +99,9 @@ public void add(
Rational newGeq = geq == null || geq.isEmpty() ? Rational.ONE : Rational.of(geq);
final UnitPreference newUnitPref = new UnitPreference(newGeq, unit, skeleton);

regionsToInfo.put(
ImmutableSet.copyOf(new TreeSet<>(SPLIT_SPACE.splitToList(regions))), newUnitPref);
final ImmutableSet<String> regionSet =
ImmutableSet.copyOf(new TreeSet<>(SPLIT_SPACE.splitToList(regions)));
boolean old = regionsToInfo.put(regionSet, newUnitPref);
}

boolean frozen;
Expand Down Expand Up @@ -195,44 +202,158 @@ public String getPath(
*
* @return
*/
public Map<String, Map<String, Map<String, UnitPreference>>> getFastMap(
UnitConverter converter) {
Map<String, Map<String, Map<String, UnitPreference>>> result = new LinkedHashMap<>();
private Map<String, Map<String, Multimap<String, UnitPreference>>> getRawFastMap() {
UnitConverter converter = SupplementalDataInfo.getInstance().getUnitConverter();
Map<String, Map<String, Multimap<String, UnitPreference>>> result = new LinkedHashMap<>();
for (Entry<String, Map<String, Multimap<Set<String>, UnitPreference>>> entry1 :
quantityToUsageToRegionsToInfo.entrySet()) {
String quantity = entry1.getKey();
Map<String, Map<String, UnitPreference>> result2 = new LinkedHashMap<>();
Map<String, Multimap<String, UnitPreference>> result2 = new LinkedHashMap<>();
result.put(quantity, result2);

for (Entry<String, Multimap<Set<String>, UnitPreference>> entry2 :
entry1.getValue().entrySet()) {
String usage = entry2.getKey();
Map<String, UnitPreference> result3 = new LinkedHashMap<>();
Multimap<String, UnitPreference> result3 = LinkedHashMultimap.create();
result2.put(usage, result3);

// split the regions
for (Entry<Set<String>, Collection<UnitPreference>> entry :
entry2.getValue().asMap().entrySet()) {
Set<String> regions = entry.getKey();
int len = entry.getValue().size();
for (UnitPreference up : entry.getValue()) {
String unit = SPLIT_AND.split(up.unit).iterator().next(); // first unit
quantity = converter.getQuantityFromUnit(unit, false);
String baseUnit = converter.getBaseUnitFromQuantity(quantity);
Rational geq = converter.parseRational(String.valueOf(up.geq));
Rational value = converter.convert(geq, unit, baseUnit, false);
if (value.equals(Rational.NaN)) {
converter.convert(geq, unit, baseUnit, true); // debug
Rational baseGeq;
if (--len == 0) { // set last value to least possible
baseGeq = Rational.NEGATIVE_INFINITY;
} else {
Rational geq = converter.parseRational(String.valueOf(up.geq));
baseGeq = converter.convert(geq, unit, baseUnit, false);
if (baseGeq.equals(Rational.NaN)) {
converter.convert(geq, unit, baseUnit, true); // debug
}
}
UnitPreference up2 = new UnitPreference(value, up.unit, up.skeleton);
UnitPreference up2 = new UnitPreference(baseGeq, up.unit, up.skeleton);
for (String region : regions) {
result3.put(region, up2);
}
}
}
}
}
return ImmutableMap.copyOf(result);
return CldrUtility.protectCollection(result);
}

Supplier<Map<String, Map<String, Multimap<String, UnitPreference>>>>
quantityToUsageToRegionToInfo = Suppliers.memoize(() -> getRawFastMap());

public Map<String, Map<String, Multimap<String, UnitPreference>>> getFastMap() {
return quantityToUsageToRegionToInfo.get();
}

public UnitPreference getUnitPreference(
Rational sourceAmount, String sourceUnit, String usage, ULocale locale) {
UnitConverter converter = SupplementalDataInfo.getInstance().getUnitConverter();
sourceUnit = converter.fixDenormalized(sourceUnit);

String mu = locale.getUnicodeLocaleType("mu");
// TODO if the value is not a unit, skip
if (mu != null) {
Rational conversion = converter.convert(sourceAmount, sourceUnit, mu, false);
return new UnitPreference(conversion, mu, null);
}

String quantity = converter.getQuantityFromUnit(sourceUnit, false);
String baseUnit = converter.getBaseUnitFromQuantity(quantity);

Map<String, Multimap<String, UnitPreference>> usageToRegionsToInfo =
getFastMap().get(quantity);

// If there is no quantity among the preferences,
// return the metric UnitPreference
if (usageToRegionsToInfo == null) {
String standardUnit = converter.getStandardUnit(sourceUnit);
if (!sourceUnit.equals(standardUnit)) {
Rational conversion =
converter.convert(sourceAmount, sourceUnit, standardUnit, false);
return new UnitPreference(conversion, standardUnit, null);
}
return new UnitPreference(sourceAmount, sourceUnit, null);
}

Multimap<String, UnitPreference> regionToInfo = usageToRegionsToInfo.get(usage);

if (regionToInfo == null) {
regionToInfo = usageToRegionsToInfo.get("default");
}

// normalize for matching
sourceAmount = sourceAmount.abs();
if (sourceAmount.equals(Rational.NaN)) {
sourceAmount = Rational.NEGATIVE_ONE;
}

String region = resolveRegion(locale);
Collection<UnitPreference> infoList = regionToInfo.get(region);
if (infoList == null || infoList.isEmpty()) {
infoList = regionToInfo.get("001");
}

Output<String> baseUnitOutput = new Output<>();
ConversionInfo sourceConversionInfo =
converter.parseUnitId(sourceUnit, baseUnitOutput, false);
Rational baseValue = sourceConversionInfo.convert(sourceAmount);

for (UnitPreference info : infoList) { // data is built to always terminate
if (baseValue.compareTo(info.geq) >= 0) {
return info;
}
}
throw new IllegalArgumentException("Fast map should always terminate");
}

public String resolveRegion(ULocale locale) {
// https://unicode.org/reports/tr35/tr35-info.html#Unit_Preferences
// en-u-rg-uszzzz-ms-ussystem
String ms = locale.getUnicodeLocaleType("ms");
if (ms != null) {
switch (ms) {
case "metric":
return "001";
case "uksystem":
return "GB";
case "ussystem":
return "US";
default:
throw new IllegalArgumentException(
"Illegal ms value in: " + locale.toLanguageTag());
}
}
String rg = locale.getUnicodeLocaleType("rg");
if (rg != null) {
// TODO: check for illegal rg value
return rg.substring(0, 2).toUpperCase(Locale.ROOT);
}
String region = locale.getCountry();
if (!region.isEmpty()) {
return region;
}
LikelySubtags LIKELY = new LikelySubtags();
String maximized = LIKELY.maximize(locale.toLanguageTag());
if (maximized != null) {
return ULocale.getCountry(maximized);
}
return "001";
}

public Set<String> getUsages() {
return usages;
}

public Set<String> getQuantities() {
return getFastMap().keySet();
}
}
Loading

0 comments on commit cfe4a4e

Please sign in to comment.