diff --git a/common/testData/units/unitLocalePreferencesTest.txt b/common/testData/units/unitLocalePreferencesTest.txt new file mode 100644 index 00000000000..1922607f9a6 --- /dev/null +++ b/common/testData/units/unitLocalePreferencesTest.txt @@ -0,0 +1,42 @@ +# Test data for unit locale preferences +# Copyright © 1991-2024 Unicode, Inc. +# For terms of use, see http://www.unicode.org/copyright.html +# SPDX-License-Identifier: Unicode-3.0 +# CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/) +# +# Format: +# input-unit; amount; usage; languageTag; expected-unit; expected-amount # comment +# +# • The amounts are both rationals +# • The comment is optional (if it isn't present the # can be omitted) +# +# Use: Convert the Input amount & unit according to the Usage and Locale. +# The result should match the Expected amount and unit. +# +# The input and expected output units are unit identifers; in particular, the output does not have further processing: +# • no localization +# +fahrenheit; 1; default; en-u-rg-uszzzz-ms-ussystem-mu-celsius; celsius; -155/9 # mu > ms > rg > (likely) region +fahrenheit; 1; default; en-u-rg-uszzzz-ms-ussystem-mu-celsius; celsius; -155/9 +fahrenheit; 1; default; en-u-rg-uszzzz-ms-metric; celsius; -155/9 +fahrenheit; 1; default; en-u-rg-dezzzz; celsius; -155/9 +fahrenheit; 1; default; en-DE; celsius; -155/9 # explicit region > likely region +fahrenheit; 1; default; en-US; fahrenheit; 1 +fahrenheit; 1; default; en; fahrenheit; 1 # likely region = US +gallon-imperial; 2.5; fluid; en-u-rg-uszzzz-ms-metric; liter; 11.365225 +gallon-imperial; 2.5; fluid; en-u-rg-dezzzz; liter; 11.365225 +gallon-imperial; 2.5; fluid; en-DE; liter; 11.365225 +gallon-imperial; 2.5; fluid; en-US-u-rg-uszzzz-ms-uksystem; gallon-imperial; 2.5 # ms-uksystem should behave like GB +gallon-imperial; 2.5; fluid; en-u-rg-gbzzzz; gallon-imperial; 2.5 +gallon-imperial; 2.5; fluid; en-GB; gallon-imperial; 2.5 +gallon-imperial; 2.5; fluid; en-u-rg-uszzzz-ms-ussystem; gallon; 1,420,653,125/473176473 +gallon-imperial; 2.5; fluid; en-u-rg-uszzzz; gallon; 1,420,653,125/473176473 +gallon-imperial; 2.5; fluid; en-US; gallon; 1,420,653,125/473176473 +gallon-imperial; 2.5; fluid; en; gallon; 1,420,653,125/473176473 # likely region = US +ampere; 2.5; default; en; ampere; 2.5 # an input unit whose quantity has no preference data should get base units +pound-force-foot; 12,345; default; en; kilowatt-hour; 0.004649325714486427205 +kilocandela; 1; default; en; candela; 1,000 # an input unit whose quantity has no preference data should get base units +candela-per-byte; 1; default; en; candela-per-bit; 0.125 # an input unit that has no quantity should get base units +candela-per-cubic-foot; 1; default; en; candela-per-cubic-meter; 1,953,125,000/55306341 # an input unit that has no quantity should get base units +foot; 1; default; de-u-mu-celsius; centimeter; 30.48 # a -mu unit that is not convertible from the input unit should get ignored +#pound; 28; default; en-u-mu-stone; stone; 2 # only temperature units are supported diff --git a/common/testData/units/unitPreferencesTest.txt b/common/testData/units/unitPreferencesTest.txt index d48a9680ce4..dbf3ce4d467 100644 --- a/common/testData/units/unitPreferencesTest.txt +++ b/common/testData/units/unitPreferencesTest.txt @@ -22,7 +22,7 @@ # • no formatted with the skeleton # • no suppression of zero values (for secondary -and- units such as pound in stone-and-pound) # -# Generation: Set GENERATE_TESTS in TestUnits.java to regenerate unitPreferencesTest.txt. +# Generation: Use GenerateUnitTestData.java to regenerate unitPreferencesTest.txt. area; default; 001; 1100000; 1100000.0; square-meter; 11/10; 1.1; square-kilometer area; default; 001; 1000000; 1000000.0; square-meter; 1; 1.0; square-kilometer diff --git a/common/testData/units/unitsTest.txt b/common/testData/units/unitsTest.txt index ea4820078f8..8999a44f8d9 100644 --- a/common/testData/units/unitsTest.txt +++ b/common/testData/units/unitsTest.txt @@ -12,7 +12,7 @@ # round to 4 decimal digits before comparing. # Note that certain conversions are approximate, such as degrees to radians # -# Generation: Set GENERATE_TESTS in TestUnits.java to regenerate unitsTest.txt. +# Generation: Use GenerateUnitTestData.java to regenerate unitsTest.txt. acceleration ; meter-per-square-second ; meter-per-square-second ; 1 * x ; 1,000.00 acceleration ; g-force ; meter-per-square-second ; 9.80665 * x ; 9806.65 diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateUnitTestData.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateUnitTestData.java index 6fbe61294d7..5bf126d657c 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateUnitTestData.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateUnitTestData.java @@ -1,15 +1,22 @@ package org.unicode.cldr.tool; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.collect.TreeMultimap; import com.ibm.icu.util.Output; +import com.ibm.icu.util.ULocale; +import java.io.IOException; +import java.io.UncheckedIOException; import java.math.BigInteger; import java.math.MathContext; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collection; import java.util.Comparator; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -17,10 +24,12 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.regex.Pattern; import org.unicode.cldr.util.CLDRPaths; import org.unicode.cldr.util.CldrUtility; import org.unicode.cldr.util.Pair; import org.unicode.cldr.util.Rational; +import org.unicode.cldr.util.Rational.FormatStyle; import org.unicode.cldr.util.StandardCodes.LstrType; import org.unicode.cldr.util.SupplementalDataInfo; import org.unicode.cldr.util.TempPrintWriter; @@ -47,6 +56,7 @@ public static void main(String[] args) { GenerateUnitTestData item = new GenerateUnitTestData(); item.TestParseUnit(); item.TestUnitPreferences(); + item.generateUnitLocalePreferences(); } static { @@ -117,7 +127,7 @@ public void TestParseUnit() { + "# round to 4 decimal digits before comparing.\n" + "# Note that certain conversions are approximate, such as degrees to radians\n" + "#\n" - + "# Generation: Set GENERATE_TESTS in TestUnits.java to regenerate unitsTest.txt.\n"); + + "# Generation: Use GenerateUnitTestData.java to regenerate unitsTest.txt.\n"); for (Entry, String> entry : testPrintout.entries()) { pw.println(entry.getValue()); } @@ -127,101 +137,179 @@ public void TestParseUnit() { public void TestUnitPreferences() { UnitPreferences prefs = SDI.getUnitPreferences(); - if (true) { - try (TempPrintWriter pw = - TempPrintWriter.openUTF8Writer( - CLDRPaths.TEST_DATA + "units", "unitPreferencesTest.txt")) { + try (TempPrintWriter pw = + TempPrintWriter.openUTF8Writer( + CLDRPaths.TEST_DATA + "units", "unitPreferencesTest.txt")) { + pw.println(getHeader("Region")); + Rational ONE_TENTH = Rational.of(1, 10); - pw.println( - "\n# Test data for unit preferences\n" - + CldrUtility.getCopyrightString("# ") - + "\n" - + "#\n" - + "# Format:\n" - + "#\tQuantity;\tUsage;\tRegion;\tInput (r);\tInput (d);\tInput Unit;\tOutput (r);\tOutput (d);\tOutput Unit\n" - + "#\n" - + "# Use: Convert the Input amount & unit according to the Usage and Region.\n" - + "#\t The result should match the Output amount and unit.\n" - + "#\t Both rational (r) and double64 (d) forms of the input and output amounts are supplied so that implementations\n" - + "#\t have two options for testing based on the precision in their implementations. For example:\n" - + "#\t 3429 / 12500; 0.27432; meter;\n" - + "#\t The Output amount and Unit are repeated for mixed units. In such a case, only the smallest unit will have\n" - + "#\t both a rational and decimal amount; the others will have a single integer value, such as:\n" - + "#\t length; person-height; CA; 3429 / 12500; 0.27432; meter; 2; foot; 54 / 5; 10.8; inch\n" - + "#\t The input and output units are unit identifers; in particular, the output does not have further processing:\n" - + "#\t\t • no localization\n" - + "#\t\t • no adjustment for pluralization\n" - + "#\t\t • no formatted with the skeleton\n" - + "#\t\t • no suppression of zero values (for secondary -and- units such as pound in stone-and-pound)\n" - + "#\n" - + "# Generation: Set GENERATE_TESTS in TestUnits.java to regenerate unitPreferencesTest.txt.\n"); - Rational ONE_TENTH = Rational.of(1, 10); + // Note that for production usage, precomputed data like the + // prefs.getFastMap(converter) would be used instead of the raw data. - // Note that for production usage, precomputed data like the - // prefs.getFastMap(converter) would be used instead of the raw data. + for (Entry, UnitPreference>>> entry : + prefs.getData().entrySet()) { + String quantity = entry.getKey(); + String baseUnit = converter.getBaseUnitFromQuantity(quantity); + for (Entry, UnitPreference>> entry2 : + entry.getValue().entrySet()) { + String usage = entry2.getKey(); - for (Entry, UnitPreference>>> entry : - prefs.getData().entrySet()) { - String quantity = entry.getKey(); - String baseUnit = converter.getBaseUnitFromQuantity(quantity); - for (Entry, UnitPreference>> entry2 : - entry.getValue().entrySet()) { - String usage = entry2.getKey(); - - // collect samples of base units - for (Entry, Collection> entry3 : - entry2.getValue().asMap().entrySet()) { - boolean first = true; - Set samples = new TreeSet<>(Comparator.reverseOrder()); - for (UnitPreference pref : entry3.getValue()) { - final String topUnit = - UnitPreferences.SPLIT_AND - .split(pref.unit) - .iterator() - .next(); - if (first) { - samples.add( - converter.convert( - pref.geq.add(ONE_TENTH), - topUnit, - baseUnit, - false)); - first = false; - } - samples.add(converter.convert(pref.geq, topUnit, baseUnit, false)); + // collect samples of base units + for (Entry, Collection> entry3 : + entry2.getValue().asMap().entrySet()) { + boolean first = true; + Set samples = new TreeSet<>(Comparator.reverseOrder()); + for (UnitPreference pref : entry3.getValue()) { + final String topUnit = + UnitPreferences.SPLIT_AND.split(pref.unit).iterator().next(); + if (first) { samples.add( converter.convert( - pref.geq.subtract(ONE_TENTH), - topUnit, - baseUnit, - false)); - } - // show samples - Set regions = entry3.getKey(); - String sampleRegion = regions.iterator().next(); - Collection uprefs = entry3.getValue(); - for (Rational sample : samples) { - showSample( - quantity, - usage, - sampleRegion, - sample, - baseUnit, - uprefs, - pw); + pref.geq.add(ONE_TENTH), topUnit, baseUnit, false)); + first = false; } - pw.println(); + samples.add(converter.convert(pref.geq, topUnit, baseUnit, false)); + samples.add( + converter.convert( + pref.geq.subtract(ONE_TENTH), + topUnit, + baseUnit, + false)); } + // show samples + Set regions = entry3.getKey(); + String sampleRegion = regions.iterator().next(); + Collection uprefs = entry3.getValue(); + for (Rational sample : samples) { + showSample(quantity, usage, sampleRegion, sample, baseUnit, uprefs, pw); + } + pw.println(); } } } } } + public void generateUnitLocalePreferences() { + try (TempPrintWriter pwLocale = + TempPrintWriter.openUTF8Writer( + CLDRPaths.TEST_DATA + "units", "unitLocalePreferencesTest.txt")) { + + try { + Set> seen = new HashSet<>(); + // first copy existing lines + // This includes the header, so modify the old header if changes are needed! + Files.lines(Path.of(CLDRPaths.TEST_DATA + "units/unitLocalePreferencesTest.txt")) + .forEach(line -> formatPwLocale(pwLocale, line, seen)); + // TODO: add more lines + formatLocaleLine( + "byte-per-millisecond", Rational.of(123), "default", "en", "", seen); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + static final Splitter SPLIT_SEMI = Splitter.on(Pattern.compile("\\s*;\\s*")).trimResults(); + + private void formatPwLocale(TempPrintWriter pwLocale, String rawLine, Set> seen) { + int hashPos = rawLine.indexOf('#'); + String line = hashPos < 0 ? rawLine : rawLine.substring(0, hashPos); + String comment = hashPos < 0 ? "" : "#" + rawLine.substring(hashPos + 1); + if (line.isBlank()) { + if (!comment.isBlank()) { + pwLocale.println(comment); + } + return; + } + List parts = SPLIT_SEMI.splitToList(line); + + String sourceUnit = parts.get(0); + Rational sourceAmount = Rational.of(parts.get(1)); + String usage = parts.get(2); + String languageTag = parts.get(3); + String newLine = + formatLocaleLine(sourceUnit, sourceAmount, usage, languageTag, comment, seen); + if (newLine != null) { + pwLocale.println(newLine); + } + } + + public String formatLocaleLine( + String sourceUnit, + Rational sourceAmount, + String usage, + String languageTag, + String comment, + Set> seen) { + List bundle = List.of(sourceUnit, sourceAmount, usage, languageTag); + if (bundle.contains(seen)) { + return null; + } + seen.add(bundle); + + UnitPreferences prefs = SDI.getUnitPreferences(); + final ULocale uLocale = ULocale.forLanguageTag(languageTag); + UnitPreference unitPreference = + prefs.getUnitPreference(sourceAmount, sourceUnit, usage, uLocale); + if (unitPreference == null) { // if the quantity isn't found + throw new IllegalArgumentException( + String.format( + "No unit preferences found for unit: %s, usage: %s, locale:%s", + sourceUnit, usage, languageTag)); + } + String actualUnit = unitPreference.unit; + Rational actualValue = + converter.convert(sourceAmount, sourceUnit, unitPreference.unit, false); + // # input-unit; amount; usage; languageTag; expected-unit; expected-amount # comment + final String newFileLine = + String.format( + "%s;\t%s;\t%s;\t%s;\t%s;\t%s%s", + sourceUnit, + sourceAmount.toString(FormatStyle.formatted), + usage, + languageTag, + actualUnit, + actualValue.toString(FormatStyle.formatted), + comment.isBlank() ? "" : "\t" + comment); + return newFileLine; + } + + static LikelySubtags likely = new LikelySubtags(); + + public String getHeader(String regionOrLocale) { + return "\n# Test data for unit region preferences\n" + + CldrUtility.getCopyrightString("# ") + + "\n" + + "#\n" + + "# Format:\n" + + "#\tQuantity;\tUsage;\t" + + regionOrLocale + + ";\tInput (r);\tInput (d);\tInput Unit;\tOutput (r);\tOutput (d);\tOutput Unit\n" + + "#\n" + + "# Use: Convert the Input amount & unit according to the Usage and " + + regionOrLocale + + ".\n" + + "#\t The result should match the Output amount and unit.\n" + + "#\t Both rational (r) and double64 (d) forms of the input and output amounts are supplied so that implementations\n" + + "#\t have two options for testing based on the precision in their implementations. For example:\n" + + "#\t 3429 / 12500; 0.27432; meter;\n" + + "#\t The Output amount and Unit are repeated for mixed units. In such a case, only the smallest unit will have\n" + + "#\t both a rational and decimal amount; the others will have a single integer value, such as:\n" + + "#\t length; person-height; CA; 3429 / 12500; 0.27432; meter; 2; foot; 54 / 5; 10.8; inch\n" + + "#\t The input and output units are unit identifers; in particular, the output does not have further processing:\n" + + "#\t\t • no localization\n" + + "#\t\t • no adjustment for pluralization\n" + + "#\t\t • no formatted with the skeleton\n" + + "#\t\t • no suppression of zero values (for secondary -and- units such as pound in stone-and-pound)\n" + + "#\n" + + "# Generation: Use GenerateUnitTestData.java to regenerate unitPreferencesTest.txt.\n"; + } + private void showSample( String quantity, String usage, - String sampleRegion, + String sampleRegionOrLocale, Rational sampleBaseValue, String baseUnit, Collection prefs, @@ -233,21 +321,28 @@ private void showSample( Rational baseGeq = converter.convert(pref.geq, topUnit, baseUnit, false); if (sampleBaseValue.compareTo(baseGeq) >= 0) { showSample2( - quantity, usage, sampleRegion, sampleBaseValue, baseUnit, pref.unit, pw); + quantity, + usage, + sampleRegionOrLocale, + sampleBaseValue, + baseUnit, + pref.unit, + pw); gotOne = true; break; } lastUnit = pref.unit; } if (!gotOne) { - showSample2(quantity, usage, sampleRegion, sampleBaseValue, baseUnit, lastUnit, pw); + showSample2( + quantity, usage, sampleRegionOrLocale, sampleBaseValue, baseUnit, lastUnit, pw); } } private void showSample2( String quantity, String usage, - String sampleRegion, + String sampleRegionOrLocale, Rational sampleBaseValue, String baseUnit, String lastUnit, @@ -279,7 +374,7 @@ private void showSample2( + TEST_SEP + usage + TEST_SEP - + sampleRegion + + sampleRegionOrLocale + TEST_SEP + originalSampleBaseValue + TEST_SEP diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/GrammarInfo.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/GrammarInfo.java index a3373aa9d74..bf64edd6b58 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/GrammarInfo.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/GrammarInfo.java @@ -755,7 +755,11 @@ public static Set getGrammarLocales() { "knot", "dalton", "kilocalorie", - "electronvolt"); + "electronvolt", + // The following may be reinstated after 45. + "dot-per-centimeter", + "millimeter-ofhg", + "milligram-ofglucose-per-deciliter"); public static Set getSpecialsToTranslate() { return INCLUDE_OTHER; diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/Rational.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/Rational.java index 98c3521689b..c5e74bfa6cc 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/Rational.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/Rational.java @@ -32,11 +32,20 @@ * * @author markdavis */ -public final class Rational implements Comparable { +public final class Rational extends Number implements Comparable { + private static final long serialVersionUID = 1L; private static final Pattern INT_POWER_10 = Pattern.compile("10*"); public final BigInteger numerator; public final BigInteger denominator; + static final BigInteger BI_TWO = BigInteger.valueOf(2); + static final BigInteger BI_FIVE = BigInteger.valueOf(5); + static final BigInteger BI_MINUS_ONE = BigInteger.valueOf(-1); + static final BigInteger BI_TEN = BigInteger.valueOf(10); + + static final BigDecimal BD_TWO = BigDecimal.valueOf(2); + static final BigDecimal BD_FIVE = BigDecimal.valueOf(5); + // Constraints: // always stored in normalized form. // no common factor > 1 (reduced) @@ -44,16 +53,18 @@ public final class Rational implements Comparable { // if numerator is zero, denominator is 1 or 0 // if denominator is zero, numerator is 1, -1, or 0 - public static final Rational ZERO = Rational.of(0); - public static final Rational ONE = Rational.of(1); - public static final Rational TWO = Rational.of(2); - public static final Rational NEGATIVE_ONE = ONE.negate(); + // NOTE, the constructor doesn't do any checking, so everything other than these goes + // through Rational.of(...) + public static final Rational ZERO = new Rational(BigInteger.ZERO, BigInteger.ONE); + public static final Rational ONE = new Rational(BigInteger.ONE, BigInteger.ONE); + public static final Rational NaN = new Rational(BigInteger.ZERO, BigInteger.ZERO); + public static final Rational INFINITY = new Rational(BigInteger.ONE, BigInteger.ZERO); - public static final Rational INFINITY = Rational.of(1, 0); + public static final Rational NEGATIVE_ONE = ONE.negate(); public static final Rational NEGATIVE_INFINITY = INFINITY.negate(); - public static final Rational NaN = Rational.of(0, 0); - public static final Rational TEN = Rational.of(10, 1); + public static final Rational TWO = new Rational(BI_TWO, BigInteger.ONE); + public static final Rational TEN = new Rational(BI_TEN, BigInteger.ONE); public static final Rational TENTH = TEN.reciprocal(); public static final char REPTEND_MARKER = '˙'; @@ -202,18 +213,41 @@ public Map getConstants() { } public static Rational of(long numerator, long denominator) { - return new Rational(BigInteger.valueOf(numerator), BigInteger.valueOf(denominator)); + return Rational.of(BigInteger.valueOf(numerator), BigInteger.valueOf(denominator)); } public static Rational of(long numerator) { - return new Rational(BigInteger.valueOf(numerator), BigInteger.ONE); + return Rational.of(BigInteger.valueOf(numerator), BigInteger.ONE); } public static Rational of(BigInteger numerator, BigInteger denominator) { - return new Rational(numerator, denominator); + int dComparison = denominator.compareTo(BigInteger.ZERO); + if (dComparison == 0) { + // catch equivalents to NaN, -INF, +INF + // 0/0 => NaN + // +/0 => INF + // -/0 => -INF + int nComparison = numerator.compareTo(BigInteger.ZERO); + return nComparison < 0 ? NEGATIVE_INFINITY : nComparison > 0 ? INFINITY : NaN; + } else { + // reduce to lowest form + BigInteger gcd = numerator.gcd(denominator); + if (gcd.compareTo(BigInteger.ONE) > 0) { + numerator = numerator.divide(gcd); + denominator = denominator.divide(gcd); + } + if (dComparison < 0) { + // ** NOTE: is already reduced, so safe to use constructor + return new Rational(numerator, denominator); + } else { + // ** NOTE: is already reduced, so safe to use constructor + return new Rational(numerator.negate(), denominator.negate()); + } + } } public static Rational of(BigInteger numerator) { + // ** NOTE: is already reduced, so safe to use constructor return new Rational(numerator, BigInteger.ONE); } @@ -236,56 +270,47 @@ private Rational(BigInteger numerator, BigInteger denominator) { } public Rational add(Rational other) { - BigInteger gcd_den = denominator.gcd(other.denominator); - return new Rational( - numerator - .multiply(other.denominator) - .divide(gcd_den) - .add(other.numerator.multiply(denominator).divide(gcd_den)), - denominator.multiply(other.denominator).divide(gcd_den)); + BigInteger newNumerator = + numerator.multiply(other.denominator).add(other.numerator.multiply(denominator)); + BigInteger newDenominator = denominator.multiply(other.denominator); + return Rational.of(newNumerator, newDenominator); } public Rational subtract(Rational other) { - BigInteger gcd_den = denominator.gcd(other.denominator); - return new Rational( + BigInteger newNumerator = numerator .multiply(other.denominator) - .divide(gcd_den) - .subtract(other.numerator.multiply(denominator).divide(gcd_den)), - denominator.multiply(other.denominator).divide(gcd_den)); + .subtract(other.numerator.multiply(denominator)); + BigInteger newDenominator = denominator.multiply(other.denominator); + return Rational.of(newNumerator, newDenominator); } public Rational multiply(Rational other) { - BigInteger gcd_num_oden = numerator.gcd(other.denominator); - boolean isZero = gcd_num_oden.equals(BigInteger.ZERO); - BigInteger smallNum = isZero ? numerator : numerator.divide(gcd_num_oden); - BigInteger smallODen = isZero ? other.denominator : other.denominator.divide(gcd_num_oden); - - BigInteger gcd_den_onum = denominator.gcd(other.numerator); - isZero = gcd_den_onum.equals(BigInteger.ZERO); - BigInteger smallONum = isZero ? other.numerator : other.numerator.divide(gcd_den_onum); - BigInteger smallDen = isZero ? denominator : denominator.divide(gcd_den_onum); + BigInteger newNumerator = numerator.multiply(other.numerator); + BigInteger newDenominator = denominator.multiply(other.denominator); + return Rational.of(newNumerator, newDenominator); + } - return new Rational(smallNum.multiply(smallONum), smallDen.multiply(smallODen)); + public Rational divide(Rational other) { + BigInteger newNumerator = numerator.multiply(other.denominator); + BigInteger newDenominator = denominator.multiply(other.numerator); + return Rational.of(newNumerator, newDenominator); } public Rational pow(int i) { - return new Rational(numerator.pow(i), denominator.pow(i)); + return Rational.of(numerator.pow(i), denominator.pow(i)); } public static Rational pow10(int i) { return i > 0 ? TEN.pow(i) : TENTH.pow(-i); } - public Rational divide(Rational other) { - return multiply(other.reciprocal()); - } - public Rational reciprocal() { - return new Rational(denominator, numerator); + return Rational.of(denominator, numerator); } public Rational negate() { + // ** NOTE: is already reduced, so safe to use constructor return new Rational(numerator.negate(), denominator); } @@ -298,17 +323,24 @@ public BigDecimal toBigDecimal(MathContext mathContext) { } } + public BigDecimal toBigDecimal() { + return toBigDecimal(MathContext.DECIMAL128); // prevent failures due to repeating fractions + } + + @Override public double doubleValue() { if (denominator.equals(BigInteger.ZERO) && numerator.equals(BigInteger.ZERO)) { return Double.NaN; } - return new BigDecimal(numerator) - .divide(new BigDecimal(denominator), MathContext.DECIMAL64) - .doubleValue(); + return toBigDecimal(MathContext.DECIMAL64).doubleValue(); } - public BigDecimal toBigDecimal() { - return toBigDecimal(MathContext.UNLIMITED); + @Override + public float floatValue() { + if (denominator.equals(BigInteger.ZERO) && numerator.equals(BigInteger.ZERO)) { + return Float.NaN; + } + return toBigDecimal(MathContext.DECIMAL32).floatValue(); } public static Rational of(BigDecimal bigDecimal) { @@ -320,16 +352,22 @@ public static Rational of(BigDecimal bigDecimal) { final int scale = bigDecimal.scale(); final BigInteger unscaled = bigDecimal.unscaledValue(); if (scale == 0) { + // ** NOTE: is already reduced, so safe to use constructor return new Rational(unscaled, BigInteger.ONE); } else if (scale >= 0) { - return new Rational(unscaled, BigDecimal.ONE.movePointRight(scale).toBigInteger()); + return Rational.of(unscaled, BigDecimal.ONE.movePointRight(scale).toBigInteger()); } else { + // ** NOTE: is already reduced, so safe to use constructor return new Rational( unscaled.multiply(BigDecimal.ONE.movePointLeft(scale).toBigInteger()), BigInteger.ONE); } } + public static Rational of(double doubleValue) { + return of(new BigDecimal(doubleValue)); + } + public enum FormatStyle { /** * Simple numerator / denominator, plain BigInteger.toString(), dropping " / 1".
@@ -496,13 +534,6 @@ public Rational abs() { return numerator.signum() >= 0 ? this : this.negate(); } - static final BigInteger BI_TWO = BigInteger.valueOf(2); - static final BigInteger BI_FIVE = BigInteger.valueOf(5); - static final BigInteger BI_MINUS_ONE = BigInteger.valueOf(-1); - - static final BigDecimal BD_TWO = BigDecimal.valueOf(2); - static final BigDecimal BD_FIVE = BigDecimal.valueOf(5); - /** * Goal is to be able to display rationals in a short but exact form, like 1,234,567/3 or * 1.234567E21/3. To do this, find the smallest denominator (excluding powers of 2 and 5), and @@ -728,4 +759,18 @@ public boolean approximatelyEquals(Rational b, Rational epsilon) { public boolean approximatelyEquals(Rational b) { return approximatelyEquals(b, EPSILON); } + + public boolean approximatelyEquals(Number b) { + return approximatelyEquals(Rational.of(b.doubleValue()), EPSILON); + } + + @Override + public int intValue() { + return toBigDecimal().intValue(); + } + + @Override + public long longValue() { + return toBigDecimal().longValue(); + } } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java index 00b139335a5..3f29adae2b1 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java @@ -15,6 +15,7 @@ import com.google.common.collect.TreeMultimap; import com.ibm.icu.impl.Row.R2; import com.ibm.icu.lang.UCharacter; +import com.ibm.icu.number.UnlocalizedNumberFormatter; import com.ibm.icu.text.PluralRules; import com.ibm.icu.util.Freezable; import com.ibm.icu.util.Output; @@ -1828,19 +1829,27 @@ public Set getSystemsEnum(String unit) { UnitId id = createUnitId(unit); // we walk through all the units in the numerator and denominator, and keep the - // *intersection* of - // the units. So {ussystem} and {ussystem, uksystem} => ussystem - // Special case: {dmetric} intersect {metric} => {dmetric}. We do that by adding dmetric to - // any set with metric, then removing dmetric if there is a metric + // *intersection* of the units. + // So {ussystem} and {ussystem, uksystem} => ussystem + // Special case: {metric_adjacent} intersect {metric} => {metric_adjacent}. + // We do that by adding metric_adjacent to any set with metric, + // then removing metric_adjacent if there is a metric. + // Same for si_acceptable. main: for (Map unitsToPowers : Arrays.asList(id.denUnitsToPowers, id.numUnitsToPowers)) { for (String subunit : unitsToPowers.keySet()) { subunit = UnitConverter.stripPrefix(subunit, null); Set systems = new TreeSet<>(sourceToSystems.get(subunit)); + if (systems.contains(UnitSystem.metric)) { + systems.add(UnitSystem.metric_adjacent); + } + if (systems.contains(UnitSystem.si)) { + systems.add(UnitSystem.si_acceptable); + } if (result == null) { - result = systems; + result = systems; // first setting } else { result.retainAll(systems); } @@ -1849,9 +1858,17 @@ public Set getSystemsEnum(String unit) { } } } - return result == null || result.isEmpty() - ? ImmutableSet.of(UnitSystem.other) - : ImmutableSet.copyOf(EnumSet.copyOf(result)); + if (result == null || result.isEmpty()) { + return ImmutableSet.of(UnitSystem.other); + } + if (result.contains(UnitSystem.metric)) { + result.remove(UnitSystem.metric_adjacent); + } + if (result.contains(UnitSystem.si)) { + result.remove(UnitSystem.si_acceptable); + } + + return ImmutableSet.copyOf(EnumSet.copyOf(result)); // the enum is to sort } // private void addSystems(Set result, String subunit) { @@ -2240,4 +2257,30 @@ public String resolve(String unit) { String resolved = unitId.resolve().toString(); return getStandardUnit(resolved.isBlank() ? unit : resolved); } + + public String format( + final String languageTag, + Rational outputAmount, + final String unit, + UnlocalizedNumberFormatter nf3) { + final CLDRConfig config = CLDRConfig.getInstance(); + Factory factory = config.getCldrFactory(); + int pos = languageTag.indexOf("-u"); + String localeBase = + (pos < 0 ? languageTag : languageTag.substring(0, pos)).replace('-', '_'); + CLDRFile localeFile = factory.make(localeBase, true); + PluralRules pluralRules = + config.getSupplementalDataInfo() + .getPluralRules( + localeBase, com.ibm.icu.text.PluralRules.PluralType.CARDINAL); + String pluralCategory = pluralRules.select(outputAmount.doubleValue()); + String path = + UnitPathType.unit.getTranslationPath( + localeFile, "long", unit, pluralCategory, "nominative", "neuter"); + String pattern = localeFile.getStringValue(path); + final ULocale uLocale = ULocale.forLanguageTag(languageTag); + String cldrFormattedNumber = + nf3.locale(uLocale).format(outputAmount.doubleValue()).toString(); + return com.ibm.icu.text.MessageFormat.format(pattern, cldrFormattedNumber); + } } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPreferences.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPreferences.java index 7eac1dd6270..5d856e35325 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPreferences.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPreferences.java @@ -2,19 +2,25 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableSet; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; -import com.ibm.icu.impl.locale.XCldrStub.ImmutableMap; import com.ibm.icu.util.Freezable; +import com.ibm.icu.util.Output; +import com.ibm.icu.util.ULocale; import java.util.Collection; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import org.unicode.cldr.tool.LikelySubtags; +import org.unicode.cldr.util.UnitConverter.ConversionInfo; public class UnitPreferences implements Freezable { Map, UnitPreference>>> quantityToUsageToRegionsToInfo = @@ -93,8 +99,9 @@ public void add( Rational newGeq = geq == null || geq.isEmpty() ? Rational.ONE : Rational.of(geq); final UnitPreference newUnitPref = new UnitPreference(newGeq, unit, skeleton); - regionsToInfo.put( - ImmutableSet.copyOf(new TreeSet<>(SPLIT_SPACE.splitToList(regions))), newUnitPref); + final ImmutableSet regionSet = + ImmutableSet.copyOf(new TreeSet<>(SPLIT_SPACE.splitToList(regions))); + boolean old = regionsToInfo.put(regionSet, newUnitPref); } boolean frozen; @@ -195,33 +202,41 @@ public String getPath( * * @return */ - public Map>> getFastMap( - UnitConverter converter) { - Map>> result = new LinkedHashMap<>(); + private Map>> getRawFastMap() { + UnitConverter converter = SupplementalDataInfo.getInstance().getUnitConverter(); + Map>> result = new LinkedHashMap<>(); for (Entry, UnitPreference>>> entry1 : quantityToUsageToRegionsToInfo.entrySet()) { String quantity = entry1.getKey(); - Map> result2 = new LinkedHashMap<>(); + Map> result2 = new LinkedHashMap<>(); result.put(quantity, result2); for (Entry, UnitPreference>> entry2 : entry1.getValue().entrySet()) { String usage = entry2.getKey(); - Map result3 = new LinkedHashMap<>(); + Multimap result3 = LinkedHashMultimap.create(); result2.put(usage, result3); + + // split the regions for (Entry, Collection> entry : entry2.getValue().asMap().entrySet()) { Set regions = entry.getKey(); + int len = entry.getValue().size(); for (UnitPreference up : entry.getValue()) { String unit = SPLIT_AND.split(up.unit).iterator().next(); // first unit quantity = converter.getQuantityFromUnit(unit, false); String baseUnit = converter.getBaseUnitFromQuantity(quantity); - Rational geq = converter.parseRational(String.valueOf(up.geq)); - Rational value = converter.convert(geq, unit, baseUnit, false); - if (value.equals(Rational.NaN)) { - converter.convert(geq, unit, baseUnit, true); // debug + Rational baseGeq; + if (--len == 0) { // set last value to least possible + baseGeq = Rational.NEGATIVE_INFINITY; + } else { + Rational geq = converter.parseRational(String.valueOf(up.geq)); + baseGeq = converter.convert(geq, unit, baseUnit, false); + if (baseGeq.equals(Rational.NaN)) { + converter.convert(geq, unit, baseUnit, true); // debug + } } - UnitPreference up2 = new UnitPreference(value, up.unit, up.skeleton); + UnitPreference up2 = new UnitPreference(baseGeq, up.unit, up.skeleton); for (String region : regions) { result3.put(region, up2); } @@ -229,10 +244,123 @@ public Map>> getFastMap( } } } - return ImmutableMap.copyOf(result); + return CldrUtility.protectCollection(result); + } + + Supplier>>> + quantityToUsageToRegionToInfo = Suppliers.memoize(() -> getRawFastMap()); + + public Map>> getFastMap() { + return quantityToUsageToRegionToInfo.get(); + } + + public UnitPreference getUnitPreference( + Rational sourceAmount, String sourceUnit, String usage, ULocale locale) { + UnitConverter converter = SupplementalDataInfo.getInstance().getUnitConverter(); + sourceUnit = converter.fixDenormalized(sourceUnit); + + String mu = locale.getUnicodeLocaleType("mu"); + // TODO if the value is not a unit, skip + if (mu != null) { + Rational conversion = converter.convert(sourceAmount, sourceUnit, mu, false); + if (!conversion.equals(Rational.NaN)) { // if we could successfully convert + return new UnitPreference(conversion, mu, null); + } + } + String region = resolveRegion(locale); + + return getUnitPreference(sourceAmount, sourceUnit, usage, region); + } + + public UnitPreference getUnitPreference( + Rational sourceAmount, String sourceUnit, String usage, String region) { + UnitConverter converter = SupplementalDataInfo.getInstance().getUnitConverter(); + String quantity = converter.getQuantityFromUnit(sourceUnit, false); + + Map> usageToRegionsToInfo = + getFastMap().get(quantity); + + // If there is no quantity among the preferences, + // return the metric UnitPreference + if (usageToRegionsToInfo == null) { + String standardUnit = converter.getStandardUnit(sourceUnit); + if (!sourceUnit.equals(standardUnit)) { + Rational conversion = + converter.convert(sourceAmount, sourceUnit, standardUnit, false); + return new UnitPreference(conversion, standardUnit, null); + } + return new UnitPreference(sourceAmount, sourceUnit, null); + } + + Multimap regionToInfo = usageToRegionsToInfo.get(usage); + + if (regionToInfo == null) { + regionToInfo = usageToRegionsToInfo.get("default"); + } + + // normalize for matching + sourceAmount = sourceAmount.abs(); + if (sourceAmount.equals(Rational.NaN)) { + sourceAmount = Rational.NEGATIVE_ONE; + } + + Collection infoList = regionToInfo.get(region); + if (infoList == null || infoList.isEmpty()) { + infoList = regionToInfo.get("001"); + } + + Output baseUnitOutput = new Output<>(); + ConversionInfo sourceConversionInfo = + converter.parseUnitId(sourceUnit, baseUnitOutput, false); + Rational baseValue = sourceConversionInfo.convert(sourceAmount); + + for (UnitPreference info : infoList) { // data is built to always terminate + if (baseValue.compareTo(info.geq) >= 0) { + return info; + } + } + throw new IllegalArgumentException("Fast map should always terminate"); + } + + public String resolveRegion(ULocale locale) { + // https://unicode.org/reports/tr35/tr35-info.html#Unit_Preferences + // en-u-rg-uszzzz-ms-ussystem + String ms = locale.getUnicodeLocaleType("ms"); + if (ms != null) { + switch (ms) { + case "metric": + return "001"; + case "uksystem": + return "GB"; + case "ussystem": + return "US"; + default: + throw new IllegalArgumentException( + "Illegal ms value in: " + locale.toLanguageTag()); + } + } + String rg = locale.getUnicodeLocaleType("rg"); + if (rg != null) { + // TODO: check for illegal rg value + return rg.substring(0, 2).toUpperCase(Locale.ROOT); + } + String region = locale.getCountry(); + if (!region.isEmpty()) { + return region; + } + LikelySubtags LIKELY = new LikelySubtags(); + String maximized = LIKELY.maximize(locale.toLanguageTag()); + if (maximized != null) { + return ULocale.getCountry(maximized); + } + return "001"; } public Set getUsages() { return usages; } + + public Set getQuantities() { + return getFastMap().keySet(); + } } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/Units.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/Units.java index e8ef091dcda..68e4e9f9825 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/Units.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/Units.java @@ -109,10 +109,12 @@ public static TypeAndCore splitUnit(String longOrShortUnit, TypeAndCore typeAndC } public static String getShort(String longUnit) { - return LONG_TO_SHORT.get(longUnit); + String result = LONG_TO_SHORT.get(longUnit); + return result == null ? longUnit : result; } public static String getLong(String shortId) { - return LONG_TO_SHORT.inverse().get(shortId); + String result = LONG_TO_SHORT.inverse().get(shortId); + return result == null ? shortId : result; } } diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java index 4e306292aa4..f93b83d70bb 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java @@ -22,18 +22,29 @@ import com.ibm.icu.impl.Row; import com.ibm.icu.impl.Row.R2; import com.ibm.icu.impl.Row.R3; +import com.ibm.icu.number.FormattedNumber; +import com.ibm.icu.number.LocalizedNumberFormatter; +import com.ibm.icu.number.NumberFormatter; +import com.ibm.icu.number.NumberFormatter.UnitWidth; +import com.ibm.icu.number.Precision; +import com.ibm.icu.number.UnlocalizedNumberFormatter; import com.ibm.icu.text.PluralRules; import com.ibm.icu.text.UnicodeSet; import com.ibm.icu.util.ICUUncheckedIOException; +import com.ibm.icu.util.Measure; +import com.ibm.icu.util.MeasureUnit; import com.ibm.icu.util.Output; +import com.ibm.icu.util.ULocale; import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.io.UncheckedIOException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.MathContext; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -43,6 +54,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -114,6 +126,8 @@ import org.unicode.cldr.util.XPathParts; public class TestUnits extends TestFmwk { + private static final boolean DEBUG = System.getProperty("TestUnits:DEBUG") != null; + private static final boolean TEST_ICU = System.getProperty("TestUnits:TEST_ICU") != null; private static final Joiner JOIN_COMMA = Joiner.on(", "); @@ -137,7 +151,7 @@ public class TestUnits extends TestFmwk { Validity.getInstance().getStatusToCodes(LstrType.unit).get(Validity.Status.regular); private static final Set DEPRECATED_REGULAR_UNITS = Validity.getInstance().getStatusToCodes(LstrType.unit).get(Validity.Status.deprecated); - private static final CLDRConfig CLDR_CONFIG = CLDRConfig.getInstance(); + public static final CLDRConfig CLDR_CONFIG = CLDRConfig.getInstance(); private static final Integer INTEGER_ONE = 1; public static boolean getFlag(String flag) { @@ -479,8 +493,6 @@ private boolean assertEquals2( return true; } - static final boolean DEBUG = false; - public void TestConversion() { String[][] tests = { {"foot", "12", "inch"}, @@ -736,12 +748,8 @@ public void TestRational() { assertEquals("", Rational.ONE, a3_5.multiply(a3_5.reciprocal())); assertEquals("", Rational.ZERO, a3_5.add(a3_5.negate())); - assertEquals("", Rational.INFINITY, Rational.ZERO.reciprocal()); - assertEquals("", Rational.NEGATIVE_INFINITY, Rational.INFINITY.negate()); assertEquals("", Rational.NEGATIVE_ONE, Rational.ONE.negate()); - assertEquals("", Rational.NaN, Rational.ZERO.divide(Rational.ZERO)); - assertEquals("", BigDecimal.valueOf(2), Rational.of(2, 1).toBigDecimal()); assertEquals("", BigDecimal.valueOf(0.5), Rational.of(1, 2).toBigDecimal()); @@ -755,6 +763,38 @@ public void TestRational() { ConversionInfo uinfo = new ConversionInfo(Rational.of(2), Rational.of(3)); assertEquals("", Rational.of(3), uinfo.convert(Rational.ZERO)); assertEquals("", Rational.of(7), uinfo.convert(Rational.of(2))); + + assertEquals("", Rational.INFINITY, Rational.ZERO.reciprocal()); + assertEquals("", Rational.NEGATIVE_INFINITY, Rational.INFINITY.negate()); + + Set anything = + ImmutableSet.of( + Rational.NaN, + Rational.NEGATIVE_INFINITY, + Rational.NEGATIVE_ONE, + Rational.ZERO, + Rational.ONE, + Rational.INFINITY); + for (Rational something : anything) { + assertEquals("0/0", Rational.NaN, Rational.NaN.add(something)); + assertEquals("0/0", Rational.NaN, Rational.NaN.subtract(something)); + assertEquals("0/0", Rational.NaN, Rational.NaN.divide(something)); + assertEquals("0/0", Rational.NaN, Rational.NaN.add(something)); + assertEquals("0/0", Rational.NaN, Rational.NaN.negate()); + + assertEquals("0/0", Rational.NaN, something.add(Rational.NaN)); + assertEquals("0/0", Rational.NaN, something.subtract(Rational.NaN)); + assertEquals("0/0", Rational.NaN, something.divide(Rational.NaN)); + assertEquals("0/0", Rational.NaN, something.add(Rational.NaN)); + } + assertEquals("0/0", Rational.NaN, Rational.ZERO.divide(Rational.ZERO)); + assertEquals("INF-INF", Rational.NaN, Rational.INFINITY.subtract(Rational.INFINITY)); + assertEquals("INF+-INF", Rational.NaN, Rational.INFINITY.add(Rational.NEGATIVE_INFINITY)); + assertEquals("-INF+INF", Rational.NaN, Rational.NEGATIVE_INFINITY.add(Rational.INFINITY)); + assertEquals("INF/INF", Rational.NaN, Rational.INFINITY.divide(Rational.INFINITY)); + + assertEquals("INF+1", Rational.INFINITY, Rational.INFINITY.add(Rational.ONE)); + assertEquals("INF-1", Rational.INFINITY, Rational.INFINITY.subtract(Rational.ONE)); } public void TestRationalParse() { @@ -990,6 +1030,7 @@ public void TestConversionLineOrder() { // Test that sorted is in same order as the file. MapComparator conversionOrder = new MapComparator<>(data.keySet()); String lastUnit = null; + Set warnings = new LinkedHashSet<>(); for (Entry entry : sorted.entries()) { final TargetInfo tInfo = entry.getKey(); final String unit = entry.getValue(); @@ -1001,7 +1042,7 @@ public void TestConversionLineOrder() { ConversionInfo info = converter.parseUnitId(unit, metricUnit, false); String metric = metricUnit.value; if (metric.equals(lastMetric)) { - warnln( + warnings.add( "Expected " + lastUnit + " < " @@ -1028,6 +1069,9 @@ public void TestConversionLineOrder() { System.out.println(" " + tInfo.formatOriginalSource(entry.getValue())); } } + if (!warnings.isEmpty()) { + warnln("Some units are not ordered by size, count=" + warnings.size()); + } } public final void TestSimplify() { @@ -1795,25 +1839,6 @@ public void TestUnitPreferences() { "If this fails, check the output of TestUnitPreferencesSource (with -DTestUnits:SHOW_DATA), fix as needed, then incorporate."); UnitPreferences prefs = SDI.getUnitPreferences(); checkUnitPreferences(prefs); - // Map>> fastMap = - // prefs.getFastMap(converter); - // for (Entry>> entry : - // fastMap.entrySet()) { - // String quantity = entry.getKey(); - // String baseUnit = converter.getBaseUnitFromQuantity(quantity); - // for (Entry> entry2 : - // entry.getValue().entrySet()) { - // String usage = entry2.getKey(); - // for (Entry entry3 : entry2.getValue().entrySet()) - // { - // String region = entry3.getKey(); - // UnitPreference pref = entry3.getValue(); - // System.out.println(quantity + "\t" + usage + "\t" + region + "\t" + - // pref.toString(baseUnit)); - // } - // } - // } - prefs.getFastMap(converter); // call just to make sure we don't get an exception if (GENERATE_TESTS) { try (TempPrintWriter pw = @@ -4261,4 +4286,337 @@ public boolean assertNotContains( units + ": " + systemSet + " does not contain " + unitSystem, systemSet.contains(unitSystem)); } + + public void testQuantitiesMissingFromPreferences() { + UnitPreferences prefs = SDI.getUnitPreferences(); + Set preferenceQuantities = prefs.getQuantities(); + Set unitQuantities = converter.getQuantities(); + assertEquals( + "pref - unit quantities", + Collections.emptySet(), + Sets.difference(preferenceQuantities, unitQuantities)); + final SetView quantitiesNotInPreferences = + Sets.difference(unitQuantities, preferenceQuantities); + if (!quantitiesNotInPreferences.isEmpty()) { + warnln("unit - pref quantities = " + quantitiesNotInPreferences); + } + for (String unit : converter.getSimpleUnits()) { + String quantity = converter.getQuantityFromUnit(unit, false); + if (!quantitiesNotInPreferences.contains(quantity)) { + continue; + } + // we have a unit whose quantity is not in preferences + // get its unit preferences + UnitPreference pref = + prefs.getUnitPreference(Rational.ONE, unit, "default", ULocale.US); + if (pref == null) { + errln( + String.format( + "Default preference is null: input unit=%s, quantity=%s", + unit, quantity)); + continue; + } + // ensure that it is metric + Set inputSystems = converter.getSystemsEnum(unit); + if (Collections.disjoint(inputSystems, UnitSystem.SiOrMetric)) { + warnln( + String.format( + "There are no explicit preferences for %s, but %s is not metric", + quantity, unit)); + } + Set prefSystems = converter.getSystemsEnum(pref.unit); + + String errorOrWarningString = + String.format( + "Test default preference is metric: input unit=%s, quantity=%s, pref-unit=%s, systems: %s", + unit, quantity, pref.unit, prefSystems); + if (Collections.disjoint(prefSystems, UnitSystem.SiOrMetric)) { + errln(errorOrWarningString); + } else { + logln("OK " + errorOrWarningString); + } + } + } + + public void testUnitPreferencesTest() { + try { + final Set warnings = new LinkedHashSet<>(); + Files.lines(Path.of(CLDRPaths.TEST_DATA + "units/unitPreferencesTest.txt")) + .forEach(line -> checkUnitPreferencesTest(line, warnings)); + if (!warnings.isEmpty()) { + warnln("Mixed unit identifiers not yet checked, count=" + warnings.size()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void checkUnitPreferencesTest(String line, Set warnings) { + if (line.startsWith("#") || line.isBlank()) { + return; + } + // # Quantity; Usage; Region; Input (r); Input (d); Input Unit; Output (r); + // Output (d); Output Unit + // Example: + // area; default; 001; 1100000; 1100000.0; square-meter; + // 11/10; 1.1; square-kilometer + // duration; media; 001; 66; 66.0; second; 1; minute; 6; + // 6.0; second + try { + UnitPreferences prefs = SDI.getUnitPreferences(); + List parts = SPLIT_SEMI.splitToList(line); + Map highMixed_unit_identifiers = new LinkedHashMap<>(); + String quantity = parts.get(0); + String usage = parts.get(1); + String region = parts.get(2); + Rational inputRational = Rational.of(parts.get(3)); + double inputDouble = Double.parseDouble(parts.get(4)); + String inputUnit = parts.get(5); + // account for multi-part output + int size = parts.size(); + // This section has larger elements with integer values + for (int i = 6; i < size - 3; i += 2) { + highMixed_unit_identifiers.put(parts.get(i + 1), Long.parseLong(parts.get(i))); + } + Rational expectedValue = Rational.of(parts.get(size - 3)); + Double expectedValueDouble = Double.parseDouble(parts.get(size - 2)); + String expectedOutputUnit = parts.get(size - 1); + + // Check that the double values are approximately the same as + // the Rational ones + assertTrue( + String.format( + "input rational ~ input double, %s %s", inputRational, inputDouble), + inputRational.approximatelyEquals(inputDouble)); + assertTrue( + String.format( + "output rational ~ output double, %s %s", + expectedValue, expectedValueDouble), + expectedValue.approximatelyEquals(expectedValueDouble)); + + // check that the quantity is consistent + String expectedQuantity = converter.getQuantityFromUnit(inputUnit, false); + assertEquals("Input: Quantity consistency check", expectedQuantity, quantity); + + // TODO handle mixed_unit_identifiers + if (!highMixed_unit_identifiers.isEmpty()) { + warnings.add("mixed_unit_identifiers not yet checked: " + line); + return; + } + // check output unit, then value + UnitPreference unitPreference = + prefs.getUnitPreference(inputRational, inputUnit, usage, region); + String actualUnit = unitPreference.unit; + assertEquals("Output unit", expectedOutputUnit, actualUnit); + + Rational actualValue = converter.convert(inputRational, inputUnit, actualUnit, false); + assertEquals("Output numeric value", expectedValue, actualValue); + } catch (Exception e) { + errln(e.getMessage() + "\n\t" + line); + } + } + + public void testUnitsTest() { + try { + Files.lines(Path.of(CLDRPaths.TEST_DATA + "units/unitsTest.txt")) + .forEach(line -> checkUnitsTest(line)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void checkUnitsTest(String line) { + if (line.startsWith("#") || line.isBlank()) { + return; + } + // Quantity ; x ; y ; conversion to y (rational) ; test: 1000 x ⟹ y + // + // Use: convert 1000 x units to the y unit; the result should match the final column, + // at the given precision. For example, when the last column is 159.1549, + // round to 4 decimal digits before comparing. + // Example: + // acceleration ; g-force ; meter-per-square-second ; 9.80665 * x ; 9806.65 + try { + UnitPreferences prefs = SDI.getUnitPreferences(); + List parts = SPLIT_SEMI.splitToList(line); + String quantity = parts.get(0); + String sourceUnit = parts.get(1); + String targetUnit = parts.get(2); + String conversion = parts.get(3); + double expectedNumericValueFor1000 = Rational.of(parts.get(4)).doubleValue(); + + String expectedQuantity = converter.getQuantityFromUnit(sourceUnit, false); + assertEquals("Input: Quantity consistency check", expectedQuantity, quantity); + + // TODO check conversion equation (not particularly important + Rational actualValue = + converter.convert(Rational.of(1000), sourceUnit, targetUnit, false); + assertTrue( + String.format( + "output rational ~ expected double, %s %s", + expectedNumericValueFor1000, actualValue.doubleValue()), + actualValue.approximatelyEquals(expectedNumericValueFor1000)); + } catch (Exception e) { + errln(e.getMessage() + "\n\t" + line); + } + } + + public void testUnitLocalePreferencesTest() { + try { + Files.lines(Path.of(CLDRPaths.TEST_DATA + "units/unitLocalePreferencesTest.txt")) + .forEach(line -> checkUnitLocalePreferencesTest(line)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void checkUnitLocalePreferencesTest(String rawLine) { + int hashPos = rawLine.indexOf('#'); + String line = hashPos < 0 ? rawLine : rawLine.substring(0, hashPos); + String comment = hashPos < 0 ? "" : "\t# " + rawLine.substring(hashPos + 1); + if (line.isBlank()) { + return; + } + // # input-unit; amount; usage; languageTag; expected-unit; expected-amount # comment + // Example: + // fahrenheit; 1; default; en-u-rg-uszzzz-ms-ussystem-mu-celsius; celsius; -155/9 # + // mu > ms > rg > (likely) region + try { + UnitPreferences prefs = SDI.getUnitPreferences(); + List parts = SPLIT_SEMI.splitToList(line); + String sourceUnit = parts.get(0); + Rational sourceAmount = Rational.of(parts.get(1)); + String usage = parts.get(2); + String languageTag = parts.get(3); + String expectedUnit = parts.get(4); + Rational expectedAmount = Rational.of(parts.get(5)); + + String actualUnit; + Rational actualValue; + try { + if (DEBUG) + System.out.println( + String.format( + "%s;\t%s;\t%s;\t%s;\t%s;\t%s%s", + sourceUnit, + sourceAmount.toString(FormatStyle.formatted), + usage, + languageTag, + expectedUnit, + expectedAmount.toString(FormatStyle.formatted), + comment)); + + final ULocale uLocale = ULocale.forLanguageTag(languageTag); + UnitPreference unitPreference = + prefs.getUnitPreference(sourceAmount, sourceUnit, usage, uLocale); + if (unitPreference == null) { // if the quantity isn't found + throw new IllegalArgumentException( + String.format( + "No unit preferences found for unit: %s, usage: %s, locale:%s", + sourceUnit, usage, languageTag)); + } + actualUnit = unitPreference.unit; + actualValue = + converter.convert(sourceAmount, sourceUnit, unitPreference.unit, false); + } catch (Exception e1) { + actualUnit = e1.getMessage(); + actualValue = Rational.NaN; + } + if (assertEquals( + String.format( + "ICU unit pref, %s %s %s %s", + sourceUnit, + sourceAmount.toString(FormatStyle.formatted), + usage, + languageTag), + expectedUnit, + actualUnit)) { + assertEquals("CLDR value", expectedAmount, actualValue); + } else if (!comment.isBlank()) { + warnln(comment); + } + + } catch (Exception e) { + errln(e.getStackTrace()[0] + ", " + e.getMessage() + "\n\t" + rawLine); + } + } + + public void testUnitLocalePreferencesTestIcu() { + if (TEST_ICU) { + try { + Files.lines(Path.of(CLDRPaths.TEST_DATA + "units/unitLocalePreferencesTest.txt")) + .forEach(line -> checkUnitLocalePreferencesTestIcu(line)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + warnln("Skipping ICU test. To enable, set -DTestUnits:TEST_ICU"); + } + } + + private void checkUnitLocalePreferencesTestIcu(String rawLine) { + int hashPos = rawLine.indexOf('#'); + String line = hashPos < 0 ? rawLine : rawLine.substring(0, hashPos); + String comment = hashPos < 0 ? "" : "\t# " + rawLine.substring(hashPos + 1); + if (line.isBlank()) { + return; + } + // # input-unit; amount; usage; languageTag; expected-unit; expected-amount # comment + // Example: + // fahrenheit; 1; default; en-u-rg-uszzzz-ms-ussystem-mu-celsius; celsius; -155/9 # + // mu > ms > rg > (likely) region + try { + List parts = SPLIT_SEMI.splitToList(line); + String sourceUnit = parts.get(0); + double sourceAmount = icuRational(parts.get(1)); + String usage = parts.get(2); + String languageTag = parts.get(3); + String expectedUnit = parts.get(4); + double expectedAmount = icuRational(parts.get(5)); + + String actualUnit; + + float actualValueFloat; + try { + UnlocalizedNumberFormatter nf = + NumberFormatter.with() + .unitWidth(UnitWidth.FULL_NAME) + .precision(Precision.maxSignificantDigits(20)); + LocalizedNumberFormatter localized = + nf.usage(usage).locale(Locale.forLanguageTag(languageTag)); + final FormattedNumber formatted = + localized.format( + new Measure(sourceAmount, MeasureUnit.forIdentifier(sourceUnit))); + MeasureUnit icuOutputUnit = formatted.getOutputUnit(); + actualUnit = icuOutputUnit.getSubtype(); + actualValueFloat = formatted.toBigDecimal().floatValue(); + } catch (Exception e) { + actualUnit = e.getMessage(); + actualValueFloat = Float.NaN; + } + if (assertEquals( + String.format( + "ICU unit pref, %s %s %s %s", + sourceUnit, sourceAmount, usage, languageTag), + expectedUnit, + actualUnit)) { + assertEquals("ICU value", (float) expectedAmount, actualValueFloat); + } else if (!comment.isBlank()) { + warnln(comment); + } + } catch (Exception e) { + errln(e.getStackTrace()[0] + ", " + e.getMessage() + "\n\t" + rawLine); + } + } + + private double icuRational(String string) { + string = string.replace(",", ""); + int slashPos = string.indexOf('/'); + if (slashPos < 0) { + return Double.parseDouble(string); + } else { + return Double.parseDouble(string.substring(0, slashPos)) + / Double.parseDouble(string.substring(slashPos + 1)); + } + } }