From d12c1da138a544933a61741d164bc93619fe1ead Mon Sep 17 00:00:00 2001 From: macchiati Date: Tue, 7 May 2024 21:33:19 -0700 Subject: [PATCH] CLDR 16668 Rebase --- .../cldr/tool/ChartUnitConversions.java | 31 +-- .../java/org/unicode/cldr/util/DtdData.java | 259 +----------------- .../org/unicode/cldr/util/UnitConverter.java | 154 ++++++++--- .../org/unicode/cldr/util/UnitPathType.java | 9 + .../org/unicode/cldr/unittest/TestUnits.java | 33 ++- 5 files changed, 164 insertions(+), 322 deletions(-) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ChartUnitConversions.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ChartUnitConversions.java index 2466cef0ae7..fd044222d50 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ChartUnitConversions.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/ChartUnitConversions.java @@ -1,15 +1,14 @@ package org.unicode.cldr.tool; import com.google.common.base.Joiner; -import com.ibm.icu.impl.Row; import com.ibm.icu.impl.Row.R4; import java.io.IOException; import java.io.PrintWriter; -import java.util.EnumSet; import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; import org.unicode.cldr.util.CldrUtility; +import org.unicode.cldr.util.MapComparator; import org.unicode.cldr.util.Rational; import org.unicode.cldr.util.Rational.FormatStyle; import org.unicode.cldr.util.UnitConverter; @@ -111,35 +110,15 @@ public void writeContents(FormattedFileWriter pw) throws IOException { UnitConverter converter = SDI.getUnitConverter(); converter.getSourceToSystems(); - Set> all = new TreeSet<>(); + Set> all = new TreeSet<>(); + MapComparator shortIdUnicodeComparator = converter.getShortUnitIdComparator(); for (Entry entry : converter.getInternalConversionData().entrySet()) { String sourceUnit = entry.getKey(); String quantity = converter.getQuantityFromUnit(sourceUnit, false); TargetInfo targetInfo = entry.getValue(); - final EnumSet systems = - EnumSet.copyOf(converter.getSystemsEnum(sourceUnit)); - - // to sort the right items together items together, put together a sort key - UnitSystem sortingSystem = systems.iterator().next(); - switch (sortingSystem) { - case si: - case si_acceptable: - sortingSystem = UnitSystem.metric; - break; - case uksystem: - sortingSystem = UnitSystem.ussystem; - break; - default: - } - UnitId targetUnitId = converter.createUnitId(targetInfo.target); - R4 sortKey = - Row.of(targetUnitId, sortingSystem, targetInfo.unitInfo.factor, sourceUnit); - all.add(sortKey); - - // get some formatted strings - // TODO: handle specials here, CLDR-16329 additional PR or follow-on ticket + final Set systems = converter.getSystemsEnum(sourceUnit); final String repeatingFactor = targetInfo.unitInfo.factor.toString(FormatStyle.repeating); @@ -159,7 +138,7 @@ public void writeContents(FormattedFileWriter pw) throws IOException { tablePrinter .addRow() - .addCell(sortKey) + .addCell(shortIdUnicodeComparator.getOrdering(sourceUnit)) .addCell(quantity) .addCell(targetDisplay) .addCell(Joiner.on(", ").join(systems)) diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdData.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdData.java index 7e656dceb24..f3022b86044 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdData.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdData.java @@ -1617,263 +1617,16 @@ public AttributeStatus getAttributeStatus(String elementName, String attributeNa // in sync. public static MapComparator getUnitOrder() { - return UnitOrderHolder.INSTANCE; + return UnitOrderHolder.getInstance(); } private static final class UnitOrderHolder { private static final MapComparator INSTANCE = - // new MapComparator() - // - // .add(Validity.getInstance().getCodeToStatus(LstrType.unit).keySet()) - // .freeze(); - // } - new MapComparator<>( - Arrays.asList( - "acceleration-g-force", - "acceleration-meter-per-square-second", - "acceleration-meter-per-second-squared", // deprecated - "angle-revolution", - "angle-radian", - "angle-degree", - "angle-arc-minute", - "angle-arc-second", - "area-square-kilometer", - "area-hectare", - "area-square-meter", - "area-square-centimeter", - "area-square-mile", - "area-acre", - "area-square-yard", - "area-square-foot", - "area-square-inch", - "area-dunam", - "concentr-karat", - "proportion-karat", // deprecated - "concentr-milligram-ofglucose-per-deciliter", - "concentr-milligram-per-deciliter", - "concentr-millimole-per-liter", - "concentr-item", - "concentr-portion", - "concentr-permillion", - "concentr-part-per-million", // deprecated - "concentr-percent", - "concentr-permille", - "concentr-permyriad", - "concentr-mole", - "concentr-ofglucose", - "consumption-liter-per-kilometer", - "consumption-liter-per-100-kilometer", - "consumption-liter-per-100kilometers", // deprecated - "consumption-mile-per-gallon", - "consumption-mile-per-gallon-imperial", - "digital-petabyte", - "digital-terabyte", - "digital-terabit", - "digital-gigabyte", - "digital-gigabit", - "digital-megabyte", - "digital-megabit", - "digital-kilobyte", - "digital-kilobit", - "digital-byte", - "digital-bit", - "duration-century", - "duration-decade", - "duration-year", - "duration-year-person", - "duration-quarter", - "duration-month", - "duration-month-person", - "duration-week", - "duration-week-person", - "duration-day", - "duration-day-person", - "duration-hour", - "duration-minute", - "duration-second", - "duration-millisecond", - "duration-microsecond", - "duration-nanosecond", - "electric-ampere", - "electric-milliampere", - "electric-ohm", - "electric-volt", - "energy-kilocalorie", - "energy-calorie", - "energy-foodcalorie", - "energy-kilojoule", - "energy-joule", - "energy-kilowatt-hour", - "energy-electronvolt", - "energy-british-thermal-unit", - "energy-therm-us", - "force-pound-force", - "force-newton", - "force-kilowatt-hour-per-100-kilometer", - "frequency-gigahertz", - "frequency-megahertz", - "frequency-kilohertz", - "frequency-hertz", - "graphics-em", - "graphics-pixel", - "graphics-megapixel", - "graphics-pixel-per-centimeter", - "graphics-pixel-per-inch", - "graphics-dot-per-centimeter", - "graphics-dot-per-inch", - "graphics-dot", - "length-earth-radius", - "length-100-kilometer", - "length-kilometer", - "length-meter", - "length-decimeter", - "length-centimeter", - "length-millimeter", - "length-micrometer", - "length-nanometer", - "length-picometer", - "length-mile", - "length-yard", - "length-foot", - "length-inch", - "length-parsec", - "length-light-year", - "length-astronomical-unit", - "length-furlong", - "length-fathom", - "length-nautical-mile", - "length-mile-scandinavian", - "length-point", - "length-solar-radius", - "light-lux", - "light-candela", - "light-lumen", - "light-solar-luminosity", - "mass-tonne", - "mass-metric-ton", - "mass-kilogram", - "mass-gram", - "mass-milligram", - "mass-microgram", - "mass-ton", - "mass-stone", - "mass-pound", - "mass-ounce", - "mass-ounce-troy", - "mass-carat", - "mass-dalton", - "mass-earth-mass", - "mass-solar-mass", - "mass-grain", - "power-gigawatt", - "power-megawatt", - "power-kilowatt", - "power-watt", - "power-milliwatt", - "power-horsepower", - "pressure-millimeter-ofhg", - "pressure-millimeter-of-mercury", // deprecated - "pressure-ofhg", - "pressure-pound-force-per-square-inch", - "pressure-pound-per-square-inch", // deprecated - "pressure-inch-ofhg", - "pressure-inch-hg", // deprecated - "pressure-bar", - "pressure-millibar", - "pressure-atmosphere", - "pressure-pascal", - "pressure-hectopascal", - "pressure-kilopascal", - "pressure-megapascal", - "speed-kilometer-per-hour", - "speed-meter-per-second", - "speed-mile-per-hour", - "speed-knot", - "speed-beaufort", - "temperature-generic", - "temperature-celsius", - "temperature-fahrenheit", - "temperature-kelvin", - "torque-pound-force-foot", - "torque-pound-foot", // deprecated - "torque-newton-meter", - "volume-cubic-kilometer", - "volume-cubic-meter", - "volume-cubic-centimeter", - "volume-cubic-mile", - "volume-cubic-yard", - "volume-cubic-foot", - "volume-cubic-inch", - "volume-megaliter", - "volume-hectoliter", - "volume-liter", - "volume-deciliter", - "volume-centiliter", - "volume-milliliter", - "volume-pint-metric", - "volume-cup-metric", - "volume-acre-foot", - "volume-bushel", - "volume-gallon", - "volume-gallon-imperial", - "volume-quart", - "volume-pint", - "volume-pint-imperial", - "volume-cup", - "volume-fluid-ounce", - "volume-fluid-ounce-imperial", - "volume-tablespoon", - "volume-teaspoon", - "volume-barrel", - "volume-dessert-spoon", - "volume-dessert-spoon-imperial", - "volume-drop", - "volume-dram", - "volume-jigger", - "volume-pinch", - "volume-quart-imperial", - "angle-steradian", - "concentr-katal", - "electric-coulomb", - "electric-farad", - "electric-henry", - "electric-siemens", - "energy-calorie-it", - "energy-british-thermal-unit-it", - "energy-becquerel", - "energy-sievert", - "energy-gray", - "force-kilogram-force", - "length-rod", - "length-chain", - "magnetic-tesla", - "magnetic-weber", - "temperature-rankine", - "duration-fortnight", - "mass-slug", - "pressure-gasoline-energy-density", - "length-rin", - "length-sun", - "length-shaku-length", - "length-shaku-cloth", - "length-ken", - "length-jo-jp", - "length-ri-jp", - "area-bu-jp", - "area-se-jp", - "area-cho", - "volume-kosaji", - "volume-osaji", - "volume-cup-jp", - "volume-shaku", - "volume-sai", - "volume-to-jp", - "volume-koku", - "speed-light-speed", - "mass-fun", - "concentr-portion-per-1e9", - "duration-night")) - .freeze(); + SupplementalDataInfo.getInstance().getUnitConverter().getLongUnitIdComparator(); + + private static final MapComparator getInstance() { + return INSTANCE; + } } static MapComparator countValueOrder = diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java index 44620cfbb13..155db3dbe5d 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitConverter.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; +import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; @@ -103,11 +104,11 @@ public class UnitConverter implements Freezable { private final MapComparator LongUnitIdOrder = new MapComparator<>(); private final MapComparator ShortUnitIdOrder = new MapComparator<>(); - public Comparator getLongUnitIdComparator() { + public MapComparator getLongUnitIdComparator() { return LongUnitIdOrder; } - public Comparator getShortUnitIdComparator() { + public MapComparator getShortUnitIdComparator() { return ShortUnitIdOrder; } @@ -194,7 +195,7 @@ public UnitConverter freeze() { } public void buildMapComparators() { - Set> all = new TreeSet<>(); + Set> all = new TreeSet<>(); Set baseSeen = new HashSet<>(); if (DEBUG) { UnitParser up = new UnitParser(componentTypeData); @@ -259,37 +260,17 @@ public void buildMapComparators() { } } - final EnumSet systems = EnumSet.copyOf(getSystemsEnum(shortUnit)); + final Set systems = getSystemsEnum(shortUnit); // to sort the right items together items together, put together a sort key - UnitSystem sortingSystem = systems.iterator().next(); - switch (sortingSystem) { - case metric: - case si: - case si_acceptable: - case astronomical: - case metric_adjacent: - case person_age: - sortingSystem = UnitSystem.metric; - break; - // country specific - case other: - case ussystem: - case uksystem: - case jpsystem: - sortingSystem = UnitSystem.other; - break; - default: - throw new IllegalArgumentException( - "Add new unitSystem to a grouping: " + sortingSystem); - } - R4 sortKey = + int sortingSystem = UnitSystem.getSortingGroup(systems); + R4 sortKey = Row.of(quantityNumericOrder, sortingSystem, conversionInfo.factor, shortUnit); all.add(sortKey); } LongUnitIdOrder.setErrorOnMissing(true); ShortUnitIdOrder.setErrorOnMissing(true); - for (R4 item : all) { + for (R4 item : all) { String shortId = item.get3(); ShortUnitIdOrder.add(shortId); LongUnitIdOrder.add(getLongId(shortId)); @@ -840,6 +821,9 @@ public String getBaseUnit(String simpleUnit) { * @return */ public String getStandardUnit(String unit) { + if (unit.equals("none")) { + return unit; + } Output metricUnit = new Output<>(); parseUnitId(unit, metricUnit, false); String result = sourceToStandard.get(metricUnit.value); @@ -1887,7 +1871,7 @@ public String getQuantityFromUnit(String unit, boolean showYourWork) { unit = fixDenormalized(unit); try { ConversionInfo unitInfo = parseUnitId(unit, metricUnit, showYourWork); - return metricUnit.value == null ? null : getQuantityFromBaseUnit(metricUnit.value); + return metricUnit.value == null ? "generic" : getQuantityFromBaseUnit(metricUnit.value); } catch (Exception e) { System.out.println("Failed with " + unit + ", " + metricUnit + "\t" + e); return null; @@ -1946,28 +1930,41 @@ public enum UnitSystem { // TODO convert getSystems and SupplementalDataInfo to si_acceptable, metric, metric_adjacent, - ussystem, - uksystem, - jpsystem, - astronomical, person_age, - other, - prefixable; + ussystem(5), + uksystem(5), + jpsystem(10), + astronomical(15), + other(20), + prefixable(1000); + public final int sortingGroup; + + private UnitSystem() { + this.sortingGroup = 0; + } + + private UnitSystem(int sortingGroup) { + this.sortingGroup = sortingGroup; + } - public static final Set SiOrMetric = - ImmutableSet.of( - UnitSystem.metric, - UnitSystem.si, - UnitSystem.metric_adjacent, - UnitSystem.si_acceptable); public static final Set ALL = ImmutableSet.copyOf(UnitSystem.values()); + public static final Set SiOrMetric = + ImmutableSortedSet.copyOf( + ALL.stream().filter(x -> x.sortingGroup > 0).collect(Collectors.toSet())); + public static Set fromStringCollection(Collection stringUnitSystems) { return stringUnitSystems.stream() .map(x -> UnitSystem.valueOf(x)) .collect(Collectors.toSet()); } + public static int getSortingGroup(Set unitSystemSet) { + return unitSystemSet.isEmpty() + ? Integer.MAX_VALUE + : unitSystemSet.iterator().next().sortingGroup; + } + @Deprecated public static Set toStringSet(Collection stringUnitSystems) { return new LinkedHashSet<>( @@ -2003,6 +2000,7 @@ public Set getSystems(String unit) { return UnitSystem.toStringSet(getSystemsEnum(unit)); } + /** Returns immutable sorted set in enum order */ public Set getSystemsEnum(String unit) { Set result = null; UnitId id = createUnitId(unit); @@ -2163,6 +2161,9 @@ public Map getConstants() { } public String getBaseUnitFromQuantity(String unitQuantity) { + if (unitQuantity.equals("generic")) { + return "none"; + } boolean invert = false; if (unitQuantity.endsWith("-inverse")) { invert = true; @@ -2464,4 +2465,77 @@ public String format( nf3.locale(uLocale).format(outputAmount.doubleValue()).toString(); return com.ibm.icu.text.MessageFormat.format(pattern, cldrFormattedNumber); } + + public enum QuantityGroup { + Duration("duration", "year-duration"), + Length("length"), + Area("area"), + Volume_Metric("volume"), + Volume_US("volume"), + Volume_Other("volume"), + Mass("mass"), + SpeedAcceleration("acceleration", "speed"), + Energy("energy"), + ForcePower("power", "force", "ionizing-radiation", "radioactivity"), + Electrical( + "electric-capacitance", + "electric-charge", + "electric-conductance", + "electric-current", // + "electric-inductance", + "electric-resistance", + "voltage", + "frequency"), + LightMagnetism( + "illuminance", + "luminous-flux", + "luminous-intensity", + "magnetic-flux", + "magnetic-induction"), + TemperaturePressure("temperature", "generic", "pressure", "pressure-per-length"), + Graphics("graphics", "resolution", "typewidth"), + Digital("digital"), + Angle("angle", "solid-angle"), + OtherUnits( + "concentration", + "concentration-mass", + "consumption", + "consumption-inverse", // + "portion", + "substance-amount", + "catalytic-activity" // + ); + private final List quantities; + + private QuantityGroup(String... quantities) { + this.quantities = Arrays.asList(quantities); + } + + List getQuantities() { + return quantities; + } + + private static final Multimap quantityToQuantityGroups; + private static final MapComparator quantityCollator = new MapComparator<>(); + + static { + Multimap temp = TreeMultimap.create(); + for (QuantityGroup qg : QuantityGroup.values()) { + for (String quantity : qg.quantities) { + temp.put(quantity, qg); + quantityCollator.add(quantity); + } + } + quantityToQuantityGroups = ImmutableMultimap.copyOf(temp); + quantityCollator.freeze(); + } + + public static final List getQuantityGroups(String quantity) { + return (List) quantityToQuantityGroups.get(quantity); + } + + public static final MapComparator getQuantityCollator() { + return quantityCollator; + } + } } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPathType.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPathType.java index 0d113d341c6..199cd34af07 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPathType.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/UnitPathType.java @@ -120,6 +120,15 @@ public static UnitPathType getPathType(XPathParts parts) { throw new IllegalArgumentException("PathType: " + parts); } + public static String getDisplayNamePathFromLongId(String longUnitId, String width) { + return "//ldml/units/unitLength[@type=\"" + + width + + "\"]/" + + "unit[@type=\"" + + longUnitId + + "\"]/displayName"; + } + public String getTranslationPath( LocaleStringProvider resolvedFile, String width, diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java index ea84b5dd502..344f9ea6a11 100644 --- a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java +++ b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUnits.java @@ -109,6 +109,7 @@ import org.unicode.cldr.util.TempPrintWriter; import org.unicode.cldr.util.UnitConverter; import org.unicode.cldr.util.UnitConverter.ConversionInfo; +import org.unicode.cldr.util.UnitConverter.QuantityGroup; import org.unicode.cldr.util.UnitConverter.TargetInfo; import org.unicode.cldr.util.UnitConverter.UnitComplexity; import org.unicode.cldr.util.UnitConverter.UnitId; @@ -3393,6 +3394,11 @@ public void TestUnitsToTranslate() { .precision(Precision.fixedSignificantDigits(7)) .locale(Locale.ENGLISH); Output base = new Output<>(); + if (isVerbose()) { + System.out.println( + "\nQuantity\tQ Order\tUnit\tLong Prefix\tST Page" + + "\tRevision\tR Order\tFactor\tBase Unit\tSystems\tType\tTranslation Status\tComment"); + } for (Entry entry : shortUnitToTranslationStatus40.entrySet()) { String shortUnit = entry.getKey(); var conversionInfo = converter.parseUnitId(shortUnit, base, false); @@ -3404,19 +3410,40 @@ public void TestUnitsToTranslate() { .replace("E", " × 10^"); TranslationStatus status40 = entry.getValue(); - if (isVerbose()) + if (isVerbose()) { + final String longId = converter.getLongId(shortUnit); + String path = UnitPathType.getDisplayNamePathFromLongId(longId, "short"); + PathHeader pf = PathHeader.getFactory().fromPath(path); + String quantity = converter.getQuantityFromUnit(shortUnit, false); + final QuantityGroup quantityGroup = + UnitConverter.QuantityGroup.getQuantityGroups(quantity).iterator().next(); System.out.println( - converter.getQuantityFromUnit(shortUnit, false) + quantity + + "\t" + + UnitConverter.QuantityGroup.getQuantityCollator() + .getNumericOrder(quantity) + "\t" + shortUnit + "\t" - + converter.getSystemsEnum(shortUnit) + + Splitter.on('-').split(longId).iterator().next() + + "\t" + + pf.getPageId() + + "\t" + + quantityGroup + + "\t" + + quantityGroup.ordinal() + "\t" + factor + "\t" + + converter.getStandardUnit( + converter.getBaseUnitFromQuantity(quantity)) + + "\t" + + converter.getSystemsEnum(shortUnit) + + "\t" + (converter.isSimple(shortUnit) ? "simple" : "complex") + "\t" + status40); + } } }