Skip to content

Commit

Permalink
CLDR-17719 update v46 charts (#3800)
Browse files Browse the repository at this point in the history
  • Loading branch information
@macchiati
macchiati authored Jun 13, 2024
1 parent 4bed18b commit 22afc9b
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 155 deletions.
2 changes: 1 addition & 1 deletion tools/cldr-code/src/main/java/org/unicode/cldr/tool/Chart.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public abstract class Chart {
public static final String CHART_VERSION_DIRECTORY =
ToolConstants.getBaseDirectory(ToolConstants.CHART_VERSION);

public static final String GITHUB_ROOT = CLDRURLS.CLDR_REPO_ROOT + "/blob/main/";
public static final String GITHUB_ROOT = CLDRURLS.CLDR_REPO_MAIN;
public static final String LDML_SPEC = "https://unicode.org/reports/tr35/";

public static String dataScrapeMessage(String specPart, String testFile, String... dataFiles) {
Expand Down
5 changes: 4 additions & 1 deletion tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateTransformCharts.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.CLDRTransforms;
import org.unicode.cldr.util.CLDRTransforms.ParsedTransformID;
import org.unicode.cldr.util.CLDRURLS;
import org.unicode.cldr.util.CldrUtility;
import org.unicode.cldr.util.FileCopier;
import org.unicode.cldr.util.Pair;
Expand Down Expand Up @@ -367,7 +368,9 @@ private static void showAllLatin() throws IOException {
+ "These charts do not show the available script and language transliterations that are not to/from Latin. "
+ "It also does not show other transforms that are available in CLDR, such as the casing transformations, "
+ "full-width and half-width transformations, and specialized transformations such as IPA-XSampa. "
+ "For the latest snapshot of the data files, see <a href='https://github.com/unicode-org/cldr/tree/main/common/transforms'>Transform XML Data</a>. "
+ "For the latest snapshot of the data files, see <a href='"
+ CLDRURLS.CLDR_REPO_BASE
+ "tree/main/common/transforms'>Transform XML Data</a>. "
+ "For more information, see below.</blockquote>");
index.flush();
index.println("<ul>");
Expand Down
316 changes: 167 additions & 149 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowLanguages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeMultimap;
import com.ibm.icu.impl.Relation;
import com.ibm.icu.impl.Row.R2;
Expand Down Expand Up @@ -1362,7 +1362,7 @@ private String getLanguagePluralMessage(String msg, String languageCode) {
}

private String getLanguageName(String languageCode) {
String result = english.getName(languageCode);
String result = english.getName(languageCode, true, CLDRFile.SHORT_ALTS);
if (!result.equals(languageCode)) return result;
Set<String> names = Iso639Data.getNames(languageCode);
if (names != null && names.size() != 0) {
Expand All @@ -1372,171 +1372,189 @@ private String getLanguageName(String languageCode) {
}

static final Set<Organization> TC_Vendors =
ImmutableSet.of(
Organization.apple,
Organization.google,
Organization.microsoft,
Organization.cldr);
Sets.union(
Organization.getTCOrgs(),
// This adds the CLDR org at the end of the list
Set.of(Organization.cldr));

private void showCoverageGoals(PrintWriter pw) throws IOException {
PrintWriter pw2 =
new PrintWriter(
new FormattedFileWriter(
null,
"Coverage Goals",
null
// "<p>" +
// "The following show default coverage goals for larger
// organizations. " +
// "<i>[n/a]</i> shows where there is no specific value for a
// given organization, " +
// "while <i>(...)</i> indicates that the goal is inherited from
// the parent. " +
// "A * is added if the goal differs from the parent locale's
// goal. " +
// "For information on what these goals mean (comprehensive,
// modern, moderate,...), see the LDML specification "
// +
// "<a
// href='http://www.unicode.org/reports/tr35/#Coverage_Levels'>Appendix M: Coverage Levels</a>. " +
// +
// "</p>"
,
null));

TablePrinter tablePrinter =
new TablePrinter()
// tablePrinter.setSortPriorities(0,4)
.addColumn("Language", "class='source'", null, "class='source'", false)
.setSortPriority(0)
.setBreakSpans(false)
try (PrintWriter pw2 =
new PrintWriter(
new FormattedFileWriter(
null,
"Coverage Goals",
null,
SUPPLEMENTAL_INDEX_ANCHORS));
PrintWriter coverageGoalsTsv =
FileUtilities.openUTF8Writer(
CLDRPaths.CHART_DIRECTORY + "tsv/", "coverage_goals.tsv"); ) {

TablePrinter tablePrinter =
new TablePrinter()
// tablePrinter.setSortPriorities(0,4)
.addColumn(
"Language", "class='source'", null, "class='source'", false)
.setSortPriority(0)
.setBreakSpans(false)
.addColumn(
"Code",
"class='source'",
"<a href=\""
+ CLDRURLS.CLDR_REPO_ROOT
+ "/blob/main/common/main/{0}.xml\">{0}</a>",
"class='source'",
false)
.addColumn(
"D. Votes",
"class='target'",
null,
"class='target'",
false);

Map<Organization, Map<String, Level>> vendordata = sc.getLocaleTypes();
Set<String> locales = new TreeSet<>();
Set<Organization> vendors = new LinkedHashSet<>();
Set<Organization> smallVendors = new LinkedHashSet<>();

for (Organization organization : TC_Vendors) {
// if (vendor.equals(Organization.java)) continue;
Map<String, Level> data = vendordata.get(organization);
vendors.add(organization);
tablePrinter
.addColumn(
"Code",
"class='source'",
"<a href=\"http://www.unicode.org/cldr/data/common/main/{0}.xml\">{0}</a>",
"class='source'",
organization.getDisplayName(),
"class='target'",
null,
"class='target'",
false)
.addColumn("D. Votes", "class='target'", null, "class='target'", false);
.setSpanRows(false);
locales.addAll(data.keySet());
showTabbedOrgLevels(coverageGoalsTsv, organization, data);
}

Map<Organization, Map<String, Level>> vendordata = sc.getLocaleTypes();
Set<String> locales = new TreeSet<>();
Set<Organization> vendors = new LinkedHashSet<>();
Set<Organization> smallVendors = new LinkedHashSet<>();
for (Entry<Organization, Map<String, Level>> vendorData : vendordata.entrySet()) {
Organization organization = vendorData.getKey();
if (!TC_Vendors.contains(organization)) {
smallVendors.add(organization);
Map<String, Level> data = vendordata.get(organization);
showTabbedOrgLevels(coverageGoalsTsv, organization, data);
continue;
}
}

for (Organization organization : TC_Vendors) {
// if (vendor.equals(Organization.java)) continue;
Map<String, Level> data = vendordata.get(organization);
vendors.add(organization);
tablePrinter
.addColumn(
organization.getDisplayName(),
"class='target'",
null,
"class='target'",
false)
.setSpanRows(false);
locales.addAll(data.keySet());
}
Collection<Comparable[]> data = new ArrayList<>();
List<String> list = new ArrayList<>();
LanguageTagParser ltp = new LanguageTagParser();
// String alias2 = getAlias("sh_YU");

for (Entry<Organization, Map<String, Level>> vendorData : vendordata.entrySet()) {
Organization vendor = vendorData.getKey();
if (!TC_Vendors.contains(vendor)) {
smallVendors.add(vendor);
continue;
}
}
pw2.append("<h2>TC Orgs</h2>");

Collection<Comparable[]> data = new ArrayList<>();
List<String> list = new ArrayList<>();
LanguageTagParser ltp = new LanguageTagParser();
// String alias2 = getAlias("sh_YU");

pw2.append("<h2>TC Orgs</h2>");

for (String locale : locales) {
list.clear();
String localeCode = locale.equals("*") ? "und" : locale;
String alias = getAlias(localeCode);
if (!alias.equals(localeCode)) {
throw new IllegalArgumentException(
"Should use canonical form: " + locale + " => " + alias);
}
String baseLang = ltp.set(localeCode).getLanguage();
String baseLangName = getLanguageName(baseLang);
list.add("und".equals(localeCode) ? "other" : baseLangName);
list.add(locale);
int defaultVotes =
supplementalDataInfo.getRequiredVotes(CLDRLocale.getInstance(locale), null);
list.add(String.valueOf(defaultVotes));
for (Organization vendor : vendors) {
String status = getVendorStatus(locale, vendor, vendordata);
// if (!baseLang.equals(locale) && !status.startsWith("<")) {
// String langStatus = getVendorStatus(baseLang, vendor,
// vendordata);
// if (!langStatus.equals(status)) {
// status += "*";
// }
// }
list.add(status);
for (String locale : locales) {
list.clear();
String localeCode = locale.equals("*") ? "und" : locale;
String alias = getAlias(localeCode);
if (!alias.equals(localeCode)) {
throw new IllegalArgumentException(
"Should use canonical form: " + locale + " => " + alias);
}
// String baseLang = ltp.set(localeCode).getLanguage();
String baseLangName = getLanguageName(localeCode);
list.add("und".equals(localeCode) ? "other" : baseLangName);
list.add(locale);
int defaultVotes =
supplementalDataInfo.getRequiredVotes(
CLDRLocale.getInstance(locale), null);
list.add(String.valueOf(defaultVotes));
for (Organization vendor : vendors) {
String status = getVendorStatus(locale, vendor, vendordata);
// if (!baseLang.equals(locale) &&
// !status.startsWith("<")) {
// String langStatus = getVendorStatus(baseLang,
// vendor,
// vendordata);
// if (!langStatus.equals(status)) {
// status += "*";
// }
// }
list.add(status);
}
data.add(list.toArray(new String[list.size()]));
}
data.add(list.toArray(new String[list.size()]));
}
Comparable[][] flattened = data.toArray(new Comparable[data.size()][]);
String value = tablePrinter.addRows(flattened).toTable();
pw2.println(value);

pw2.append("<h2>Others</h2><div align='left'><ul>");

for (Organization vendor2 : smallVendors) {
pw2.append("<li><b>");
pw2.append(TransliteratorUtilities.toHTML.transform(vendor2.getDisplayName()))
.append(": </b>");
boolean first1 = true;
for (Level level : Level.values()) {
boolean first2 = true;
Level other = null;
for (Entry<String, Level> data2 : vendordata.get(vendor2).entrySet()) {
String key = data2.getKey();
Level level2 = data2.getValue();
if (level != level2) {
continue;
}
if (key.equals("*")) {
other = level2;
continue;
}
if (first2) {
if (first1) {
first1 = false;
Comparable[][] flattened = data.toArray(new Comparable[data.size()][]);
String value = tablePrinter.addRows(flattened).toTable();
pw2.println(value);

pw2.append("<h2>Others</h2><div align='left'><ul>");

for (Organization vendor2 : smallVendors) {
pw2.append("<li><b>");
pw2.append(TransliteratorUtilities.toHTML.transform(vendor2.getDisplayName()))
.append(": </b>");
boolean first1 = true;
for (Level level : Level.values()) {
boolean first2 = true;
Level other = null;
for (Entry<String, Level> data2 : vendordata.get(vendor2).entrySet()) {
String key = data2.getKey();
Level level2 = data2.getValue();
if (level != level2) {
continue;
}
if (key.equals("*")) {
other = level2;
continue;
}
if (first2) {
if (first1) {
first1 = false;
} else {
pw2.append("; ");
}
pw2.append(level2.toString()).append(": ");
first2 = false;
} else {
pw2.append("; ");
pw2.append(", ");
}
pw2.append(level2.toString()).append(": ");
first2 = false;
} else {
pw2.append(", ");
pw2.append(TransliteratorUtilities.toHTML.transform(key));
}
pw2.append(TransliteratorUtilities.toHTML.transform(key));
}
if (other != null) {
if (first2) {
if (first1) {
first1 = false;
if (other != null) {
if (first2) {
if (first1) {
first1 = false;
} else {
pw2.append("; ");
}
pw2.append(level.toString()).append(": ");
first2 = false;
} else {
pw2.append("; ");
pw2.append(", ");
}
pw2.append(level.toString()).append(": ");
first2 = false;
} else {
pw2.append(", ");
pw2.append("<i>other</i>");
}
pw2.append("<i>other</i>");
}
pw2.append("</li>");
}
pw2.append("</li>");
pw2.append("</ul></div>");
}
}

public void showTabbedOrgLevels(
PrintWriter coverageGoalsTsv, Organization organization, Map<String, Level> data) {
coverageGoalsTsv.println(
String.format(
"\n#%s\t;\t%s\t;\t%s\t;\t%s\n",
"Org", "Locale", "Level", "Locale Name"));
for (Entry<String, Level> entry : data.entrySet()) {
String locale = entry.getKey();
Level level = entry.getValue();
final String name =
locale.equals("*")
? "ALL"
: english.getName(locale, true, CLDRFile.SHORT_ALTS);
coverageGoalsTsv.println(
String.format(
"%s\t;\t%s\t;\t%s\t;\t%s", organization, locale, level, name));
}
pw2.append("</ul></div>");
pw2.close();
}

LanguageTagParser lpt2 = new LanguageTagParser();
Expand Down
Loading

0 comments on commit 22afc9b

Please sign in to comment.