Skip to content

Commit

Permalink
CLDR-17153 Cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@macchiati
macchiati committed Oct 18, 2023
1 parent f20bee6 commit e47f5bd
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 42 deletions.
79 changes: 62 additions & 17 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/ChartLocaleGrowth.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import com.ibm.icu.util.VersionInfo;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -21,7 +20,6 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import org.unicode.cldr.draft.FileUtilities;
import org.unicode.cldr.tool.Option.Options;
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRFile;
Expand All @@ -41,6 +39,7 @@
import org.unicode.cldr.util.SimpleFactory;
import org.unicode.cldr.util.StandardCodes;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.TempPrintWriter;
import org.unicode.cldr.util.VettingViewer;
import org.unicode.cldr.util.VettingViewer.MissingStatus;

Expand All @@ -55,6 +54,9 @@ public class ChartLocaleGrowth {
testInfo.getSupplementalDataInfo();
static final Set<String> CldrModernLocales =
StandardCodes.make().getLocaleCoverageLocales(Organization.cldr, Set.of(Level.MODERN));
static final Set<String> SpecialLocales =
StandardCodes.make()
.getLocaleCoverageLocales(Organization.special, Set.of(Level.MODERN));

private static org.unicode.cldr.util.Factory factory =
testInfo.getCommonAndSeedAndMainAndAnnotationsFactory();
Expand Down Expand Up @@ -106,15 +108,26 @@ public static void main(String[] args) throws IOException {
Matcher localeMatcher = PatternCache.get(MyOptions.filter.option.getValue()).matcher("");
Matcher versionMatcher = PatternCache.get(MyOptions.Versions.option.getValue()).matcher("");

try (PrintWriter out =
FileUtilities.openUTF8Writer(
CLDRPaths.CHART_DIRECTORY + "tsv/", "locale-growth.tsv")) {
doGrowth(localeMatcher, versionMatcher, out);
try (TempPrintWriter out =
new TempPrintWriter(
CLDRPaths.CHART_DIRECTORY + "tsv/", "locale-growth.tsv");
TempPrintWriter log =
new TempPrintWriter(
CLDRPaths.CHART_DIRECTORY + "tsv/", "locale-growth-log.tsv");
TempPrintWriter logPaths =
new TempPrintWriter(
CLDRPaths.CHART_DIRECTORY + "tsv/", "locale-growth-paths.tsv"); ) {
doGrowth(localeMatcher, versionMatcher, out, log, logPaths);
return;
}
}

private static void doGrowth(Matcher localeMatcher, Matcher versionMatcher, PrintWriter out) {
private static void doGrowth(
Matcher localeMatcher,
Matcher versionMatcher,
TempPrintWriter out,
TempPrintWriter log,
TempPrintWriter logPaths) {
TreeMap<String, List<Double>> growthData =
new TreeMap<>(Ordering.natural().reverse()); // sort by version, descending
Map<String, FoundAndTotal> latestData = null;
Expand All @@ -131,7 +144,7 @@ private static void doGrowth(Matcher localeMatcher, Matcher versionMatcher, Prin
String dir = ToolConstants.getBaseDirectory(version.getVersionString(2, 3));
boolean showMissing = last == versionNormalizedVersionAndYear;
Map<String, FoundAndTotal> currentData =
addGrowth(factory, dir, localeMatcher, showMissing);
addGrowth(factory, dir, localeMatcher, showMissing, log, logPaths);
long found = 0;
long total = 0;
for (Entry<String, FoundAndTotal> entry : currentData.entrySet()) {
Expand Down Expand Up @@ -187,7 +200,7 @@ private ReleaseInfo(VersionInfo versionInfo, int year) {

static {
Object[][] mapping = {
{VersionInfo.getInstance(43), 2023},
{VersionInfo.getInstance(44), 2023},
{VersionInfo.getInstance(42), 2022},
{VersionInfo.getInstance(40), 2021},
{VersionInfo.getInstance(38), 2020},
Expand Down Expand Up @@ -277,7 +290,9 @@ private static Map<String, FoundAndTotal> addGrowth(
org.unicode.cldr.util.Factory latestFactory,
String dir,
Matcher localeMatcher,
boolean showMissing) {
boolean showMissing,
TempPrintWriter log,
TempPrintWriter logPaths) {
final File mainDir = new File(dir + "/common/main/");
final File annotationDir = new File(dir + "/common/annotations/");
File[] paths =
Expand Down Expand Up @@ -355,17 +370,24 @@ private static Map<String, FoundAndTotal> addGrowth(

if (showMissing) {
if (CldrModernLocales.contains(locale)) {
final boolean isSpecial = SpecialLocales.contains(locale);
if (firstShowMissing) {
firstShowMissing = false;
System.out.println(
"\nLocale"
log.printlnWithTabs(
16,
"Locale\tTC"
+ "\tCore\tUnc\tMiss"
+ "\tBasic\tUnc\tMiss"
+ "\tModer\tUnc\tMiss"
+ "\tModern\tUnc\tMiss");
+ "\tModern\tUnc\tMiss"
+ "\tTotal\tUnc\tMiss");
logPaths.printlnWithTabs(3, "Locale\tLevel\tStatus\tPath");
}
System.out.println(
log.printlnWithTabs(
16,
locale
+ "\t"
+ (isSpecial ? "" : "TC")
+ show(
Level.CORE,
foundCounter,
Expand All @@ -385,7 +407,30 @@ private static Map<String, FoundAndTotal> addGrowth(
Level.MODERN,
foundCounter,
unconfirmedCounter,
missingCounter)
+ show(
null, // total
foundCounter,
unconfirmedCounter,
missingCounter));
if (!isSpecial) {
long count = unconfirmedCounter.getTotal() + missingCounter.getTotal();
for (Entry<MissingStatus, String> statusAndPath : missingPaths.entrySet()) {
logPaths.printlnWithTabs(
3,
locale
+ "\t"
+ count
+ "\t"
+ statusAndPath.getKey()
+ "\t"
+ statusAndPath.getValue());
}
for (String path : unconfirmedPaths) {
logPaths.printlnWithTabs(
3, locale + "\t" + count + "\tunconfirmed\t" + path);
}
}
int line = 0;
}
}
Expand Down Expand Up @@ -446,10 +491,10 @@ private static String show(
Counter<Level> unconfirmedCounter,
Counter<Level> missingCounter) {
return "\t"
+ foundCounter.get(level)
+ (level != null ? foundCounter.get(level) : foundCounter.getTotal())
+ "\t"
+ unconfirmedCounter.get(level)
+ (level != null ? unconfirmedCounter.get(level) : unconfirmedCounter.getTotal())
+ "\t"
+ missingCounter.get(level);
+ (level != null ? missingCounter.get(level) : missingCounter.getTotal());
}
}
29 changes: 4 additions & 25 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/ShowLocaleCoverage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public class ShowLocaleCoverage {
"https://github.com/unicode-org/cldr-staging/blob/main/docs/charts/"
+ ToolConstants.CHART_VI.getVersionString(1, 2)
+ "/tsv/";
private static final Splitter LF_SPLITTER = Splitter.on('\n');
public static final Splitter LF_SPLITTER = Splitter.on('\n');

// thresholds for measuring Level attainment
private static final double BASIC_THRESHOLD = 1;
Expand Down Expand Up @@ -343,7 +343,7 @@ private static void showCoverage(
tsv_missing_counts.println(TSV_MISSING_COUNTS_HEADER);

final int propertiesCoverageTabCount = 2;
printlnWithTabs(propertiesCoverage, propertiesCoverageTabCount, PROPERTIES_HEADER);
propertiesCoverage.printlnWithTabs(propertiesCoverageTabCount, PROPERTIES_HEADER);

Set<String> checkModernLocales =
STANDARD_CODES.getLocaleCoverageLocales(
Expand Down Expand Up @@ -908,8 +908,7 @@ private static void showCoverage(
// now write properties file line

if (computed != Level.UNDETERMINED) {
printlnWithTabs(
propertiesCoverage,
propertiesCoverage.printlnWithTabs(
propertiesCoverageTabCount,
locale
+ " ;\t"
Expand Down Expand Up @@ -949,7 +948,7 @@ private static void showCoverage(
}
}
String lineToPrint = "\n#EOF";
printlnWithTabs(propertiesCoverage, propertiesCoverageTabCount, lineToPrint);
propertiesCoverage.printlnWithTabs(propertiesCoverageTabCount, lineToPrint);

pw.println("<h3><a name='main_table' href='#main_table'>Main Table</a></h3>");
pw.println(tablePrinter.toTable());
Expand Down Expand Up @@ -1065,26 +1064,6 @@ private static void showCoverage(
}
}

/** Println with extra tabs to appear as table in github */
public static void printlnWithTabs(
TempPrintWriter printWriter, int desiredCount, String textToPrint) {
StringBuilder result = new StringBuilder();
for (String line : LF_SPLITTER.split(textToPrint)) {
long count = desiredCount - line.chars().filter(ch -> ch == '\t').count();
if (count < 0) {
throw new IllegalArgumentException("Too many tabs in line.");
}
result.append(line);
if (count != 0) {
for (int i = 0; i < count; ++i) {
result.append('\t');
}
}
result.append('\n');
}
printWriter.print(result);
}

private static String linkTsv(String tsvFileName) {
return "<a href='" + TSV_BASE + tsvFileName + "' target='cldr-tsv'>" + tsvFileName + "</a>";
}
Expand Down
20 changes: 20 additions & 0 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/TempPrintWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import java.nio.file.StandardCopyOption;
import java.util.Random;
import org.unicode.cldr.draft.FileUtilities;
import org.unicode.cldr.tool.ShowLocaleCoverage;

/**
* Simple utility to create a temporary file, write into it, then close it. If the file differs from
Expand Down Expand Up @@ -105,6 +106,25 @@ public void println() {
tempPrintWriter.println();
}

/** Println with extra tabs on each line, as needed, to appear as table in github */
public void printlnWithTabs(int desiredCount, String textToPrint) {
StringBuilder result = new StringBuilder();
for (String line : ShowLocaleCoverage.LF_SPLITTER.split(textToPrint)) {
long count = desiredCount - line.chars().filter(ch -> ch == '\t').count();
if (count < 0) {
throw new IllegalArgumentException("Too many tabs in line.");
}
result.append(line);
if (count != 0) {
for (int i = 0; i < count; ++i) {
result.append('\t');
}
}
result.append('\n');
}
print(result);
}

/**
* If contents(newFile) ≠ contents(oldFile), rename newFile to old. Otherwise delete newfile.
* Return true if replaced. *
Expand Down

0 comments on commit e47f5bd

Please sign in to comment.