From 34cb8f4aa7a04d1c5c9a1c186a6d7bb9bb0dcf28 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 2 May 2024 17:09:59 -0500 Subject: [PATCH] CLDR-17106 add more fixed candidate items - restructure DataPage / DiskDataCache to hold the candidate providers - add a FixedCandidateProvider structure, designed to allow quick additions of future fixed candidates. --- .../java/org/unicode/cldr/web/DataPage.java | 24 ++---- .../org/unicode/cldr/web/DiskDataCache.java | 70 +++++++++++++++- .../cldr/web/FixedCandidateProvider.java | 84 +++++++++++++++++++ .../java/org/unicode/cldr/web/STFactory.java | 10 ++- .../java/org/unicode/cldr/web/SurveyMain.java | 17 ++-- 5 files changed, 174 insertions(+), 31 deletions(-) create mode 100644 tools/cldr-apps/src/main/java/org/unicode/cldr/web/FixedCandidateProvider.java diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DataPage.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DataPage.java index 2dfa2c7b3e0..767ecee8cba 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DataPage.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DataPage.java @@ -29,9 +29,6 @@ import org.unicode.cldr.util.CLDRInfo.CandidateInfo; import org.unicode.cldr.util.CLDRInfo.PathValueInfo; import org.unicode.cldr.util.CLDRInfo.UserInfo; -import org.unicode.cldr.util.GrammarInfo.GrammaticalFeature; -import org.unicode.cldr.util.GrammarInfo.GrammaticalScope; -import org.unicode.cldr.util.GrammarInfo.GrammaticalTarget; import org.unicode.cldr.util.PathHeader.PageId; import org.unicode.cldr.util.PathHeader.SurveyToolStatus; import org.unicode.cldr.util.VoteResolver.Status; @@ -566,8 +563,7 @@ public DataRow(String xpath) { /** check to see if there are any 'fixed' values, i.e. no freeform input is allowed. */ public void addFixedCandidates() { - Collection candidates = getFixedCandidates(); - // Could have other XPaths here + Collection candidates = stFactory.getFixedCandidates(locale, xpath); if (candidates == null || candidates.isEmpty()) { return; @@ -578,18 +574,6 @@ public void addFixedCandidates() { } } - private Collection getFixedCandidates() { - if (PatternCache.get("^//ldml/units/unitLength.*/unit.*/gender") - .matcher(xpath) - .matches()) { - return grammarInfo.get( - GrammaticalTarget.nominal, - GrammaticalFeature.grammaticalGender, - GrammaticalScope.units); - } - return Collections.emptySet(); - } - /** * Add a new CandidateItem to this DataRow, with the given value; or, if this DataRow * already has an item with this value, return it. @@ -1445,6 +1429,7 @@ public static CheckCLDR.Options getSimpleOptions(CLDRLocale locale) { private final XPathMatcher matcher; private final PageId pageId; private CLDRFile diskFile; + private final STFactory stFactory; private static final boolean DEBUG_DATA_PAGE = false; private String creationTime = null; // only used if DEBUG_DATA_PAGE @@ -1456,7 +1441,9 @@ public static CheckCLDR.Options getSimpleOptions(CLDRLocale locale) { this.sm = sm; this.matcher = matcher; xpathPrefix = prefix; - ballotBox = sm.getSTFactory().ballotBoxForLocale(locale); + this.stFactory = sm.getSTFactory(); + ballotBox = stFactory.ballotBoxForLocale(locale); + this.pageId = pageId; if (DEBUG_DATA_PAGE) { @@ -1465,7 +1452,6 @@ public static CheckCLDR.Options getSimpleOptions(CLDRLocale locale) { .format(Calendar.getInstance().getTime()); System.out.println("🌴 Created new DataPage for loc " + loc + " at " + creationTime); } - grammarInfo = sm.getSupplementalDataInfo().getGrammarInfo(locale.getBaseName()); } /** diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java index 84ab7277ed8..f062523e7a5 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java @@ -1,24 +1,49 @@ package org.unicode.cldr.web; +import static org.unicode.cldr.web.FixedCandidateProvider.forEnumWithFixedXpath; + import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; +import javax.annotation.Nonnull; import org.unicode.cldr.util.CLDRFile; import org.unicode.cldr.util.CLDRLocale; import org.unicode.cldr.util.Factory; +import org.unicode.cldr.util.GrammarInfo; +import org.unicode.cldr.util.GrammarInfo.GrammaticalFeature; +import org.unicode.cldr.util.GrammarInfo.GrammaticalScope; +import org.unicode.cldr.util.GrammarInfo.GrammaticalTarget; import org.unicode.cldr.util.PathHeader; +import org.unicode.cldr.util.SupplementalDataInfo; import org.unicode.cldr.util.XMLSource; +import org.unicode.cldr.util.personname.PersonNameFormatter; +import org.unicode.cldr.web.FixedCandidateProvider.PatternCacheCandidateProvider; /** Cache for on-disk immutable data. */ public class DiskDataCache { static final Logger logger = Logger.getLogger(DiskDataCache.class.getSimpleName()); private final Factory factory; - private final CLDRFile english; + private final CLDRFile english; // TODO: Unused? private final PathHeader.Factory phf; + final SupplementalDataInfo sdi; + + final List personNameProviders = + ImmutableList.of( + forEnumWithFixedXpath( + "//ldml/personNames/parameterDefault[@parameter=\"formality\"]", + PersonNameFormatter.Formality.values()), + forEnumWithFixedXpath( + "//ldml/personNames/parameterDefault[@parameter=\"length\"]", + PersonNameFormatter.Length.values())); /** this is the immutable cousin of STFactory.PerLocaleData, for the on-disk data */ class DiskDataEntry { @@ -27,18 +52,57 @@ class DiskDataEntry { final CLDRFile diskFile; final Set pathsForFile; + private final List fixedCandidateProviders = new LinkedList<>(); + public DiskDataEntry(CLDRLocale locale) { this.locale = locale; diskData = factory.makeSource(locale.getBaseName()).freeze(); diskFile = factory.make(locale.getBaseName(), true).freeze(); pathsForFile = getPathHeaderFactory().pathsForFile(diskFile); + + addFixedCandidateProviders(); + } + + private void addFixedCandidateProviders() { + // Add all Candidate Providers here + fixedCandidateProviders.add(new GrammarCandidateProvider()); + fixedCandidateProviders.addAll(personNameProviders); + } + + /** Candidate provider for a regex */ + class GrammarCandidateProvider extends PatternCacheCandidateProvider { + final GrammarInfo grammarInfo = sdi.getGrammarInfo(locale.getBaseName()); + + public GrammarCandidateProvider() { + super("^//ldml/units/unitLength.*/unit.*gender"); + } + + @Override + protected Collection getCandidates() { + return grammarInfo.get( + GrammaticalTarget.nominal, + GrammaticalFeature.grammaticalGender, + GrammaticalScope.units); + } + } + + /** + * @returns a list of values (or empty list) of any 'fixed' candidates for this xpath + */ + Collection getFixedCandidates(final String xpath) { + for (final FixedCandidateProvider fcp : fixedCandidateProviders) { + final Collection r = fcp.apply(xpath); + if (r != null) return r; + } + return Collections.emptySet(); } } - public DiskDataCache(Factory f, CLDRFile english) { + public DiskDataCache(Factory f, CLDRFile english, SupplementalDataInfo sdi) { this.factory = f; this.english = english; this.phf = PathHeader.getFactory(english); + this.sdi = sdi; } public PathHeader.Factory getPathHeaderFactory() { @@ -51,7 +115,7 @@ public PathHeader.Factory getPathHeaderFactory() { new CacheLoader() { @Override - public DiskDataEntry load(CLDRLocale l) throws Exception { + public DiskDataEntry load(@Nonnull CLDRLocale l) throws Exception { return new DiskDataEntry(l); } }); diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/FixedCandidateProvider.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/FixedCandidateProvider.java new file mode 100644 index 00000000000..714774f0eff --- /dev/null +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/FixedCandidateProvider.java @@ -0,0 +1,84 @@ +package org.unicode.cldr.web; + +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; +import org.unicode.cldr.util.PatternCache; + +abstract class FixedCandidateProvider + implements java.util.function.Function> { + /** + * @returns a list of values (or null if not applicable) of any 'fixed' candidates for this + * XPath + */ + public abstract Collection apply(String xpath); + + /** helper function to convert an Enum to an array of strings */ + static final > Collection enumValueStrings(T forValues[]) { + final List l = new ArrayList(forValues.length); + for (final T t : forValues) { + l.add(t.toString()); + } + return ImmutableList.copyOf(l); + } + + /** Candidate provider using a Pattern Cache */ + abstract static class PatternCacheCandidateProvider extends FixedCandidateProvider { + final Pattern pattern; + + public PatternCacheCandidateProvider(String patternString) { + pattern = PatternCache.get(patternString); + } + + public Collection apply(String xpath) { + if (pattern.matcher(xpath).matches()) { + return getCandidates(); + } else { + return null; // not applicable + } + } + + protected abstract Collection getCandidates(); + } + /** Candidate provider for a single string (not regex) */ + abstract static class StringCandidateProvider extends FixedCandidateProvider { + final String pattern; + + public StringCandidateProvider(String xpath) { + pattern = xpath; + } + + public Collection apply(String xpath) { + if (pattern.equals(xpath)) { + return getCandidates(); + } else { + return null; // not applicable + } + } + + protected abstract Collection getCandidates(); + } + + /** create a provider that matches an fixed XPath and returns a set of values */ + public static > FixedCandidateProvider forEnumWithFixedXpath( + String xpath, T[] values) { + return new EnumStringCandidateProvider( + xpath, FixedCandidateProvider.enumValueStrings(values)); + } + + private static class EnumStringCandidateProvider extends StringCandidateProvider { + private final Collection values; + + @Override + protected Collection getCandidates() { + return values; + } + + EnumStringCandidateProvider(final String xpath, final Collection values) { + super(xpath); + this.values = values; + } + } +} diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java index 71d32a87685..3451e94764d 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java @@ -19,6 +19,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -341,7 +342,6 @@ public Date getLastModDate() { readonly = isReadOnlyLocale(locale); stamp = mintLocaleStamp(locale); pathsForFile = diskDataEntry.pathsForFile; - if (readonly) { rFile = diskDataEntry.diskFile; xmlsource = diskDataEntry.diskData; @@ -1219,7 +1219,9 @@ public STFactory(SurveyMain sm) { progress.update("reload all users"); sm.reg.getVoterInfoList(); progress.update("setup pathheader factory"); - diskDataCache = new DiskDataCache(sm.getDiskFactory(), sm.getEnglishFile()); + diskDataCache = + new DiskDataCache( + sm.getDiskFactory(), sm.getEnglishFile(), sm.getSupplementalDataInfo()); } } @@ -1978,4 +1980,8 @@ public static List removeExcludedChecks(List tests) { tests.removeIf((status) -> status.getSubtype() == Subtype.coverageLevel); return tests; } + + public Collection getFixedCandidates(CLDRLocale locale, String xpath) { + return diskDataCache.get(locale).getFixedCandidates(xpath); + } } diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyMain.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyMain.java index fe34defb3eb..84bb0c4b862 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyMain.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyMain.java @@ -2734,14 +2734,17 @@ private void showPathList(WebContext ctx) { ctx.redirect(vurl); } - private SupplementalDataInfo supplementalDataInfo = null; + private Supplier supplementalDataInfo = + Suppliers.memoize( + () -> { + final SupplementalDataInfo newSdi = + SupplementalDataInfo.getInstance(getSupplementalDirectory()); + newSdi.setAsDefaultInstance(); + return newSdi; + }); - public final synchronized SupplementalDataInfo getSupplementalDataInfo() { - if (supplementalDataInfo == null) { - supplementalDataInfo = SupplementalDataInfo.getInstance(getSupplementalDirectory()); - supplementalDataInfo.setAsDefaultInstance(); - } - return supplementalDataInfo; + public final SupplementalDataInfo getSupplementalDataInfo() { + return supplementalDataInfo.get(); } File getSupplementalDirectory() {