From a6a05ed94ad4b2912b3f68074fd74db949f7abf5 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 30 Jan 2024 03:59:27 -0600 Subject: [PATCH] CLDR-17313 split out immutable DiskData in STFactory - stuff we only want to calculate once, versus what can be expired from the cache. --- .../org/unicode/cldr/web/DiskDataCache.java | 69 +++++++++++++++++++ .../java/org/unicode/cldr/web/STFactory.java | 66 +++++++++--------- 2 files changed, 104 insertions(+), 31 deletions(-) create mode 100644 tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java new file mode 100644 index 00000000000..84ab7277ed8 --- /dev/null +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java @@ -0,0 +1,69 @@ +package org.unicode.cldr.web; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.logging.Logger; +import org.unicode.cldr.util.CLDRFile; +import org.unicode.cldr.util.CLDRLocale; +import org.unicode.cldr.util.Factory; +import org.unicode.cldr.util.PathHeader; +import org.unicode.cldr.util.XMLSource; + +/** Cache for on-disk immutable data. */ +public class DiskDataCache { + static final Logger logger = Logger.getLogger(DiskDataCache.class.getSimpleName()); + + private final Factory factory; + private final CLDRFile english; + private final PathHeader.Factory phf; + + /** this is the immutable cousin of STFactory.PerLocaleData, for the on-disk data */ + class DiskDataEntry { + final CLDRLocale locale; + final XMLSource diskData; + final CLDRFile diskFile; + final Set pathsForFile; + + public DiskDataEntry(CLDRLocale locale) { + this.locale = locale; + diskData = factory.makeSource(locale.getBaseName()).freeze(); + diskFile = factory.make(locale.getBaseName(), true).freeze(); + pathsForFile = getPathHeaderFactory().pathsForFile(diskFile); + } + } + + public DiskDataCache(Factory f, CLDRFile english) { + this.factory = f; + this.english = english; + this.phf = PathHeader.getFactory(english); + } + + public PathHeader.Factory getPathHeaderFactory() { + return phf; + } + + private LoadingCache cache = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + + @Override + public DiskDataEntry load(CLDRLocale l) throws Exception { + return new DiskDataEntry(l); + } + }); + + public DiskDataEntry get(CLDRLocale locale) { + logger.fine(() -> "Loading " + locale); + try { + return cache.get(locale); + } catch (ExecutionException e) { + SurveyLog.logException(logger, e, "Trying to construct " + locale); + SurveyMain.busted("Loading " + locale, e); + return null; /* notreached */ + } + } +} diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java index 92ce5d74cfa..60c5a12b568 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java @@ -1,6 +1,8 @@ /** */ package org.unicode.cldr.web; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -97,10 +99,6 @@ public final class PerLocaleData implements Comparable, BallotBox private final boolean readonly; /** Stamp that tracks if this locale has been modified (by a vote) */ private final MutableStamp stamp; - /** unresolved XMLSource for on-disk data. */ - private final XMLSource diskData; - /** resolved CLDRFile backed by disk data */ - private final CLDRFile diskFile; /** unresolved XMLSource backed by the DB, or null for readonly */ private final BallotBoxXMLSource dataBackedSource; /** unresolved XMLSource: == dataBackedSource, or for readonly == diskData */ @@ -109,7 +107,7 @@ public final class PerLocaleData implements Comparable, BallotBox private final CLDRFile file; /** Resolved CLDRFile backed by {@link #xmlsource} */ private final CLDRFile rFile; - /** List of all XPaths present */ + /** List of all XPaths present. Only mutated by makeSureInPathsForFile */ private Set pathsForFile; /** which XPaths had votes? */ BitSet votesSometimeThisRelease = null; @@ -327,6 +325,8 @@ public Date getLastModDate() { } } + final DiskDataCache.DiskDataEntry diskDataEntry; + /** * Constructor is called from the 'locales' cache, and in turn by STFactory.get() All parent * locales have already been initialized. @@ -341,23 +341,22 @@ public Date getLastModDate() { PerLocaleData(CLDRLocale locale) { logger.info("Load: " + locale); this.locale = locale; + diskDataEntry = diskDataCache.get(locale); + sm.xpt.loadXPaths(diskDataEntry.diskData); readonly = isReadOnlyLocale(locale); - diskData = sm.getDiskFactory().makeSource(locale.getBaseName()).freeze(); - sm.xpt.loadXPaths(diskData); - diskFile = sm.getDiskFactory().make(locale.getBaseName(), true).freeze(); - pathsForFile = phf.pathsForFile(diskFile); stamp = mintLocaleStamp(locale); if (readonly) { - rFile = diskFile; - xmlsource = diskData; + rFile = diskDataEntry.diskFile; + xmlsource = diskDataEntry.diskData; // null for readonly dataBackedSource = null; } else { xmlsource = dataBackedSource = - new BallotBoxXMLSource(diskData.cloneAsThawed(), this); + new BallotBoxXMLSource( + diskDataEntry.diskData.cloneAsThawed(), this); registerXmlSource(dataBackedSource); loadVoteValues(dataBackedSource, VoteLoadingContext.ORDINARY_LOAD_VOTES); nextStamp(); @@ -571,7 +570,7 @@ private void loadVoteValues( Set xpathSet; if (voteLoadingContext == VoteLoadingContext.VXML_GENERATION) { xpathSet = new HashSet<>(allPXDPaths()); - for (String xp : diskData) { + for (String xp : diskDataEntry.diskData) { xpathSet.add(xp); } } else { // voteLoadingContext == VoteLoadingContext.ORDINARY_LOAD_VOTES @@ -642,13 +641,13 @@ private VoteResolver getResolverInternal( r.setLocale(locale, getPathHeader(path)); // set current Trunk (baseline) value (if present) - final String currentValue = diskData.getValueAtDPath(path); - final Status currentStatus = VoteResolver.calculateStatus(diskFile, path); + final String currentValue = diskDataEntry.diskData.getValueAtDPath(path); + final Status currentStatus = VoteResolver.calculateStatus(diskDataEntry.diskFile, path); r.setBaseline(currentValue, currentStatus); r.add(currentValue); /** Note that rFile may not have all votes filled in yet as we're in startup phase */ - final CLDRFile baseFile = (rFile != null) ? rFile : diskFile; + final CLDRFile baseFile = (rFile != null) ? rFile : diskDataEntry.diskFile; r.setBaileyValue(baseFile.getBaileyValue(path, null, null)); // add each vote @@ -721,7 +720,7 @@ public Set getValues(String xpath) { } } // include the on-disk value, if not present. - String fbValue = diskData.getValueAtDPath(xpath); + String fbValue = diskDataEntry.diskData.getValueAtDPath(xpath); if (fbValue != null) { ts.add(fbValue); } @@ -789,7 +788,7 @@ public String getVoteValue(User user, String distinguishingXpath) { */ private synchronized XMLSource makeVettedSource() { BallotBoxXMLSource vxmlSource = - new BallotBoxXMLSource(diskData.cloneAsThawed(), this); + new BallotBoxXMLSource(diskDataEntry.diskData.cloneAsThawed(), this); if (!readonly) { loadVoteValues(vxmlSource, VoteLoadingContext.VXML_GENERATION); } @@ -1248,7 +1247,7 @@ public static void unimp() { /** The infamous back-pointer. */ public SurveyMain sm; - private final org.unicode.cldr.util.PathHeader.Factory phf; + private final DiskDataCache diskDataCache; /** Construct one. */ public STFactory(SurveyMain sm) { @@ -1267,7 +1266,7 @@ public STFactory(SurveyMain sm) { progress.update("reload all users"); sm.reg.getVoterInfoList(); progress.update("setup pathheader factory"); - phf = PathHeader.getFactory(sm.getEnglishFile()); + diskDataCache = new DiskDataCache(sm.getDiskFactory(), sm.getEnglishFile()); } } @@ -1876,25 +1875,26 @@ public STFactory TESTING_shutdownAndRestart() { public final PathHeader getPathHeader(String xpath) { try { - return phf.fromPath(xpath); + return getPathHeaderFactory().fromPath(xpath); } catch (Throwable t) { SurveyLog.warnOnce(logger, "PH for path " + xpath + t); return null; } } - private SurveyMenus surveyMenus = null; + private Supplier surveyMenus = + Suppliers.memoize( + () -> { + try (CLDRProgressTask progress = + sm.openProgress("STFactory: setup surveymenus")) { + progress.update("setup surveymenus"); + return new SurveyMenus(this, getPathHeaderFactory()); + } + }); - public final synchronized SurveyMenus getSurveyMenus() { - if (surveyMenus == null) { - try (CLDRProgressTask progress = sm.openProgress("STFactory: setup surveymenus")) { - progress.update("setup surveymenus"); - surveyMenus = new SurveyMenus(this, phf); - } - } - return surveyMenus; + public SurveyMenus getSurveyMenus() { + return surveyMenus.get(); } - /** * Resolving disk file, or null if none. * @@ -1905,6 +1905,10 @@ public CLDRFile getDiskFile(CLDRLocale locale) { return sm.getDiskFactory().make(locale.getBaseName(), true); } + private final PathHeader.Factory getPathHeaderFactory() { + return diskDataCache.getPathHeaderFactory(); + } + /** * Return all xpaths for this locale. uses CLDRFile iterator, etc *