Skip to content

Commit

Permalink
CLDR-17313 split out immutable DiskData in STFactory
Browse files Browse the repository at this point in the history 
- stuff we only want to calculate once, versus what can be expired from the cache.
  • Loading branch information
@srl295
srl295 committed Feb 7, 2024
1 parent 01b8b77 commit a6a05ed
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 31 deletions.
69 changes: 69 additions & 0 deletions tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package org.unicode.cldr.web;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRLocale;
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.PathHeader;
import org.unicode.cldr.util.XMLSource;

/** Cache for on-disk immutable data. */
public class DiskDataCache {
static final Logger logger = Logger.getLogger(DiskDataCache.class.getSimpleName());

private final Factory factory;
private final CLDRFile english;
private final PathHeader.Factory phf;

/** this is the immutable cousin of STFactory.PerLocaleData, for the on-disk data */
class DiskDataEntry {
final CLDRLocale locale;
final XMLSource diskData;
final CLDRFile diskFile;
final Set<String> pathsForFile;

public DiskDataEntry(CLDRLocale locale) {
this.locale = locale;
diskData = factory.makeSource(locale.getBaseName()).freeze();
diskFile = factory.make(locale.getBaseName(), true).freeze();
pathsForFile = getPathHeaderFactory().pathsForFile(diskFile);
}
}

public DiskDataCache(Factory f, CLDRFile english) {
this.factory = f;
this.english = english;
this.phf = PathHeader.getFactory(english);
}

public PathHeader.Factory getPathHeaderFactory() {
return phf;
}

private LoadingCache<CLDRLocale, DiskDataEntry> cache =
CacheBuilder.newBuilder()
.build(
new CacheLoader<CLDRLocale, DiskDataEntry>() {

@Override
public DiskDataEntry load(CLDRLocale l) throws Exception {
return new DiskDataEntry(l);
}
});

public DiskDataEntry get(CLDRLocale locale) {
logger.fine(() -> "Loading " + locale);
try {
return cache.get(locale);
} catch (ExecutionException e) {
SurveyLog.logException(logger, e, "Trying to construct " + locale);
SurveyMain.busted("Loading " + locale, e);
return null; /* notreached */
}
}
}
66 changes: 35 additions & 31 deletions tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/** */
package org.unicode.cldr.web;

import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
Expand Down Expand Up @@ -97,10 +99,6 @@ public final class PerLocaleData implements Comparable<PerLocaleData>, BallotBox
private final boolean readonly;
/** Stamp that tracks if this locale has been modified (by a vote) */
private final MutableStamp stamp;
/** unresolved XMLSource for on-disk data. */
private final XMLSource diskData;
/** resolved CLDRFile backed by disk data */
private final CLDRFile diskFile;
/** unresolved XMLSource backed by the DB, or null for readonly */
private final BallotBoxXMLSource<User> dataBackedSource;
/** unresolved XMLSource: == dataBackedSource, or for readonly == diskData */
Expand All @@ -109,7 +107,7 @@ public final class PerLocaleData implements Comparable<PerLocaleData>, BallotBox
private final CLDRFile file;
/** Resolved CLDRFile backed by {@link #xmlsource} */
private final CLDRFile rFile;
/** List of all XPaths present */
/** List of all XPaths present. Only mutated by makeSureInPathsForFile */
private Set<String> pathsForFile;
/** which XPaths had votes? */
BitSet votesSometimeThisRelease = null;
Expand Down Expand Up @@ -327,6 +325,8 @@ public Date getLastModDate() {
}
}

final DiskDataCache.DiskDataEntry diskDataEntry;

/**
* Constructor is called from the 'locales' cache, and in turn by STFactory.get() All parent
* locales have already been initialized.
Expand All @@ -341,23 +341,22 @@ public Date getLastModDate() {
PerLocaleData(CLDRLocale locale) {
logger.info("Load: " + locale);
this.locale = locale;
diskDataEntry = diskDataCache.get(locale);
sm.xpt.loadXPaths(diskDataEntry.diskData);
readonly = isReadOnlyLocale(locale);
diskData = sm.getDiskFactory().makeSource(locale.getBaseName()).freeze();
sm.xpt.loadXPaths(diskData);
diskFile = sm.getDiskFactory().make(locale.getBaseName(), true).freeze();
pathsForFile = phf.pathsForFile(diskFile);
stamp = mintLocaleStamp(locale);

if (readonly) {
rFile = diskFile;
xmlsource = diskData;
rFile = diskDataEntry.diskFile;
xmlsource = diskDataEntry.diskData;

// null for readonly
dataBackedSource = null;
} else {
xmlsource =
dataBackedSource =
new BallotBoxXMLSource<User>(diskData.cloneAsThawed(), this);
new BallotBoxXMLSource<User>(
diskDataEntry.diskData.cloneAsThawed(), this);
registerXmlSource(dataBackedSource);
loadVoteValues(dataBackedSource, VoteLoadingContext.ORDINARY_LOAD_VOTES);
nextStamp();
Expand Down Expand Up @@ -571,7 +570,7 @@ private void loadVoteValues(
Set<String> xpathSet;
if (voteLoadingContext == VoteLoadingContext.VXML_GENERATION) {
xpathSet = new HashSet<>(allPXDPaths());
for (String xp : diskData) {
for (String xp : diskDataEntry.diskData) {
xpathSet.add(xp);
}
} else { // voteLoadingContext == VoteLoadingContext.ORDINARY_LOAD_VOTES
Expand Down Expand Up @@ -642,13 +641,13 @@ private VoteResolver<String> getResolverInternal(
r.setLocale(locale, getPathHeader(path));

// set current Trunk (baseline) value (if present)
final String currentValue = diskData.getValueAtDPath(path);
final Status currentStatus = VoteResolver.calculateStatus(diskFile, path);
final String currentValue = diskDataEntry.diskData.getValueAtDPath(path);
final Status currentStatus = VoteResolver.calculateStatus(diskDataEntry.diskFile, path);
r.setBaseline(currentValue, currentStatus);
r.add(currentValue);

/** Note that rFile may not have all votes filled in yet as we're in startup phase */
final CLDRFile baseFile = (rFile != null) ? rFile : diskFile;
final CLDRFile baseFile = (rFile != null) ? rFile : diskDataEntry.diskFile;
r.setBaileyValue(baseFile.getBaileyValue(path, null, null));

// add each vote
Expand Down Expand Up @@ -721,7 +720,7 @@ public Set<String> getValues(String xpath) {
}
}
// include the on-disk value, if not present.
String fbValue = diskData.getValueAtDPath(xpath);
String fbValue = diskDataEntry.diskData.getValueAtDPath(xpath);
if (fbValue != null) {
ts.add(fbValue);
}
Expand Down Expand Up @@ -789,7 +788,7 @@ public String getVoteValue(User user, String distinguishingXpath) {
*/
private synchronized XMLSource makeVettedSource() {
BallotBoxXMLSource<User> vxmlSource =
new BallotBoxXMLSource<User>(diskData.cloneAsThawed(), this);
new BallotBoxXMLSource<User>(diskDataEntry.diskData.cloneAsThawed(), this);
if (!readonly) {
loadVoteValues(vxmlSource, VoteLoadingContext.VXML_GENERATION);
}
Expand Down Expand Up @@ -1248,7 +1247,7 @@ public static void unimp() {
/** The infamous back-pointer. */
public SurveyMain sm;

private final org.unicode.cldr.util.PathHeader.Factory phf;
private final DiskDataCache diskDataCache;

/** Construct one. */
public STFactory(SurveyMain sm) {
Expand All @@ -1267,7 +1266,7 @@ public STFactory(SurveyMain sm) {
progress.update("reload all users");
sm.reg.getVoterInfoList();
progress.update("setup pathheader factory");
phf = PathHeader.getFactory(sm.getEnglishFile());
diskDataCache = new DiskDataCache(sm.getDiskFactory(), sm.getEnglishFile());
}
}

Expand Down Expand Up @@ -1876,25 +1875,26 @@ public STFactory TESTING_shutdownAndRestart() {

public final PathHeader getPathHeader(String xpath) {
try {
return phf.fromPath(xpath);
return getPathHeaderFactory().fromPath(xpath);
} catch (Throwable t) {
SurveyLog.warnOnce(logger, "PH for path " + xpath + t);
return null;
}
}

private SurveyMenus surveyMenus = null;
private Supplier<SurveyMenus> surveyMenus =
Suppliers.memoize(
() -> {
try (CLDRProgressTask progress =
sm.openProgress("STFactory: setup surveymenus")) {
progress.update("setup surveymenus");
return new SurveyMenus(this, getPathHeaderFactory());
}
});

public final synchronized SurveyMenus getSurveyMenus() {
if (surveyMenus == null) {
try (CLDRProgressTask progress = sm.openProgress("STFactory: setup surveymenus")) {
progress.update("setup surveymenus");
surveyMenus = new SurveyMenus(this, phf);
}
}
return surveyMenus;
public SurveyMenus getSurveyMenus() {
return surveyMenus.get();
}

/**
* Resolving disk file, or null if none.
*
Expand All @@ -1905,6 +1905,10 @@ public CLDRFile getDiskFile(CLDRLocale locale) {
return sm.getDiskFactory().make(locale.getBaseName(), true);
}

private final PathHeader.Factory getPathHeaderFactory() {
return diskDataCache.getPathHeaderFactory();
}

/**
* Return all xpaths for this locale. uses CLDRFile iterator, etc
*
Expand Down

0 comments on commit a6a05ed

Please sign in to comment.