Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CLDR-17313 split out immutable DiskData in STFactory #3476

Merged
merged 2 commits into from
Feb 7, 2024
Merged

CLDR-17313 split out immutable DiskData in STFactory #3476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1fc7753
CLDR-17313 split out immutable DiskData in STFactory
srl295 Jan 30, 2024
97f5daf
CLDR-17313 pathsForFile fix
srl295 Feb 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions tools/cldr-apps/src/main/java/org/unicode/cldr/web/DiskDataCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package org.unicode.cldr.web;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRLocale;
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.PathHeader;
import org.unicode.cldr.util.XMLSource;

/** Cache for on-disk immutable data. */
public class DiskDataCache {
static final Logger logger = Logger.getLogger(DiskDataCache.class.getSimpleName());

private final Factory factory;
private final CLDRFile english;
private final PathHeader.Factory phf;

/** this is the immutable cousin of STFactory.PerLocaleData, for the on-disk data */
class DiskDataEntry {
final CLDRLocale locale;
final XMLSource diskData;
final CLDRFile diskFile;
final Set<String> pathsForFile;

public DiskDataEntry(CLDRLocale locale) {
this.locale = locale;
diskData = factory.makeSource(locale.getBaseName()).freeze();
diskFile = factory.make(locale.getBaseName(), true).freeze();
pathsForFile = getPathHeaderFactory().pathsForFile(diskFile);
}
}

public DiskDataCache(Factory f, CLDRFile english) {
this.factory = f;
this.english = english;
this.phf = PathHeader.getFactory(english);
}

public PathHeader.Factory getPathHeaderFactory() {
return phf;
}

private LoadingCache<CLDRLocale, DiskDataEntry> cache =
CacheBuilder.newBuilder()
.build(
new CacheLoader<CLDRLocale, DiskDataEntry>() {

@Override
public DiskDataEntry load(CLDRLocale l) throws Exception {
return new DiskDataEntry(l);
}
});

public DiskDataEntry get(CLDRLocale locale) {
logger.fine(() -> "Loading " + locale);
try {
return cache.get(locale);
} catch (ExecutionException e) {
SurveyLog.logException(logger, e, "Trying to construct " + locale);
SurveyMain.busted("Loading " + locale, e);
return null; /* notreached */
}
}
}
67 changes: 36 additions & 31 deletions tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/** */
package org.unicode.cldr.web;

import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
Expand Down Expand Up @@ -97,10 +99,6 @@ public final class PerLocaleData implements Comparable<PerLocaleData>, BallotBox
private final boolean readonly;
/** Stamp that tracks if this locale has been modified (by a vote) */
private final MutableStamp stamp;
/** unresolved XMLSource for on-disk data. */
private final XMLSource diskData;
/** resolved CLDRFile backed by disk data */
private final CLDRFile diskFile;
/** unresolved XMLSource backed by the DB, or null for readonly */
private final BallotBoxXMLSource<User> dataBackedSource;
/** unresolved XMLSource: == dataBackedSource, or for readonly == diskData */
Expand All @@ -109,7 +107,7 @@ public final class PerLocaleData implements Comparable<PerLocaleData>, BallotBox
private final CLDRFile file;
/** Resolved CLDRFile backed by {@link #xmlsource} */
private final CLDRFile rFile;
/** List of all XPaths present */
/** List of all XPaths present. Only mutated by makeSureInPathsForFile */
private Set<String> pathsForFile;
/** which XPaths had votes? */
BitSet votesSometimeThisRelease = null;
Expand Down Expand Up @@ -327,6 +325,8 @@ public Date getLastModDate() {
}
}

final DiskDataCache.DiskDataEntry diskDataEntry;

/**
* Constructor is called from the 'locales' cache, and in turn by STFactory.get() All parent
* locales have already been initialized.
Expand All @@ -341,23 +341,23 @@ public Date getLastModDate() {
PerLocaleData(CLDRLocale locale) {
logger.info("Load: " + locale);
this.locale = locale;
diskDataEntry = diskDataCache.get(locale);
sm.xpt.loadXPaths(diskDataEntry.diskData);
readonly = isReadOnlyLocale(locale);
diskData = sm.getDiskFactory().makeSource(locale.getBaseName()).freeze();
sm.xpt.loadXPaths(diskData);
diskFile = sm.getDiskFactory().make(locale.getBaseName(), true).freeze();
pathsForFile = phf.pathsForFile(diskFile);
stamp = mintLocaleStamp(locale);
pathsForFile = diskDataEntry.pathsForFile;

if (readonly) {
rFile = diskFile;
xmlsource = diskData;
rFile = diskDataEntry.diskFile;
xmlsource = diskDataEntry.diskData;

// null for readonly
dataBackedSource = null;
} else {
xmlsource =
dataBackedSource =
new BallotBoxXMLSource<User>(diskData.cloneAsThawed(), this);
new BallotBoxXMLSource<User>(
diskDataEntry.diskData.cloneAsThawed(), this);
registerXmlSource(dataBackedSource);
loadVoteValues(dataBackedSource, VoteLoadingContext.ORDINARY_LOAD_VOTES);
nextStamp();
Expand Down Expand Up @@ -571,7 +571,7 @@ private void loadVoteValues(
Set<String> xpathSet;
if (voteLoadingContext == VoteLoadingContext.VXML_GENERATION) {
xpathSet = new HashSet<>(allPXDPaths());
for (String xp : diskData) {
for (String xp : diskDataEntry.diskData) {
xpathSet.add(xp);
}
} else { // voteLoadingContext == VoteLoadingContext.ORDINARY_LOAD_VOTES
Expand Down Expand Up @@ -642,13 +642,13 @@ private VoteResolver<String> getResolverInternal(
r.setLocale(locale, getPathHeader(path));

// set current Trunk (baseline) value (if present)
final String currentValue = diskData.getValueAtDPath(path);
final Status currentStatus = VoteResolver.calculateStatus(diskFile, path);
final String currentValue = diskDataEntry.diskData.getValueAtDPath(path);
final Status currentStatus = VoteResolver.calculateStatus(diskDataEntry.diskFile, path);
r.setBaseline(currentValue, currentStatus);
r.add(currentValue);

/** Note that rFile may not have all votes filled in yet as we're in startup phase */
final CLDRFile baseFile = (rFile != null) ? rFile : diskFile;
final CLDRFile baseFile = (rFile != null) ? rFile : diskDataEntry.diskFile;
r.setBaileyValue(baseFile.getBaileyValue(path, null, null));

// add each vote
Expand Down Expand Up @@ -721,7 +721,7 @@ public Set<String> getValues(String xpath) {
}
}
// include the on-disk value, if not present.
String fbValue = diskData.getValueAtDPath(xpath);
String fbValue = diskDataEntry.diskData.getValueAtDPath(xpath);
if (fbValue != null) {
ts.add(fbValue);
}
Expand Down Expand Up @@ -789,7 +789,7 @@ public String getVoteValue(User user, String distinguishingXpath) {
*/
private synchronized XMLSource makeVettedSource() {
BallotBoxXMLSource<User> vxmlSource =
new BallotBoxXMLSource<User>(diskData.cloneAsThawed(), this);
new BallotBoxXMLSource<User>(diskDataEntry.diskData.cloneAsThawed(), this);
if (!readonly) {
loadVoteValues(vxmlSource, VoteLoadingContext.VXML_GENERATION);
}
Expand Down Expand Up @@ -1248,7 +1248,7 @@ public static void unimp() {
/** The infamous back-pointer. */
public SurveyMain sm;

private final org.unicode.cldr.util.PathHeader.Factory phf;
private final DiskDataCache diskDataCache;

/** Construct one. */
public STFactory(SurveyMain sm) {
Expand All @@ -1267,7 +1267,7 @@ public STFactory(SurveyMain sm) {
progress.update("reload all users");
sm.reg.getVoterInfoList();
progress.update("setup pathheader factory");
phf = PathHeader.getFactory(sm.getEnglishFile());
diskDataCache = new DiskDataCache(sm.getDiskFactory(), sm.getEnglishFile());
}
}

Expand Down Expand Up @@ -1876,25 +1876,26 @@ public STFactory TESTING_shutdownAndRestart() {

public final PathHeader getPathHeader(String xpath) {
try {
return phf.fromPath(xpath);
return getPathHeaderFactory().fromPath(xpath);
} catch (Throwable t) {
SurveyLog.warnOnce(logger, "PH for path " + xpath + t);
return null;
}
}

private SurveyMenus surveyMenus = null;
private Supplier<SurveyMenus> surveyMenus =
Suppliers.memoize(
() -> {
try (CLDRProgressTask progress =
sm.openProgress("STFactory: setup surveymenus")) {
progress.update("setup surveymenus");
return new SurveyMenus(this, getPathHeaderFactory());
}
});

public final synchronized SurveyMenus getSurveyMenus() {
if (surveyMenus == null) {
try (CLDRProgressTask progress = sm.openProgress("STFactory: setup surveymenus")) {
progress.update("setup surveymenus");
surveyMenus = new SurveyMenus(this, phf);
}
}
return surveyMenus;
public SurveyMenus getSurveyMenus() {
return surveyMenus.get();
}

/**
* Resolving disk file, or null if none.
*
Expand All @@ -1905,6 +1906,10 @@ public CLDRFile getDiskFile(CLDRLocale locale) {
return sm.getDiskFactory().make(locale.getBaseName(), true);
}

private final PathHeader.Factory getPathHeaderFactory() {
return diskDataCache.getPathHeaderFactory();
}

/**
* Return all xpaths for this locale. uses CLDRFile iterator, etc
*
Expand Down
Loading