CLDR-7646 use TestCache everywhere

tools/cldr-apps/src/main/java/org/unicode/cldr/web/DataPage.java

@@ -307,30 +307,19 @@ public String getPClass() {
              */
             private boolean setTests(List<CheckStatus> testList) {
                 tests = ImmutableList.copyOf(testList);
-                // only consider non-example tests as notable.
                 boolean weHaveTests = false;
                 int errorCount = 0;
                 int warningCount = 0;
-                for (CheckStatus status : tests) {
-                    if (!status.getType().equals(CheckStatus.exampleType)) {
-                        // skip codefallback exemplar complaints (i.e. 'JPY'
-                        // isn't in exemplars).. they'll show up in missing
-                        if (DEBUG)
-                            System.err.println(
-                                    "err: "
-                                            + status.getMessage()
-                                            + ", test: "
-                                            + status.getClass()
-                                            + ", cause: "
-                                            + status.getCause()
-                                            + " on "
-                                            + xpath);
-                        weHaveTests = true;
-                        if (status.getType().equals(CheckStatus.errorType)) {
-                            errorCount++;
-                        } else if (status.getType().equals(CheckStatus.warningType)) {
-                            warningCount++;
-                        }
+                for (final CheckStatus status : tests) {
+                    logger.finest(() -> status + " on " + xpath);
+                    if (status.getType() == CheckStatus.exampleType) {
+                        continue; // does not count as an error or warning but included in payload
+                    }
+                    weHaveTests = true;
+                    if (status.getType().equals(CheckStatus.errorType)) {
+                        errorCount++;
+                    } else if (status.getType().equals(CheckStatus.warningType)) {
+                        warningCount++;
                     }
                 }
                 if (weHaveTests) {
@@ -759,6 +748,7 @@ private void setShimTests(String base_xpath_string, TestResultBundle checkCldr)
                 CandidateItem shimItem = new CandidateItem(null);
                 List<CheckStatus> iTests = new ArrayList<>();
                 checkCldr.check(base_xpath_string, iTests, null);
+                STFactory.removeExcludedChecks(iTests);
                 if (!iTests.isEmpty()) {
                     // Got a bite.
                     if (shimItem.setTests(iTests)) {
@@ -875,6 +865,7 @@ private void updateInheritedValue(CLDRFile ourSrc, TestResultBundle checkCldr) {
                 List<CheckStatus> iTests = new ArrayList<>();
 
                 checkCldr.check(xpath, iTests, inheritedValue);
+                STFactory.removeExcludedChecks(iTests);
 
                 if (TRACE_TIME) {
                     System.err.println("@@6:" + (System.currentTimeMillis() - lastTime));
@@ -1935,6 +1926,7 @@ private void populateFromThisXpath(
         List<CheckStatus> examplesResult = new ArrayList<>();
         if (checkCldr != null) {
             checkCldr.check(xpath, checkCldrResult, isExtraPath ? null : ourValue);
+            STFactory.removeExcludedChecks(checkCldrResult);
             checkCldr.getExamples(xpath, isExtraPath ? null : ourValue, examplesResult);
         }
         if (ourValue != null && ourValue.length() > 0) {
@@ -1966,6 +1958,7 @@ private void populateFromThisXpathAddItemsForVotes(
             if (avalue != null && checkCldr != null) {
                 List<CheckStatus> item2Result = new ArrayList<>();
                 checkCldr.check(xpath, item2Result, avalue);
+                STFactory.removeExcludedChecks(item2Result);
                 if (!item2Result.isEmpty()) {
                     item2.setTests(item2Result);
                 }

tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java

@@ -31,8 +31,9 @@
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 import org.unicode.cldr.test.CheckCLDR;
+import org.unicode.cldr.test.CheckCLDR.CheckStatus;
+import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype;
 import org.unicode.cldr.test.TestCache;
-import org.unicode.cldr.test.TestCache.TestResultBundle;
 import org.unicode.cldr.util.CLDRConfig;
 import org.unicode.cldr.util.CLDRFile;
 import org.unicode.cldr.util.CLDRFile.DraftStatus;
@@ -357,9 +358,9 @@ public Date getLastModDate() {
                 xmlsource =
                         dataBackedSource =
                                 new BallotBoxXMLSource<User>(diskData.cloneAsThawed(), this);
+                registerXmlSource(dataBackedSource);
                 loadVoteValues(dataBackedSource, VoteLoadingContext.ORDINARY_LOAD_VOTES);
                 nextStamp();
-                dataBackedSource.addListener(gTestCache);
                 XMLSource resolvedXmlsource = makeResolvingSource();
                 rFile =
                         new CLDRFile(resolvedXmlsource)
@@ -379,7 +380,7 @@ private XMLSource makeResolvingSource() {
                                     + sourceList.stream()
                                             .map(l -> l.getLocaleID())
                                             .collect(Collectors.joining("»")));
-            return new XMLSource.ResolvingSource(sourceList);
+            return registerXmlSource(new XMLSource.ResolvingSource(sourceList));
         }
 
         void addXMLSources(List<XMLSource> sourceList) {
@@ -886,6 +887,7 @@ public void voteForValueWithType(
             }
 
             String oldVal = dataBackedSource.getValueAtDPath(distinguishingXpath);
+            String oldFullPath = dataBackedSource.getFullPathAtDPath(distinguishingXpath);
 
             // sanity check, should have been caught before
             if (readonly) {
@@ -912,7 +914,8 @@ public void voteForValueWithType(
             }
 
             String newVal = dataBackedSource.getValueAtDPath(distinguishingXpath);
-            if (newVal != null && !newVal.equals(oldVal)) {
+            String newFullPath = dataBackedSource.getFullPathAtDPath(distinguishingXpath);
+            if (newVal != null && (!newVal.equals(oldVal) || !oldFullPath.equals(newFullPath))) {
                 dataBackedSource.notifyListeners(distinguishingXpath);
             }
         }
@@ -1164,12 +1167,6 @@ public VoteType getUserVoteType(User myUser, String somePath) {
             return voteType;
         }
 
-        public TestResultBundle getTestResultData(CheckCLDR.Options options) {
-            synchronized (gTestCache) {
-                return gTestCache.getBundle(options);
-            }
-        }
-
         public Set<String> getPathsForFile() {
             return pathsForFile;
         }
@@ -1239,11 +1236,6 @@ public static void unimp() {
 
     boolean dbIsSetup = false;
 
-    /** Test cache against (this) */
-    TestCache gTestCache = new TestCache();
-    /** Test cache against disk. For rejecting items. */
-    TestCache gDiskTestCache = new TestCache();
-
     /** The infamous back-pointer. */
     public SurveyMain sm;
 
@@ -1263,10 +1255,6 @@ public STFactory(SurveyMain sm) {
                 throw new NullPointerException("getSupplementalDirectory() == null!");
             }
 
-            progress.update("setup test cache");
-            gTestCache.setFactory(this, "(?!.*(CheckCoverage).*).*");
-            progress.update("setup disk test cache");
-            gDiskTestCache.setFactory(sm.getDiskFactory(), "(?!.*(CheckCoverage).*).*");
             progress.update("reload all users");
             sm.reg.getVoterInfoList();
             progress.update("setup pathheader factory");
@@ -1285,15 +1273,7 @@ public String toString() {
                 good++;
             }
         }
-        sb.append(
-                good
-                        + "/"
-                        + locales.size()
-                        + " locales. TestCache:"
-                        + gTestCache
-                        + ", diskTestCache:"
-                        + gDiskTestCache
-                        + "}");
+        sb.append(good + "/" + locales.size() + " locales. }");
         return sb.toString();
     }
 
@@ -1386,7 +1366,7 @@ private PerLocaleData get(String locale) {
     }
 
     public TestCache.TestResultBundle getTestResult(CLDRLocale loc, CheckCLDR.Options options) {
-        return get(loc).getTestResultData(options);
+        return getTestCache().getBundle(options);
     }
 
     /*
@@ -2054,4 +2034,10 @@ public boolean isVisibleInSurveyTool(CLDRLocale l, String dPath) {
 
         return true; // OK.
     }
+
+    /** remove tests excluded by SurveyTool */
+    public static List<CheckStatus> removeExcludedChecks(List<CheckStatus> tests) {
+        tests.removeIf((status) -> status.getSubtype() == Subtype.coverageLevel);
+        return tests;
+    }
 }

tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyMain.java

@@ -394,6 +394,7 @@ public final void init(final ServletConfig config) throws ServletException {
             SurveyThreadManager.getExecutorService().submit(this::doStartup);
             klm = new KeepLoggedInManager(null);
         } catch (Throwable t) {
+            t.printStackTrace();
             SurveyLog.logException(logger, t, "Initializing SurveyTool");
             SurveyMain.busted("Error initializing SurveyTool.", t);
         }

tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyTool.java

@@ -11,8 +11,8 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.json.JSONException;
+import org.unicode.cldr.util.CLDRURLS;
 import org.unicode.cldr.util.CldrUtility;
-import org.unicode.cldr.util.VettingViewer;
 
 public class SurveyTool extends HttpServlet {
     static final Logger logger = SurveyLog.forClass(SurveyTool.class);
@@ -95,7 +95,7 @@ private void serveWaitingPage(HttpServletRequest request, PrintWriter out, Surve
         out.write("<meta http-equiv='Content-Type' content='text/html; charset=UTF-8'>\n");
         out.write("<title>CLDR Survey Tool | Starting</title>\n");
         includeCss(request, out);
-        out.write(VettingViewer.getHeaderStyles() + "\n");
+        out.write(CLDRURLS.getVettingViewerHeaderStyles() + "\n");
         try {
             includeJavaScript(request, out);
         } catch (JSONException e) {
@@ -173,7 +173,7 @@ private void serveRunnningNormallyPage(
         out.write("<meta name='gigabot' content='nofollow'>\n");
         out.write(FAVICON_LINK);
         includeCss(request, out);
-        out.write(VettingViewer.getHeaderStyles() + "\n");
+        out.write(CLDRURLS.getVettingViewerHeaderStyles() + "\n");
         try {
             includeJavaScript(request, out);
         } catch (JSONException e) {

tools/cldr-code/src/main/java/org/unicode/cldr/test/CheckCLDR.java

@@ -447,6 +447,15 @@ public Options(Options options2) {
             this.locale = options2.locale;
         }
 
+        public Options(CLDRLocale locale) {
+            this.locale = locale;
+            options = new String[Option.values().length];
+            set(Option.locale, locale.getBaseName());
+            StringBuilder sb = new StringBuilder();
+            sb.append(locale.getBaseName()).append('/');
+            key = sb.toString().intern();
+        }
+
         public Options(
                 CLDRLocale locale,
                 CheckCLDR.Phase testPhase,

tools/cldr-code/src/main/java/org/unicode/cldr/test/CheckCoverage.java

@@ -53,15 +53,15 @@ public CheckCoverage(Factory factory) {
     public CheckCLDR handleCheck(
             String path, String fullPath, String value, Options options, List<CheckStatus> result) {
 
-        if (!accept(result)) return this;
-
         CLDRFile resolvedCldrFileToCheck = getResolvedCldrFileToCheck();
 
         // skip if we are not the winning path
         if (!resolvedCldrFileToCheck.isWinningPath(path)) {
             return this;
         }
 
+        if (!accept(result)) return this;
+
         Status status = new Status();
 
         /*

tools/cldr-code/src/main/java/org/unicode/cldr/test/ConsoleCheckCLDR.java

@@ -479,8 +479,8 @@ public static void main(String[] args) throws Throwable {
         Factory cldrFactory =
                 SimpleFactory.make(sourceDirectories, factoryFilter)
                         .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY));
-        TestCache testCache = new TestCache(); // for parallelism
-        testCache.setFactory(cldrFactory, checkFilter);
+        final TestCache testCache = cldrFactory.getTestCache();
+        testCache.setNameMatcher(checkFilter);
 
         {
             // we create an extraneous CompoundCheckCLDR here just to check the filters

tools/cldr-code/src/main/java/org/unicode/cldr/test/TestCache.java

@@ -7,6 +7,7 @@
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Logger;
 import org.unicode.cldr.test.CheckCLDR.CheckStatus;
 import org.unicode.cldr.test.CheckCLDR.Options;
 import org.unicode.cldr.util.CLDRConfig;
@@ -25,6 +26,8 @@
  * @see XMLSource#addListener(org.unicode.cldr.util.XMLSource.Listener)
  */
 public class TestCache implements XMLSource.Listener {
+    private static final Logger logger = Logger.getLogger(TestCache.class.getSimpleName());
+
     public class TestResultBundle {
         private final CheckCLDR cc = CheckCLDR.getCheckAll(getFactory(), nameMatcher);
         final CLDRFile file;
@@ -88,18 +91,15 @@ public List<CheckStatus> getPossibleProblems() {
                     .softValues()
                     .build();
 
-    private Factory factory = null;
+    private final Factory factory;
 
-    private String nameMatcher = null;
+    private String nameMatcher = ".*";
 
     /** Get the bundle for this test */
     public TestResultBundle getBundle(CheckCLDR.Options options) {
         TestResultBundle b = testResultCache.getIfPresent(options);
-        if (DEBUG) {
-            if (b != null) {
-                System.err.println("Bundle refvalid: " + options + " -> " + (b != null));
-            }
-            System.err.println("Bundle " + b + " for " + options + " in " + this.toString());
+        if (b != null) {
+            logger.finest(() -> this + " Bundle refvalid: " + options);
         }
         if (b == null) {
             // ElapsedTimer et = new ElapsedTimer("New test bundle " + locale + " opt " + options);
@@ -114,19 +114,17 @@ protected Factory getFactory() {
         return factory;
     }
 
-    /**
-     * Set up the basic info needed for tests
-     *
-     * @param factory
-     * @param nameMatcher
-     * @param displayInformation
-     */
-    public void setFactory(Factory factory, String nameMatcher) {
-        if (this.factory != null) {
-            throw new InternalError("setFactory() can only be called once.");
-        }
-        this.factory = factory;
+    /** construct a new TestCache with this factory. Intended for use from within Factory. */
+    public TestCache(Factory f) {
+        this.factory = f;
+        logger.fine(() -> toString() + " - init(" + f + ")");
+    }
+
+    /** Change which checks are run. Invalidates all caches. */
+    public void setNameMatcher(String nameMatcher) {
+        logger.finest(() -> toString() + " - setNameMatcher(" + nameMatcher + ")");
         this.nameMatcher = nameMatcher;
+        invalidateAllCached();
     }
 
     /**
@@ -141,6 +139,8 @@ public String toString() {
                 "{"
                         + this.getClass().getSimpleName()
                         + super.toString()
+                        + " F="
+                        + factory.getClass().getSimpleName()
                         + " Size: "
                         + testResultCache.size()
                         + " (");
@@ -183,9 +183,7 @@ public void valueChanged(String xpath, XMLSource source) {
      * @param locale the CLDRLocale
      */
     private void valueChangedInvalidateRecursively(String xpath, final CLDRLocale locale) {
-        if (DEBUG) {
-            System.err.println("BundDelLoc " + locale + " @ " + xpath);
-        }
+        logger.finer(() -> "BundDelLoc " + locale + " @ " + xpath);
         /*
          * Call self recursively for all sub-locales
          */
@@ -312,8 +310,9 @@ private static void updateExampleGeneratorCache(String xpath, CLDRLocale locale)
         }
     }
 
-    /** For tests. Invalidate cache. */
+    /** Public for tests. Invalidate cache. */
     public void invalidateAllCached() {
+        logger.fine(() -> toString() + " - invalidateAllCached()");
         testResultCache.invalidateAll();
         exampleGeneratorCache.invalidateAll();
     }

tools/cldr-code/src/main/java/org/unicode/cldr/tool/CldrVersion.java

@@ -14,7 +14,6 @@
 import java.util.TreeSet;
 import org.unicode.cldr.util.CLDRFile;
 import org.unicode.cldr.util.CLDRPaths;
-import org.unicode.cldr.util.SupplementalDataInfo;
 
 /**
  * Enums that should exactly match what is in cldr-archive; eg, v2_0_1 means that there is a folder
@@ -140,9 +139,8 @@ private CldrVersion() {
         } else {
             dotName = oldName;
             baseDirectory = CLDRPaths.BASE_DIRECTORY;
-            SupplementalDataInfo sdi = SupplementalDataInfo.getInstance();
-            versionInfo =
-                    "baseline".equals(oldName) ? sdi.getCldrVersion() : VersionInfo.getInstance(0);
+            final VersionInfo cldrVersion = VersionInfo.getInstance(CLDRFile.GEN_VERSION);
+            versionInfo = "baseline".equals(oldName) ? cldrVersion : VersionInfo.getInstance(0);
         }
     }