Current Status: {{ status }}
++ +
+ +
+
Invoked by pasting a url like this into a browser: - * http://localhost:8080/cldr-apps/admin-OutputAllFiles.jsp?vap=... - *
This function was started using code moved here from admin-OutputAllFiles.jsp. - * Reference: CLDR-12016 and CLDR-11877 - *
TODO: link to gear menu and use JavaScript for a front-end. + * .../cldr-apps/admin-OutputAllFiles.jsp?vap=... + *
TODO: remove this method, and admin-OutputAllFiles.jsp, and other obsolete code once + * the new implementation with GenerateVxml.java is well tested + *
Reference: https://unicode-org.atlassian.net/browse/CLDR-14913
*/
public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer out) {
String vap = request.getParameter("vap");
@@ -120,10 +109,33 @@ public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer ou
out.write("verify=true/false
\n");
return;
}
+ generateVxml(null, out, outputFiles, removeEmpty, verifyConsistent);
+ } catch (Exception e) {
+ System.err.println("Exception in outputAndVerifyAllFiles: " + e);
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Generate VXML
+ *
+ *
Called indirectly by modern api/GenerateVxml, as well as by legacy outputAndVerifyAllFiles + * + * @param vxmlGenerator the VxmlGenerator, or null if called by outputAndVerifyAllFiles + * @param out + * @param outputFiles + * @param removeEmpty + * @param verifyConsistent + */ + public static void generateVxml( + VxmlGenerator vxmlGenerator, + Writer out, + boolean outputFiles, + boolean removeEmpty, + boolean verifyConsistent) { + try { /* * Sync on OutputFileManager.class here prevents re-entrance if invoked repeatedly before completion. - * Performance problem if run while Survey Tool has multiple users/requests? - * Completion of http request/response may take over ten minutes! TODO: use ajax. */ synchronized (OutputFileManager.class) { SurveyMain sm = CookieSession.sm; @@ -136,9 +148,9 @@ public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer ou out.write("Directory creation for vetting data failed."); return; } - out.write("
Created new directory: " + vetdataDir.toString() + "
"); + out.write("Created new directory: " + vetdataDir + "
"); - if (outputFiles && !ofm.outputAllFiles(out, vetdataDir)) { + if (outputFiles && !ofm.outputAllFiles(vxmlGenerator, out, vetdataDir)) { out.write("File output failed."); return; } @@ -148,18 +160,18 @@ public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer ou } File vxmlDir = null; if (removeEmpty || verifyConsistent) { - vxmlDir = new File(vetdataDir.toString() + "/" + Kind.vxml.name()); + vxmlDir = new File(vetdataDir + "/" + Kind.vxml.name()); } if (removeEmpty) { - ofm.removeEmptyFiles(out, vxmlDir); + ofm.removeEmptyFiles(vxmlDir); } if (verifyConsistent) { - ofm.verifyAllFiles(out, vxmlDir); + ofm.verifyAllFiles(vxmlGenerator, out, vxmlDir); } } - System.out.println("outputAndVerifyAllFiles finished"); + System.out.println("reallyOutputAndVerifyAllFiles finished"); } catch (Exception e) { - System.err.println("Exception in outputAndVerifyAllFiles: " + e); + System.err.println("Exception in reallyOutputAndVerifyAllFiles: " + e); e.printStackTrace(); } } @@ -193,7 +205,6 @@ private static File createNewManualVetdataDir(File vetdataDir) { * Copy the DTD file from trunk into subfolders of the given vetdata folder ("auto" or "manual") * * @param vetdataDir the File for the vetdata directory - * @param common the name of the "common" folder * @return true for success, or false for failure *The dtd is required for removeEmptyFiles when it calls XMLFileReader.loadPathValues. * The xml files all have something like: This function was first created using code moved here from admin-OutputAllFiles.jsp. - * Reference: CLDR-12016 and CLDR-11877 and CLDR-11850 */ - private boolean outputAllFiles(Writer out, File vetDataDir) { + private boolean outputAllFiles(VxmlGenerator vxmlGenerator, Writer out, File vetDataDir) { try { long start = System.currentTimeMillis(); ElapsedTimer overallTimer = @@ -251,14 +261,8 @@ private boolean outputAllFiles(Writer out, File vetDataDir) { out.write("
If kind is vxml (for example), we may write to both common/main and common/annotations, or * to both seed/main and seed/annotations. * - *
Note: this is only used for "manually" generated files. Compare writeOutputFile which is - * for "automatic" scheduled generation of files. - * * @param loc the CLDRLocale * @param kind the Kind, currently Kind.vxml and Kind.pxml are supported - * @return the File, or null for failure */ - private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { + private void writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { long st = System.currentTimeMillis(); CLDRFile cldrFile; if (kind == Kind.vxml) { @@ -353,15 +362,7 @@ private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { File baseDir = CLDRConfig.getInstance().getCldrBaseDirectory(); String commonOrSeed = DirNames.justCommon; for (String c : DirNames.commonAndSeed) { - String path = - baseDir - + "/" - + c - + "/" - + DirNames.justMain - + "/" - + loc.toString() - + XML_SUFFIX; + String path = baseDir + "/" + c + "/" + DirNames.justMain + "/" + loc + XML_SUFFIX; if (new File(path).exists()) { commonOrSeed = c; break; @@ -371,20 +372,14 @@ private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { * Only create the file in "main" here; doWriteFile will then create the file in "annotations" */ String outDirName = - vetDataDir - + "/" - + kind.toString() - + "/" - + commonOrSeed - + "/" - + DirNames.justMain; + vetDataDir + "/" + kind + "/" + commonOrSeed + "/" + DirNames.justMain; File outDir = new File(outDirName); if (!outDir.exists() && !outDir.mkdirs()) { throw new InternalError("Unable to create directory: " + outDirName); } - String outFileName = outDirName + "/" + loc.toString() + XML_SUFFIX; + String outFileName = outDirName + "/" + loc + XML_SUFFIX; File outFile = new File(outFileName); - doWriteFile(loc, cldrFile, kind, outFile); + doWriteFile(cldrFile, kind, outFile); SurveyLog.debug( "Updater: MANUALLY wrote: " + kind @@ -392,28 +387,21 @@ private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { + loc + " - " + ElapsedTimer.elapsedTime(st)); - return outFile; } catch (IOException e) { e.printStackTrace(); - throw new RuntimeException("IO Exception " + e.toString(), e); + throw new RuntimeException("IO Exception " + e, e); } } - /** - * Remove "empty" VXML files in a set of directories - * - * @param out the Writer, to receive HTML output - *
Compare RemoveEmptyCLDR.main - *
Reference: https://unicode-org.atlassian.net/browse/CLDR-12016
- */
- private void removeEmptyFiles(Writer out, File vxmlDir) throws IOException {
+ /** Remove "empty" VXML files in a set of directories */
+ private void removeEmptyFiles(File vxmlDir) throws IOException {
for (String c : DirNames.commonAndSeed) {
/*
* Skip main. Only do common/annotations and seed/annotations.
*/
File dirFile = new File(vxmlDir + "/" + c + "/" + DirNames.justAnnotations);
if (dirFile.exists()) {
- removeEmptyFilesOneDir(out, dirFile);
+ removeEmptyFilesOneDir(dirFile);
}
}
}
@@ -421,11 +409,10 @@ private void removeEmptyFiles(Writer out, File vxmlDir) throws IOException {
/**
* Remove "empty" VXML files in the given directory
*
- * @param out the Writer, to receive HTML output
* @param dirFile the given directory
* @throws IOException
*/
- private void removeEmptyFilesOneDir(Writer out, File dirFile) throws IOException {
+ private void removeEmptyFilesOneDir(File dirFile) throws IOException {
Set The following need to be verified on the server when generating vxml: • The same file
* must not occur in both the common/X and seed/X directories, for any X=main|annotations •
@@ -497,7 +485,8 @@ private static void addNameAndParents(Set vetdata └── vxml ├── common │ ├── annotations │ └── main └── seed ├──
* annotations └── main
*/
- private void verifyAllFiles(Writer out, File vxmlDir) throws IOException {
+ private void verifyAllFiles(VxmlGenerator vxmlGenerator, Writer out, File vxmlDir)
+ throws IOException {
int failureCount = 0;
/*
@@ -522,13 +511,21 @@ private void verifyAllFiles(Writer out, File vxmlDir) throws IOException {
if (failureCount == 0) {
out.write(" Called by SurveyMain.doGet when get a request.
*/
public boolean doRawXml(HttpServletRequest request, HttpServletResponse response)
- throws IOException, ServletException {
+ throws IOException {
/*
* request.getPathInfo returns what follows "survey" in the url.
* If the url is ".../cldr-apps/survey/vxml/main/aa.xml", it returns "vxml/main/aa.xml".
@@ -885,7 +879,7 @@ public boolean doRawXml(HttpServletRequest request, HttpServletResponse response
ctx.println("Return to SurveyTool ");
ctx.println(" These fields get filled in by getOutput, and referenced by the caller after getOutput
+ * returns
+ */
+ public static class Results {
+ public Status status = Status.WAITING;
+ public Appendable output = new StringBuilder();
+ }
+
+ /*
+ * Messages returned by getOutput
+ */
+ private static final String VXML_MESSAGE_STOPPED_ON_REQUEST = "Stopped on request";
+ private static final String VXML_MESSAGE_PROGRESS = "In Progress";
+ private static final String VXML_MESSAGE_STOPPED_STUCK = "Stopped (refresh if stuck)";
+ private static final String VXML_MESSAGE_NOT_LOADING = "Not loading. Click the button to load.";
+ private static final String VXML_MESSAGE_STARTED = "Started new task";
+
+ /**
+ * Start running, or continue running, the long-running task that generates VXML. This is called
+ * once for each request, including the initial request that starts VXML generation, and
+ * subsequent requests that query the status or trigger early termination of the task.
+ *
+ * @param args the VxmlQueue.Args
+ * @param results the VxmlQueue.Results
+ * @return the status message
+ * @throws IOException if thrown by results.output.append
+ */
+ public synchronized String getOutput(Args args, Results results) throws IOException {
+ QueueEntry entry = getEntry(args.qmi);
+ Task t = entry.currentTask;
+ if (t == null) {
+ logger.info("Got null Task in getOutput");
+ } else {
+ results.output.append(t.output.toString());
+ }
+ if (args.loadingPolicy == LoadingPolicy.STOP) {
+ stop(entry);
+ results.status = Status.STOPPED;
+ return VXML_MESSAGE_STOPPED_ON_REQUEST;
+ } else if (entry.done) {
+ setPercent(100);
+ results.status = Status.READY;
+ stop(entry);
+ return entry.verificationStatus.toString();
+ }
+ if (t != null) {
+ String waiting = waitingString();
+ results.status = Status.PROCESSING;
+ if (t.myThread.isAlive()) {
+ results.status = t.statusCode;
+ if (results.status != Status.WAITING) {
+ waiting = "";
+ }
+ setPercent(t.getPercent());
+ return VXML_MESSAGE_PROGRESS + ": " + waiting + t.status;
+ } else {
+ setPercent(0);
+ return VXML_MESSAGE_STOPPED_STUCK + " " + t.status;
+ }
+ }
+ if (args.loadingPolicy == LoadingPolicy.CONTINUE) {
+ results.status = Status.STOPPED;
+ setPercent(0);
+ return VXML_MESSAGE_NOT_LOADING;
+ }
+
+ // Note: similar code in VettingViewerQueue.getPriorityItemsSummaryOutput includes
+ // a comment suggesting an alternative: SurveyThreadManager.getExecutorService().invoke()
+ t = entry.currentTask = new Task(entry);
+ t.myThread = SurveyThreadManager.getThreadFactory().newThread(t);
+ t.myThread.start();
+
+ results.status = Status.PROCESSING;
+ setPercent(0);
+ final String waitStr = waitingString();
+ if (WAITING_IN_LINE_MESSAGE.equals(t.status) && waitStr.isEmpty()) {
+ // Simplify “Started new task: Waiting in line” to "Waiting in line"
+ return WAITING_IN_LINE_MESSAGE;
+ }
+ return VXML_MESSAGE_STARTED + ": " + waitStr + t.status;
+ }
+
+ private String waitingString() {
+ int aheadOfMe = OnlyOneVetter.getQueueLength();
+ return (aheadOfMe > 0) ? (aheadOfMe + " users waiting - ") : "";
+ }
+
+ private void stop(QueueEntry entry) {
+ Task t = entry.currentTask;
+ if (t != null) {
+ if (t.myThread.isAlive() && !t.stop) {
+ t.stop = true;
+ t.myThread.interrupt();
+ }
+ entry.currentTask = null;
+ entry.done = true;
+ }
+ }
+
+ private QueueEntry getEntry(QueueMemberId qmi) {
+ QueueEntry entry = (QueueEntry) qmi.get(KEY);
+ if (entry == null) {
+ entry = new QueueEntry();
+ qmi.put(KEY, entry);
+ }
+ return entry;
+ }
+
+ /** Estimated percentage complete for VXML generation */
+ private int percent = 0;
+
+ private void setPercent(int p) {
+ percent = p;
+ }
+
+ public int getPercent() {
+ return percent;
+ }
+}
diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java
index 5bf26a477d4..b32f6ee842f 100644
--- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java
+++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java
@@ -200,7 +200,7 @@ public synchronized void loadXPaths(XMLSource source) {
Connection conn = null;
PreparedStatement queryStmt = null;
try {
- conn = DBUtils.getInstance().getDBConnection();
+ conn = DBUtils.getInstance().getAConnection();
if (!DEBUG) {
addXpaths(unloadedXpaths, conn);
} else {
@@ -226,7 +226,7 @@ public synchronized void loadXPaths(XMLSource source) {
* Add a set of xpaths to the database.
*
* @param xpaths
- * @param conn
+ * @param conn the connection (auto-commit)
* @throws SQLException
*/
private synchronized void addXpaths(Set✅ VXML verification succeeded
\nOK
");
- System.out.println("VXML verification succeeded");
+ if (vxmlGenerator != null) {
+ vxmlGenerator.setVerificationStatus(VxmlGenerator.VerificationStatus.SUCCESSFUL);
+ } else {
+ System.out.println("VXML verification succeeded");
+ }
} else {
out.write(
"❌ VXML verification failed!
\nFailure count = "
+ failureCount
+ "
");
- System.out.println("VXML verification failed! Failure count = " + failureCount);
+ if (vxmlGenerator != null) {
+ vxmlGenerator.setVerificationStatus(VxmlGenerator.VerificationStatus.FAILED);
+ } else {
+ System.out.println("VXML verification failed! Failure count = " + failureCount);
+ }
}
}
@@ -606,7 +603,7 @@ private boolean verifyParentChildSameDirectory(Writer out, File vxmlDir) throws
}
CLDRLocale parLoc = childLoc.getParent();
if (parLoc != null) {
- String parentName = parLoc.toString() + XML_SUFFIX;
+ String parentName = parLoc + XML_SUFFIX;
if (!childName.equals(parentName)
&& !"en.xml".equals(parentName)
&& !"root.xml".equals(parentName)) {
@@ -760,13 +757,12 @@ private static SetLocales
");
ctx.println("");
- CLDRLocale locales[] = SurveyMain.getLocales();
+ CLDRLocale[] locales = SurveyMain.getLocales();
int nrInFiles = locales.length;
for (int i = 0; i < nrInFiles; i++) {
CLDRLocale locale = locales[i];
@@ -909,7 +903,7 @@ public boolean doRawXml(HttpServletRequest request, HttpServletResponse response
response.sendRedirect(ctx.schemeHostPort() + ctx.base() + XML_PREFIX + "/");
} else {
boolean found = false;
- CLDRLocale locales[] = SurveyMain.getLocales();
+ CLDRLocale[] locales = SurveyMain.getLocales();
CLDRLocale foundLocale = null;
int nrInFiles = locales.length;
for (int i = 0; (!found) && (i < nrInFiles); i++) {
@@ -951,67 +945,4 @@ public boolean doRawXml(HttpServletRequest request, HttpServletResponse response
}
}
}
-
- // statistics helpers
- private static Map