Current Status: {{ status }}
++ +
+ +
+
Invoked by pasting a url like this into a browser: - * http://localhost:8080/cldr-apps/admin-OutputAllFiles.jsp?vap=... - *
This function was started using code moved here from admin-OutputAllFiles.jsp. - * Reference: CLDR-12016 and CLDR-11877 - *
TODO: link to gear menu and use JavaScript for a front-end. + * .../cldr-apps/admin-OutputAllFiles.jsp?vap=... + *
TODO: remove this method, and admin-OutputAllFiles.jsp, and other obsolete code once + * the new implementation with GenerateVxml.java is well tested + *
Reference: https://unicode-org.atlassian.net/browse/CLDR-14913
*/
public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer out) {
String vap = request.getParameter("vap");
@@ -120,10 +109,33 @@ public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer ou
out.write("verify=true/false
\n");
return;
}
+ generateVxml(null, out, outputFiles, removeEmpty, verifyConsistent);
+ } catch (Exception e) {
+ System.err.println("Exception in outputAndVerifyAllFiles: " + e);
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Generate VXML
+ *
+ *
Called indirectly by modern api/GenerateVxml, as well as by legacy outputAndVerifyAllFiles + * + * @param vxmlGenerator the VxmlGenerator, or null if called by outputAndVerifyAllFiles + * @param out + * @param outputFiles + * @param removeEmpty + * @param verifyConsistent + */ + public static void generateVxml( + VxmlGenerator vxmlGenerator, + Writer out, + boolean outputFiles, + boolean removeEmpty, + boolean verifyConsistent) { + try { /* * Sync on OutputFileManager.class here prevents re-entrance if invoked repeatedly before completion. - * Performance problem if run while Survey Tool has multiple users/requests? - * Completion of http request/response may take over ten minutes! TODO: use ajax. */ synchronized (OutputFileManager.class) { SurveyMain sm = CookieSession.sm; @@ -136,9 +148,9 @@ public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer ou out.write("Directory creation for vetting data failed."); return; } - out.write("
Created new directory: " + vetdataDir.toString() + "
"); + out.write("Created new directory: " + vetdataDir + "
"); - if (outputFiles && !ofm.outputAllFiles(out, vetdataDir)) { + if (outputFiles && !ofm.outputAllFiles(vxmlGenerator, out, vetdataDir)) { out.write("File output failed."); return; } @@ -148,18 +160,18 @@ public static void outputAndVerifyAllFiles(HttpServletRequest request, Writer ou } File vxmlDir = null; if (removeEmpty || verifyConsistent) { - vxmlDir = new File(vetdataDir.toString() + "/" + Kind.vxml.name()); + vxmlDir = new File(vetdataDir + "/" + Kind.vxml.name()); } if (removeEmpty) { - ofm.removeEmptyFiles(out, vxmlDir); + ofm.removeEmptyFiles(vxmlDir); } if (verifyConsistent) { - ofm.verifyAllFiles(out, vxmlDir); + ofm.verifyAllFiles(vxmlGenerator, out, vxmlDir); } } - System.out.println("outputAndVerifyAllFiles finished"); + System.out.println("reallyOutputAndVerifyAllFiles finished"); } catch (Exception e) { - System.err.println("Exception in outputAndVerifyAllFiles: " + e); + System.err.println("Exception in reallyOutputAndVerifyAllFiles: " + e); e.printStackTrace(); } } @@ -193,7 +205,6 @@ private static File createNewManualVetdataDir(File vetdataDir) { * Copy the DTD file from trunk into subfolders of the given vetdata folder ("auto" or "manual") * * @param vetdataDir the File for the vetdata directory - * @param common the name of the "common" folder * @return true for success, or false for failure *The dtd is required for removeEmptyFiles when it calls XMLFileReader.loadPathValues. * The xml files all have something like: This function was first created using code moved here from admin-OutputAllFiles.jsp. - * Reference: CLDR-12016 and CLDR-11877 and CLDR-11850 */ - private boolean outputAllFiles(Writer out, File vetDataDir) { + private boolean outputAllFiles(VxmlGenerator vxmlGenerator, Writer out, File vetDataDir) { try { long start = System.currentTimeMillis(); ElapsedTimer overallTimer = @@ -251,14 +261,8 @@ private boolean outputAllFiles(Writer out, File vetDataDir) { out.write("
If kind is vxml (for example), we may write to both common/main and common/annotations, or * to both seed/main and seed/annotations. * - *
Note: this is only used for "manually" generated files. Compare writeOutputFile which is - * for "automatic" scheduled generation of files. - * * @param loc the CLDRLocale * @param kind the Kind, currently Kind.vxml and Kind.pxml are supported - * @return the File, or null for failure */ - private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { + private void writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { long st = System.currentTimeMillis(); CLDRFile cldrFile; if (kind == Kind.vxml) { @@ -353,15 +362,7 @@ private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { File baseDir = CLDRConfig.getInstance().getCldrBaseDirectory(); String commonOrSeed = DirNames.justCommon; for (String c : DirNames.commonAndSeed) { - String path = - baseDir - + "/" - + c - + "/" - + DirNames.justMain - + "/" - + loc.toString() - + XML_SUFFIX; + String path = baseDir + "/" + c + "/" + DirNames.justMain + "/" + loc + XML_SUFFIX; if (new File(path).exists()) { commonOrSeed = c; break; @@ -371,20 +372,14 @@ private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { * Only create the file in "main" here; doWriteFile will then create the file in "annotations" */ String outDirName = - vetDataDir - + "/" - + kind.toString() - + "/" - + commonOrSeed - + "/" - + DirNames.justMain; + vetDataDir + "/" + kind + "/" + commonOrSeed + "/" + DirNames.justMain; File outDir = new File(outDirName); if (!outDir.exists() && !outDir.mkdirs()) { throw new InternalError("Unable to create directory: " + outDirName); } - String outFileName = outDirName + "/" + loc.toString() + XML_SUFFIX; + String outFileName = outDirName + "/" + loc + XML_SUFFIX; File outFile = new File(outFileName); - doWriteFile(loc, cldrFile, kind, outFile); + doWriteFile(cldrFile, kind, outFile); SurveyLog.debug( "Updater: MANUALLY wrote: " + kind @@ -392,28 +387,21 @@ private File writeManualOutputFile(File vetDataDir, CLDRLocale loc, Kind kind) { + loc + " - " + ElapsedTimer.elapsedTime(st)); - return outFile; } catch (IOException e) { e.printStackTrace(); - throw new RuntimeException("IO Exception " + e.toString(), e); + throw new RuntimeException("IO Exception " + e, e); } } - /** - * Remove "empty" VXML files in a set of directories - * - * @param out the Writer, to receive HTML output - *
Compare RemoveEmptyCLDR.main - *
Reference: https://unicode-org.atlassian.net/browse/CLDR-12016
- */
- private void removeEmptyFiles(Writer out, File vxmlDir) throws IOException {
+ /** Remove "empty" VXML files in a set of directories */
+ private void removeEmptyFiles(File vxmlDir) throws IOException {
for (String c : DirNames.commonAndSeed) {
/*
* Skip main. Only do common/annotations and seed/annotations.
*/
File dirFile = new File(vxmlDir + "/" + c + "/" + DirNames.justAnnotations);
if (dirFile.exists()) {
- removeEmptyFilesOneDir(out, dirFile);
+ removeEmptyFilesOneDir(dirFile);
}
}
}
@@ -421,11 +409,10 @@ private void removeEmptyFiles(Writer out, File vxmlDir) throws IOException {
/**
* Remove "empty" VXML files in the given directory
*
- * @param out the Writer, to receive HTML output
* @param dirFile the given directory
* @throws IOException
*/
- private void removeEmptyFilesOneDir(Writer out, File dirFile) throws IOException {
+ private void removeEmptyFilesOneDir(File dirFile) throws IOException {
Set The following need to be verified on the server when generating vxml: • The same file
* must not occur in both the common/X and seed/X directories, for any X=main|annotations •
@@ -497,7 +485,8 @@ private static void addNameAndParents(Set vetdata └── vxml ├── common │ ├── annotations │ └── main └── seed ├──
* annotations └── main
*/
- private void verifyAllFiles(Writer out, File vxmlDir) throws IOException {
+ private void verifyAllFiles(VxmlGenerator vxmlGenerator, Writer out, File vxmlDir)
+ throws IOException {
int failureCount = 0;
/*
@@ -522,13 +511,21 @@ private void verifyAllFiles(Writer out, File vxmlDir) throws IOException {
if (failureCount == 0) {
out.write(" Called by SurveyMain.doGet when get a request.
*/
public boolean doRawXml(HttpServletRequest request, HttpServletResponse response)
- throws IOException, ServletException {
+ throws IOException {
/*
* request.getPathInfo returns what follows "survey" in the url.
* If the url is ".../cldr-apps/survey/vxml/main/aa.xml", it returns "vxml/main/aa.xml".
@@ -885,7 +879,7 @@ public boolean doRawXml(HttpServletRequest request, HttpServletResponse response
ctx.println("Return to SurveyTool ");
ctx.println(" These fields get filled in by getOutput, and referenced by the caller after getOutput
+ * returns
+ */
+ public static class Results {
+ public Status status = Status.WAITING;
+ public Appendable output = new StringBuilder();
+ }
+
+ /*
+ * Messages returned by getOutput
+ */
+ private static final String VXML_MESSAGE_STOPPED_ON_REQUEST = "Stopped on request";
+ private static final String VXML_MESSAGE_PROGRESS = "In Progress";
+ private static final String VXML_MESSAGE_STOPPED_STUCK = "Stopped (refresh if stuck)";
+ private static final String VXML_MESSAGE_NOT_LOADING = "Not loading. Click the button to load.";
+ private static final String VXML_MESSAGE_STARTED = "Started new task";
+
+ /**
+ * Start running, or continue running, the long-running task that generates VXML. This is called
+ * once for each request, including the initial request that starts VXML generation, and
+ * subsequent requests that query the status or trigger early termination of the task.
+ *
+ * @param args the VxmlQueue.Args
+ * @param results the VxmlQueue.Results
+ * @return the status message
+ * @throws IOException if thrown by results.output.append
+ */
+ public synchronized String getOutput(Args args, Results results) throws IOException {
+ QueueEntry entry = getEntry(args.qmi);
+ Task t = entry.currentTask;
+ if (t == null) {
+ logger.info("Got null Task in getOutput");
+ } else {
+ results.output.append(t.output.toString());
+ }
+ if (args.loadingPolicy == LoadingPolicy.STOP) {
+ stop(entry);
+ results.status = Status.STOPPED;
+ return VXML_MESSAGE_STOPPED_ON_REQUEST;
+ } else if (entry.done) {
+ setPercent(100);
+ results.status = Status.READY;
+ stop(entry);
+ return entry.verificationStatus.toString();
+ }
+ if (t != null) {
+ String waiting = waitingString();
+ results.status = Status.PROCESSING;
+ if (t.myThread.isAlive()) {
+ results.status = t.statusCode;
+ if (results.status != Status.WAITING) {
+ waiting = "";
+ }
+ setPercent(t.getPercent());
+ return VXML_MESSAGE_PROGRESS + ": " + waiting + t.status;
+ } else {
+ setPercent(0);
+ return VXML_MESSAGE_STOPPED_STUCK + " " + t.status;
+ }
+ }
+ if (args.loadingPolicy == LoadingPolicy.CONTINUE) {
+ results.status = Status.STOPPED;
+ setPercent(0);
+ return VXML_MESSAGE_NOT_LOADING;
+ }
+
+ // Note: similar code in VettingViewerQueue.getPriorityItemsSummaryOutput includes
+ // a comment suggesting an alternative: SurveyThreadManager.getExecutorService().invoke()
+ t = entry.currentTask = new Task(entry);
+ t.myThread = SurveyThreadManager.getThreadFactory().newThread(t);
+ t.myThread.start();
+
+ results.status = Status.PROCESSING;
+ setPercent(0);
+ final String waitStr = waitingString();
+ if (WAITING_IN_LINE_MESSAGE.equals(t.status) && waitStr.isEmpty()) {
+ // Simplify “Started new task: Waiting in line” to "Waiting in line"
+ return WAITING_IN_LINE_MESSAGE;
+ }
+ return VXML_MESSAGE_STARTED + ": " + waitStr + t.status;
+ }
+
+ private String waitingString() {
+ int aheadOfMe = OnlyOneVetter.getQueueLength();
+ return (aheadOfMe > 0) ? (aheadOfMe + " users waiting - ") : "";
+ }
+
+ private void stop(QueueEntry entry) {
+ Task t = entry.currentTask;
+ if (t != null) {
+ if (t.myThread.isAlive() && !t.stop) {
+ t.stop = true;
+ t.myThread.interrupt();
+ }
+ entry.currentTask = null;
+ entry.done = true;
+ }
+ }
+
+ private QueueEntry getEntry(QueueMemberId qmi) {
+ QueueEntry entry = (QueueEntry) qmi.get(KEY);
+ if (entry == null) {
+ entry = new QueueEntry();
+ qmi.put(KEY, entry);
+ }
+ return entry;
+ }
+
+ /**
+ * Estimated percentage complete for VXML generation
+ */
+ private int percent = 0;
+
+ private void setPercent(int p) {
+ percent = p;
+ }
+
+ public int getPercent() {
+ return percent;
+ }
+}
diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java
index 5bf26a477d4..d0f545bdada 100644
--- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java
+++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java
@@ -200,7 +200,7 @@ public synchronized void loadXPaths(XMLSource source) {
Connection conn = null;
PreparedStatement queryStmt = null;
try {
- conn = DBUtils.getInstance().getDBConnection();
+ conn = DBUtils.getInstance().getAConnection();
if (!DEBUG) {
addXpaths(unloadedXpaths, conn);
} else {
diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/GenerateVxml.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/GenerateVxml.java
new file mode 100644
index 00000000000..72a48ce6c29
--- /dev/null
+++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/GenerateVxml.java
@@ -0,0 +1,111 @@
+package org.unicode.cldr.web.api;
+
+import java.io.IOException;
+import javax.enterprise.context.ApplicationScoped;
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.eclipse.microprofile.openapi.annotations.Operation;
+import org.eclipse.microprofile.openapi.annotations.media.Content;
+import org.eclipse.microprofile.openapi.annotations.media.Schema;
+import org.eclipse.microprofile.openapi.annotations.responses.APIResponse;
+import org.eclipse.microprofile.openapi.annotations.responses.APIResponses;
+import org.eclipse.microprofile.openapi.annotations.tags.Tag;
+import org.unicode.cldr.web.*;
+
+@ApplicationScoped
+@Path("/vxml")
+@Tag(name = "Generate VXML", description = "APIs for Survey Tool VXML (Vetted XML) generation")
+public class GenerateVxml {
+ @POST
+ @Produces(MediaType.APPLICATION_JSON)
+ @Operation(summary = "Generate VXML", description = "Generate VXML")
+ @APIResponses(
+ value = {
+ @APIResponse(
+ responseCode = "200",
+ description = "Generate VXML",
+ content =
+ @Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = VxmlResponse.class))),
+ @APIResponse(responseCode = "403", description = "Forbidden"),
+ @APIResponse(
+ responseCode = "500",
+ description = "Internal Server Error",
+ content =
+ @Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = STError.class))),
+ @APIResponse(responseCode = "503", description = "Not ready yet"),
+ })
+ public Response generateVxml(
+ VxmlRequest request, @HeaderParam(Auth.SESSION_HEADER) String sessionString) {
+ try {
+ CookieSession cs = Auth.getSession(sessionString);
+ if (cs == null) {
+ return Auth.noSessionResponse();
+ }
+ if (!UserRegistry.userIsAdmin(cs.user)) {
+ return Response.status(Response.Status.FORBIDDEN).build();
+ }
+ if (SurveyMain.isBusted()
+ || !SurveyMain.wasInitCalled()
+ || !SurveyMain.triedToStartUp()) {
+ return STError.surveyNotQuiteReady();
+ }
+ // Make sure setupDB has run before we start to generate VXML, to avoid the risk of
+ // concurrency problems, since otherwise if the server just started, setupDB may run
+ // concurrently with the vxml worker thread.
+ CookieSession.sm.getSTFactory().setupDB();
+ cs.userDidAction();
+ VxmlResponse vr = getVxmlResponse(request.loadingPolicy, cs);
+ return Response.ok(vr).build();
+ } catch (Exception e) {
+ return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ /**
+ * Get the response for VXML
+ *
+ * @param loadingPolicy the LoadingPolicy
+ * @param cs the CookieSession
+ * @return the VxmlResponse
+ * @throws IOException if thrown by VxmlQueue.getOutput
+ */
+ private VxmlResponse getVxmlResponse(VxmlQueue.LoadingPolicy loadingPolicy, CookieSession cs)
+ throws IOException {
+ VxmlQueue queue = VxmlQueue.getInstance();
+ QueueMemberId qmi = new QueueMemberId(cs);
+ VxmlResponse response = new VxmlResponse();
+ VxmlQueue.Args args = new VxmlQueue.Args(qmi, loadingPolicy);
+ VxmlQueue.Results results = new VxmlQueue.Results();
+ response.message = queue.getOutput(args, results);
+ response.percent = queue.getPercent();
+ response.status = results.status;
+ response.output = results.output.toString();
+ return response;
+ }
+
+ @Schema(description = "VXML Request")
+ public static final class VxmlRequest {
+ @Schema(implementation = VxmlQueue.LoadingPolicy.class)
+ public VxmlQueue.LoadingPolicy loadingPolicy;
+ }
+
+ @Schema(description = "VXML Response")
+ public static final class VxmlResponse {
+ @Schema(description = "VXML Response status enum")
+ public VxmlQueue.Status status;
+
+ @Schema(description = "Current status message")
+ public String message = "";
+
+ @Schema(description = "Estimated percentage complete")
+ public Number percent;
+
+ @Schema(description = "Output on success")
+ public String output;
+ }
+}
✅ VXML verification succeeded
\nOK
");
- System.out.println("VXML verification succeeded");
+ if (vxmlGenerator != null) {
+ vxmlGenerator.setVerificationStatus(VxmlGenerator.VerificationStatus.SUCCESSFUL);
+ } else {
+ System.out.println("VXML verification succeeded");
+ }
} else {
out.write(
"❌ VXML verification failed!
\nFailure count = "
+ failureCount
+ "
");
- System.out.println("VXML verification failed! Failure count = " + failureCount);
+ if (vxmlGenerator != null) {
+ vxmlGenerator.setVerificationStatus(VxmlGenerator.VerificationStatus.FAILED);
+ } else {
+ System.out.println("VXML verification failed! Failure count = " + failureCount);
+ }
}
}
@@ -606,7 +603,7 @@ private boolean verifyParentChildSameDirectory(Writer out, File vxmlDir) throws
}
CLDRLocale parLoc = childLoc.getParent();
if (parLoc != null) {
- String parentName = parLoc.toString() + XML_SUFFIX;
+ String parentName = parLoc + XML_SUFFIX;
if (!childName.equals(parentName)
&& !"en.xml".equals(parentName)
&& !"root.xml".equals(parentName)) {
@@ -760,13 +757,12 @@ private static SetLocales
");
ctx.println("");
- CLDRLocale locales[] = SurveyMain.getLocales();
+ CLDRLocale[] locales = SurveyMain.getLocales();
int nrInFiles = locales.length;
for (int i = 0; i < nrInFiles; i++) {
CLDRLocale locale = locales[i];
@@ -909,7 +903,7 @@ public boolean doRawXml(HttpServletRequest request, HttpServletResponse response
response.sendRedirect(ctx.schemeHostPort() + ctx.base() + XML_PREFIX + "/");
} else {
boolean found = false;
- CLDRLocale locales[] = SurveyMain.getLocales();
+ CLDRLocale[] locales = SurveyMain.getLocales();
CLDRLocale foundLocale = null;
int nrInFiles = locales.length;
for (int i = 0; (!found) && (i < nrInFiles); i++) {
@@ -951,67 +945,4 @@ public boolean doRawXml(HttpServletRequest request, HttpServletResponse response
}
}
}
-
- // statistics helpers
- private static Map