Merge remote-tracking branch 'origin/candidate-9.6.x' into candidate-…

…9.8.x

common/thorhelper/thorsoapcall.cpp

@@ -50,6 +50,40 @@ using roxiemem::OwnedRoxieString;
 #define CONNECTION "Connection"
 
 unsigned soapTraceLevel = 1;
+static StringBuffer soapSepString;
+
+void setSoapSepString(const char *_soapSepString)
+{
+    soapSepString.set(_soapSepString);
+}
+
+static void multiLineAppendReplace(StringBuffer &origStr, StringBuffer &newStr)
+{
+    if (origStr.isEmpty())
+        return;
+
+    newStr.ensureCapacity(origStr.length());
+
+    const char *cursor = origStr;
+    while (*cursor)
+    {
+        switch (*cursor)
+        {
+            case '\r':
+                newStr.append(soapSepString);
+                if ('\n' == *(cursor+1))
+                    cursor++;
+                break;
+            case '\n':
+                newStr.append(soapSepString);
+                break;
+            default:
+                newStr.append(*cursor);
+                break;
+        }
+        ++cursor;
+    }
+}
 
 #define WSCBUFFERSIZE 0x10000
 #define MAXWSCTHREADS 50    //Max Web Service Call Threads
@@ -1940,10 +1974,18 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
     {
         if (soapTraceLevel > 6 || master->logXML)
         {
-            if (!contentEncoded)
-                master->logctx.mCTXLOG("%s: request(%s)", master->wscCallTypeText(), request.str());
+            StringBuffer contentStr;
+            if (contentEncoded)
+                contentStr.append(", content encoded.");
+            // Only do translation if soapcall LOG option set and soapSepString defined
+            if ( (master->logXML) && (soapSepString.length() > 0) )
+            {
+                StringBuffer request2;
+                multiLineAppendReplace(request, request2);
+                master->logctx.CTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request2.str(), contentStr.str());
+            }
             else
-                master->logctx.mCTXLOG("%s: request(%s), content encoded.", master->wscCallTypeText(), request.str());
+                master->logctx.mCTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request.str(), contentStr.str());
         }
     }
 
@@ -2247,9 +2289,18 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
         if (checkContentDecoding(dbgheader, response, contentEncoding))
             decodeContent(contentEncoding.str(), response);
         if (soapTraceLevel > 6 || master->logXML)
-            master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str());
-        else if (soapTraceLevel > 8)
-            master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but...
+        {
+            // Only do translation if soapcall LOG option set and soapSepString defined
+            if ( (master->logXML) && (soapSepString.length() > 0) )
+            {
+                StringBuffer response2;
+                multiLineAppendReplace(dbgheader, response2);
+                multiLineAppendReplace(response, response2);
+                master->logctx.CTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response2.str());
+            }
+            else
+                master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str());
+        }
         return rval;
     }
 

common/thorhelper/thorsoapcall.hpp

@@ -86,5 +86,6 @@ interface IRoxieAbortMonitor
 extern THORHELPER_API unsigned soapTraceLevel;
 extern THORHELPER_API IWSCHelper * createSoapCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor);
 extern THORHELPER_API IWSCHelper * createHttpCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor);
+extern THORHELPER_API void setSoapSepString(const char *_soapSepString);
 
 #endif /* __THORSOAPCALL_HPP_ */

ecl/hthor/hthor.cpp

@@ -7738,6 +7738,10 @@ void CHThorWSCBaseActivity::init()
         JBASE64_Encode(uidpair.str(), uidpair.length(), authToken, false);
     }
     soapTraceLevel = agent.queryWorkUnit()->getDebugValueInt("soapTraceLevel", 1);
+    StringBuffer soapSepStr;
+    StringBufferAdaptor soapSepAdaptor(soapSepStr);
+    agent.queryWorkUnit()->getDebugValue("soapLogSepString", soapSepAdaptor);
+    setSoapSepString(soapSepStr.str());
 }
 
 //---------------------------------------------------------------------------

esp/src/eclwatch/WUDetailsWidget.js

@@ -117,6 +117,16 @@ define([
             this.emailFrom = registry.byId(this.id + "EmailFrom");
             this.emailSubject = registry.byId(this.id + "EmailSubject");
             this.emailBody = registry.byId(this.id + "EmailBody");
+
+            //Zap LogFilters
+            this.logFilterStartDateTime = dom.byId(this.id + "StartDateTime");
+            this.logFilterStartDate = registry.byId(this.id + "StartDate");
+            this.logFilterStartTime = registry.byId(this.id + "StartTime");
+            this.logFilterEndDateTime = dom.byId(this.id + "EndDateTime");
+            this.logFilterEndDate = registry.byId(this.id + "EndDate");
+            this.logFilterEndTime = registry.byId(this.id + "EndTime");
+            this.logFilterRelativeTimeRangeBuffer = registry.byId(this.id + "RelativeTimeRangeBuffer");
+
             this.protected = registry.byId(this.id + "Protected");
             this.infoGridWidget = registry.byId(this.id + "InfoContainer");
             this.zapDialog = registry.byId(this.id + "ZapDialog");
@@ -146,14 +156,33 @@ define([
             this.checkThorLogStatus();
         },
 
+        formatLogFilterDateTime: function (dateField, timeField, dateTimeField) {
+            if (dateField.value.toString() !== "Invalid Date") {
+                const d = new Date(dateField.value);
+                const date = `${d.getFullYear()}-${(d.getMonth() < 9 ? "0" : "") + parseInt(d.getMonth() + 1, 10)}-${d.getDate()}`;
+                const time = timeField.value.toString().replace(/.*1970\s(\S+).*/, "$1");
+                dateTimeField.value = `${date}T${time}.000Z`;
+            }
+        },
+
         _onSubmitDialog: function () {
             var context = this;
             var includeSlaveLogsCheckbox = this.includeSlaveLogsCheckbox.get("checked");
+            if (this.logFilterRelativeTimeRangeBuffer.value !== "") {
+                this.logFilterEndDate.required = "";
+                this.logFilterStartDate.required = "";
+            }
             if (this.zapForm.validate()) {
                 //WUCreateAndDownloadZAPInfo is not a webservice so relying on form to submit.
                 //Server treats "on" and '' as the same thing.
                 this.includeSlaveLogsCheckbox.set("value", includeSlaveLogsCheckbox ? "on" : "off");
+
+                // Log Filters
+                this.formatLogFilterDateTime(this.logFilterStartDate, this.logFilterStartTime, this.logFilterStartDateTime);
+                this.formatLogFilterDateTime(this.logFilterEndDate, this.logFilterEndTime, this.logFilterEndDateTime);
+
                 this.zapForm.set("action", "/WsWorkunits/WUCreateAndDownloadZAPInfo");
+
                 this.zapDialog.hide();
                 this.checkThorLogStatus();
                 if (this.logAccessorMessage !== "") {

esp/src/eclwatch/css/hpcc.css

@@ -74,6 +74,10 @@ form li label {
     padding-top: 4px;
 }
 
+.dijitDialogPaneContent {
+    overflow-x: hidden !important;
+}
+
 .dijitDialogPaneContent form li label {
     float: left;
     width: 25%;

esp/src/eclwatch/templates/WUDetailsWidget.html

@@ -183,7 +183,7 @@ <h2>
     <div id="${id}ZapDialog" data-dojo-type="dijit.Dialog" title="${i18n.ZippedAnalysisPackage}">
         <div class="dijitDialogPaneContentArea">
             <div id="${id}ZapForm" style="width:460px;" method="post" enctype="application/x-www-form-urlencoded" data-dojo-type="dijit.form.Form">
-                <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
+                <div data-dojo-props="cols:2" style="width:410px" data-dojo-type="hpcc.TableContainer">
                     <input id="${id}ZapName" title="${i18n.FileName}:" name="ZAPFileName" colspan="2" data-dojo-props="trim: true," data-dojo-type="dijit.form.TextBox" />
                     <input id="${id}ZapWUID" title="${i18n.WUID}:" name="Wuid" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
                     <input id="${id}BuildVersion" title="${i18n.ESPBuildVersion}:" name="BuildVersion" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
@@ -199,8 +199,41 @@ <h2>
                     <input id="${id}EmailFrom" title="${i18n.EmailFrom}:" name="EmailFrom" colspan="2" data-dojo-props="trim:true, placeHolder:'See Configuration Manager.'" data-dojo-type="dijit.form.TextBox" />
                     <input id="${id}EmailSubject" title="${i18n.EmailSubject}:" name="EmailSubject" colspan="2" data-dojo-props="trim:true" data-dojo-type="dijit.form.ValidationTextBox" required="false" />
                     <input id="${id}EmailBody" title="${i18n.EmailBody}:" name="EmailBody" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea" />
+                    <input id="${id}StartDateTime" name="LogFilter_AbsoluteTimeRange_StartDate" type="hidden" />
+                    <input id="${id}StartDate" title="${i18n.FromDate}:" name="StartDate" data-dojo-props="trim: true" required="required" data-dojo-type="dijit.form.DateTextBox" />
+                    <input id="${id}StartTime" title="" name="StartTime" data-dojo-props="trim: true" value="T07:30:00" data-dojo-type="dijit.form.TimeTextBox" />
+                    <input id="${id}EndDateTime" name="LogFilter_AbsoluteTimeRange_EndDate" type="hidden" />
+                    <input id="${id}EndDate" title="${i18n.ToDate}:" name="EndDate" data-dojo-props="trim: true" required="required" data-dojo-type="dijit.form.DateTextBox" />
+                    <input id="${id}EndTime" title="" name="EndTime" data-dojo-props="trim: true" value="T19:30:00" data-dojo-type="dijit.form.TimeTextBox" />
+                    <input id="${id}RelativeTimeRangeBuffer" title="${i18n.RelativeTimeRange}" name="LogFilter_RelativeTimeRangeBuffer" colspan="2" data-dojo-type="dijit.form.TextBox" />
+                    <input id="${id}LineLimit" title="${i18n.LogLineLimit}" name="LogFilter_LineLimit" colspan="2" value="10000" data-dojo-type="dijit.form.TextBox" />
+                    <input id="${id}LineStartFrom" title="${i18n.LogLineStartFrom}" name="LogFilter_LineStartFrom" value="0" colspan="2" data-dojo-type="dijit.form.TextBox" />
+                    <select id="${id}SelectColumnMode" title="${i18n.ColumnMode}" name="LogFilter_SelectColumnMode" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option value="0">MIN</option>
+                        <option selected="selected" value="1">DEFAULT</option>
+                        <option value="2">ALL</option>
+                    </select>
+                    <select id="${id}LogFilterFormat" title="${i18n.LogFormat}" name="LogFilter_Format" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option selected="selected" value="csv">CSV</option>
+                        <option value="json">JSON</option>
+                        <option value="xml">XML</option>
+                    </select>
+                    <input id="${id}WildcardFilter" title="${i18n.WildcardFilter}" name="LogFilter_WildcardFilter" colspan="2" data-dojo-type="dijit.form.TextBox" />
+                    <select id="${id}sortByTimeDirection" title="${i18n.Sort} ${i18n.TimeStamp}" name="LogFilter_sortByTimeDirection" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option value="0">ASC</option>
+                        <option selected="selected" value="1">DESC</option>
+                    </select>
+                    <select id="${id}LogEventType" title="${i18n.LogEventType}" name="LogFilter_LogEventType" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option selected="selected" value="ALL">All</option>
+                        <option value="DIS">Disaster</option>
+                        <option value="ERR">Error</option>
+                        <option value="WRN">Warning</option>
+                        <option value="INF">Information</option>
+                        <option value="PRO">Progress</option>
+                        <option value="MET">Metric</option>
+                    </select>
                 </div>
-                <div class="dijitDialogPaneActionBar">
+                <div style="width:410px" class="dijitDialogPaneActionBar">
                     <button id="${id}onZapSubmit" data-dojo-attach-event="onClick:_onSubmitDialog" type="submit" data-dojo-type="dijit.form.Button">${i18n.Apply}</button>
                     <button style="float:left" data-dojo-attach-event="onClick:_onCancelDialog" data-dojo-type="dijit.form.Button">${i18n.Cancel}</button>
                 </div>

esp/src/package.json

@@ -44,7 +44,7 @@
     "@hpcc-js/chart": "2.84.1",
     "@hpcc-js/codemirror": "2.63.0",
     "@hpcc-js/common": "2.72.0",
-    "@hpcc-js/comms": "2.96.1",
+    "@hpcc-js/comms": "2.97.0",
     "@hpcc-js/dataflow": "8.1.7",
     "@hpcc-js/eclwatch": "2.75.3",
     "@hpcc-js/graph": "2.86.0",

esp/src/src-react/components/Logs.tsx

@@ -3,7 +3,8 @@ import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluen
 import { GetLogsExRequest, LogaccessService, TargetAudience, LogType } from "@hpcc-js/comms";
 import { Level, scopedLogger } from "@hpcc-js/util";
 import nlsHPCC from "src/nlsHPCC";
-import { logColor, wuidToDate, wuidToTime } from "src/Utility";
+import { logColor, timestampToDate, wuidToDate, wuidToTime } from "src/Utility";
+import { useLoggingEngine } from "../hooks/platform";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { pushParams } from "../util/history";
 import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid";
@@ -110,28 +111,49 @@ export const Logs: React.FunctionComponent<LogsProps> = ({
 
     const now = React.useMemo(() => new Date(), []);
 
+    const loggingEngine = useLoggingEngine();
+
     //  Grid ---
     const columns = React.useMemo((): FluentColumns => {
-        return {
-            timestamp: { label: nlsHPCC.TimeStamp, width: 140, sortable: false, },
-            message: { label: nlsHPCC.Message, width: 600, sortable: false, },
-            components: { label: nlsHPCC.ContainerName, width: 150, sortable: false },
-            instance: { label: nlsHPCC.PodName, width: 150, sortable: false },
-            audience: { label: nlsHPCC.Audience, width: 60, sortable: false, },
-            class: {
-                label: nlsHPCC.Class, width: 40, sortable: false,
-                formatter: level => {
-                    const colors = logColor(levelMap(level));
-                    const styles = { backgroundColor: colors.background, padding: "2px 6px", color: colors.foreground };
-                    return <span style={styles}>{level}</span>;
-                }
+        let retVal = {
+            timestamp: {
+                label: nlsHPCC.TimeStamp, width: 140, sortable: false,
+                formatter: ts => {
+                    if (ts) {
+                        if (ts.indexOf(":") < 0) {
+                            return timestampToDate(ts).toISOString();
+                        }
+                        return new Date(ts).toISOString();
+                    }
+                },
             },
-            workunits: { label: nlsHPCC.JobID, width: 50, sortable: false, hidden: wuid !== undefined, },
-            processid: { label: nlsHPCC.ProcessID, width: 75, sortable: false, },
-            logid: { label: nlsHPCC.Sequence, width: 70, sortable: false, },
-            threadid: { label: nlsHPCC.ThreadID, width: 60, sortable: false, },
+            message: { label: nlsHPCC.Message, width: 600, sortable: false, },
         };
-    }, [wuid]);
+        if (loggingEngine === "grafanacurl") {
+            retVal = Object.assign(retVal, {
+                pod: { label: nlsHPCC.PodName, width: 150, sortable: false },
+            });
+        } else {
+            retVal = Object.assign(retVal, {
+                instance: { label: nlsHPCC.PodName, width: 150, sortable: false },
+                components: { label: nlsHPCC.ContainerName, width: 150, sortable: false },
+                audience: { label: nlsHPCC.Audience, width: 60, sortable: false, },
+                class: {
+                    label: nlsHPCC.Class, width: 40, sortable: false,
+                    formatter: level => {
+                        const colors = logColor(levelMap(level));
+                        const styles = { backgroundColor: colors.background, padding: "2px 6px", color: colors.foreground };
+                        return <span style={styles}>{level}</span>;
+                    }
+                },
+                workunits: { label: nlsHPCC.JobID, width: 50, sortable: false, hidden: wuid !== undefined, },
+                processid: { label: nlsHPCC.ProcessID, width: 75, sortable: false, },
+                logid: { label: nlsHPCC.Sequence, width: 70, sortable: false, },
+                threadid: { label: nlsHPCC.ThreadID, width: 60, sortable: false, },
+            });
+        }
+        return retVal;
+    }, [loggingEngine, wuid]);
 
     const copyButtons = useCopyButtons(columns, selection, "logaccess");